diff --git a/.github/workflows/ci-fmt.yml b/.github/workflows/ci-fmt.yml deleted file mode 100644 index 7982fb0ba..000000000 --- a/.github/workflows/ci-fmt.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Run CI - Format - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - run_make_ci_format: - runs-on: ubuntu-latest - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v2 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-ci-fmt-v001" - rust_toolchain_release: "nightly" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: make ci-fmt - shell: bash - working-directory: magicblock-validator - - - name: Run ci-fmt in test-integration - run: | - cd test-integration - make ci-fmt - shell: bash - working-directory: magicblock-validator diff --git a/.github/workflows/ci-lint.yml b/.github/workflows/ci-lint.yml deleted file mode 100644 index 7a83a1ade..000000000 --- a/.github/workflows/ci-lint.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Run CI - Lint - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - run_make_ci_lint: - runs-on: ubuntu-latest - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v2 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-ci-lint-v002" - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - run: make ci-lint - shell: bash - working-directory: magicblock-validator - - - name: Run ci-lint in test-integration - run: | - cd test-integration - make ci-lint - shell: bash - working-directory: magicblock-validator diff --git a/.github/workflows/ci-test-integration.yml b/.github/workflows/ci-test-integration.yml deleted file mode 100644 index 5980aaec4..000000000 --- a/.github/workflows/ci-test-integration.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Run CI - Integration Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - build: - runs-on: ubuntu-latest-m - name: Build Project - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v5 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-ci-test-integration-${{ github.ref_name }}-${{ hashFiles('magicblock-validator/Cargo.lock') }}" - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./magicblock-validator/.github/actions/setup-solana - - - name: Build project and test programs - run: | - cargo build --locked - make -C test-integration programs - shell: bash - working-directory: magicblock-validator - - run_integration_tests: - needs: build - runs-on: ubuntu-latest-m - strategy: - matrix: - batch_tests: - - "schedulecommit" - - "chainlink" - - "cloning" - - "restore_ledger" - - "magicblock_api" - - "config" - - "table_mania" - - "committor" - - "pubsub" - - "schedule_intents" - - "task-scheduler" - fail-fast: false - name: Integration Tests - ${{ matrix.batch_tests }} - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v5 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-ci-test-integration-${{ github.ref_name }}-${{ hashFiles('magicblock-validator/Cargo.lock') }}" - rust_toolchain_release: "1.84.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./magicblock-validator/.github/actions/setup-solana - - - name: Run integration tests - ${{ matrix.batch_tests }} - run: | - sudo prlimit --pid $$ --nofile=1048576:1048576 - sudo sysctl fs.inotify.max_user_instances=1280 - sudo sysctl fs.inotify.max_user_watches=655360 - make ci-test-integration - shell: bash - working-directory: magicblock-validator - env: - RUN_TESTS: ${{ matrix.batch_tests }} diff --git a/.github/workflows/ci-test-unit.yml b/.github/workflows/ci-test-unit.yml deleted file mode 100644 index 8061769fe..000000000 --- a/.github/workflows/ci-test-unit.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Run CI - Unit Tests - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -on: - push: - branches: [master, dev] - pull_request: - types: [opened, reopened, synchronize, ready_for_review] - -jobs: - run_make_ci_test: - runs-on: ubuntu-latest-m - steps: - - name: Checkout this magicblock-validator - uses: actions/checkout@v2 - with: - path: magicblock-validator - - - uses: ./magicblock-validator/.github/actions/setup-build-env - with: - build_cache_key_name: "magicblock-validator-ci-test-unit-v001" - rust_toolchain_release: "1.91.1" - github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - - uses: ./magicblock-validator/.github/actions/setup-solana - - - name: Run unit tests - run: | - sudo prlimit --pid $$ --nofile=1048576:1048576 - sudo sysctl fs.inotify.max_user_instances=1280 - sudo sysctl fs.inotify.max_user_watches=655360 - make ci-test-unit - shell: bash - working-directory: magicblock-validator diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..8b2e94fb8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,174 @@ +name: CI + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +on: + push: + branches: [master, dev] + pull_request: + types: [opened, reopened, synchronize] + +jobs: + prepare-env: + runs-on: self-hosted + outputs: + target_dir: ${{ steps.calc.outputs.dir }} + steps: + - id: calc + env: + REF_NAME: ${{ github.ref_name }} + REPO_NAME: ${{ github.event.repository.name }} + run: | + SAFE_BRANCH=$(echo "$REF_NAME" | tr '/' '-') + TARGET_DIR="/var/lib/gh-runners/global-targets/${REPO_NAME}-${SAFE_BRANCH}" + mkdir -p "$TARGET_DIR" + echo "dir=$TARGET_DIR" >> $GITHUB_OUTPUT + + build: + needs: prepare-env + runs-on: self-hosted + env: + CARGO_TARGET_DIR: ${{ needs.prepare-env.outputs.target_dir }} + CARGO_INCREMENTAL: 1 + CARGO_BUILD_JOBS: 30 + RUSTFLAGS: -C link-arg=-fuse-ld=mold + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + + - name: Build SBF programs + env: + RUSTFLAGS: "" + run: | + ( + unset CARGO_TARGET_DIR + make -C test-integration programs + make chainlink-prep-programs -C test-integration/test-chainlink + ) + + - name: Build binaries and tests + run: cargo build --bins --tests --locked + + - name: Format check + run: make ci-fmt + + - name: Lint + run: make ci-lint + + - name: Unit tests + run: make ci-test-unit + + - name: Pre-fetch integration test dependencies + run: cargo test --test '*' --no-run + + - name: Cache SBF artifacts + run: | + SBF_CACHE_DIR="$CARGO_TARGET_DIR/sbf-artifacts" + mkdir -p "$SBF_CACHE_DIR/test-integration" "$SBF_CACHE_DIR/root" + + if [ -d "test-integration/target/deploy" ]; then + find test-integration/target/deploy -mindepth 1 -maxdepth 1 -exec cp -r {} "$SBF_CACHE_DIR/test-integration/" \; + else + echo "::warning::test-integration/target/deploy not found" + fi + + if ls target/deploy/*.so >/dev/null 2>&1; then + cp target/deploy/*.so "$SBF_CACHE_DIR/root/" + else + echo "::warning::No SBF artifacts found in target/deploy" + fi + + integration-tests: + needs: [prepare-env, build] + runs-on: self-hosted + strategy: + fail-fast: false + matrix: + batch: + - "committor config" + - "restore_ledger schedulecommit" + - "table_mania cloning" + - "magicblock_api schedule_intents" + - "task-scheduler chainlink pubsub" + env: + TMPDIR: /dev/shm + CARGO_NET_OFFLINE: "true" + CARGO_BUILD_JOBS: 6 + NEXTEST_RETRIES: 0 + steps: + - uses: actions/checkout@v5 + with: + submodules: recursive + + - name: Restore source file timestamps + run: git restore-mtime + + - name: Link pre-built artifacts + run: | + GLOBAL_TARGET="${{ needs.prepare-env.outputs.target_dir }}" + mkdir -p target + + if ! cp -rl "$GLOBAL_TARGET/debug" target/ 2>/dev/null; then + echo "::warning::Hardlink failed (cross-device?), falling back to copy" + cp -r "$GLOBAL_TARGET/debug" target/ + fi + + rm -rf target/debug/incremental + + - name: Restore SBF artifacts + run: | + SBF_CACHE_DIR="${{ needs.prepare-env.outputs.target_dir }}/sbf-artifacts" + + LOCAL_DEPLOY_TEST="test-integration/target/deploy" + mkdir -p "$LOCAL_DEPLOY_TEST" + if [ -d "$SBF_CACHE_DIR/test-integration" ]; then + find "$SBF_CACHE_DIR/test-integration" -mindepth 1 -maxdepth 1 -exec cp -r {} "$LOCAL_DEPLOY_TEST/" \; + find "$LOCAL_DEPLOY_TEST" -type f -exec touch {} + + else + echo "::warning::No cached SBF artifacts for test-integration" + fi + + LOCAL_DEPLOY_ROOT="target/deploy" + mkdir -p "$LOCAL_DEPLOY_ROOT" + if ls "$SBF_CACHE_DIR/root"/*.so >/dev/null 2>&1; then + cp "$SBF_CACHE_DIR/root"/*.so "$LOCAL_DEPLOY_ROOT/" + touch "$LOCAL_DEPLOY_ROOT"/*.so + else + echo "::warning::No cached SBF artifacts for root" + fi + + - name: "Integration: ${{ matrix.batch }}" + env: + TEST_BATCH: ${{ matrix.batch }} + run: | + unshare -r -n bash -c ' + set -euo pipefail + ip link set lo up + FAILED="" + for test in $TEST_BATCH; do + echo "========== Running: $test ==========" + if ! RUN_TESTS=$test make ci-test-integration PROGRAMS_SO= SKIP_CHAINLINK_PREP=1; then + FAILED="$FAILED $test" + fi + done + if [ -n "$FAILED" ]; then + echo "::error::Failed test suites:$FAILED" + exit 1 + fi + ' + + ci-status: + if: always() + needs: [build, integration-tests] + runs-on: self-hosted + steps: + - name: Verify pipeline status + run: | + if [[ "${{ needs.build.result }}" != "success" || + "${{ needs.integration-tests.result }}" != "success" ]]; then + echo "::error::CI failed" + exit 1 + fi diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index aca45c5eb..3bdd9f237 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -429,7 +429,7 @@ impl MagicValidator { let accounts_bank = accountsdb.clone(); let mut chainlink_config = ChainlinkConfig::default_with_lifecycle_mode( - LifecycleMode::Ephemeral, + config.lifecycle.clone(), ) .with_remove_confined_accounts( config.chainlink.remove_confined_accounts, diff --git a/test-integration/Makefile b/test-integration/Makefile index 95ff8551e..1c82858f2 100644 --- a/test-integration/Makefile +++ b/test-integration/Makefile @@ -3,6 +3,13 @@ DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) DEPLOY_DIR := $(DIR)target/deploy ROOT_DEPLOY_DIR := $(DIR)../target/deploy +# Use CARGO_TARGET_DIR if provided by the environment (CI), +# otherwise default to ../target for local development. +CARGO_TARGET_DIR ?= $(DIR)../target + +# Prebuilt test-runner binary path (dev profile) +TEST_RUNNER_BIN := $(CARGO_TARGET_DIR)/debug/run-tests + RUST_LOG ?= 'warn,geyser_plugin=warn,magicblock=trace,magicblock_chainlink::remote_account_provider::chain_pubsub_actor=debug,rpc=trace,bank=trace,banking_stage=warn,solana_geyser_plugin_manager=warn,solana_svm=warn,test_tools=trace,schedulecommit_test=trace,' FLEXI_COUNTER_DIR := $(DIR)programs/flexi-counter @@ -30,10 +37,21 @@ list-programs: programs: $(PROGRAMS_SO) test: $(PROGRAMS_SO) - $(MAKE) chainlink-prep-programs -C ./test-chainlink && \ - RUST_BACKTRACE=1 \ - RUST_LOG=$(RUST_LOG) \ - cargo run --package test-runner --bin run-tests + @# Only prepare chainlink programs when running chainlink tests + @if [ -z "$(SKIP_CHAINLINK_PREP)" ] && ([ -z "$(RUN_TESTS)" ] || [ "$(RUN_TESTS)" = "chainlink" ]); then \ + echo "Preparing chainlink programs (RUN_TESTS=$(RUN_TESTS))"; \ + $(MAKE) chainlink-prep-programs -C ./test-chainlink; \ + else \ + echo "Skipping chainlink-prep-programs (SKIP_CHAINLINK_PREP=$(SKIP_CHAINLINK_PREP), RUN_TESTS=$(RUN_TESTS))"; \ + fi + @# Prefer running the prebuilt test-runner binary to avoid concurrent Cargo locking + @if [ -x "$(TEST_RUNNER_BIN)" ]; then \ + echo "Using prebuilt test-runner binary: $(TEST_RUNNER_BIN)"; \ + CARGO_TEST_THREADS=$${CARGO_TEST_THREADS:-1} CARGO_MANIFEST_DIR=$(DIR)test-runner RUST_BACKTRACE=1 RUST_LOG=$(RUST_LOG) $(TEST_RUNNER_BIN); \ + else \ + echo "test-runner binary not found, falling back to 'cargo run'"; \ + CARGO_TEST_THREADS=$${CARGO_TEST_THREADS:-1} RUST_BACKTRACE=1 RUST_LOG=$(RUST_LOG) cargo run --package test-runner --bin run-tests; \ + fi test-force-mb: $(PROGRAMS_SO) RUST_LOG=$(RUST_LOG) \ diff --git a/test-integration/test-runner/bin/run_tests.rs b/test-integration/test-runner/bin/run_tests.rs index 6a5a6e9c7..8d117337c 100644 --- a/test-integration/test-runner/bin/run_tests.rs +++ b/test-integration/test-runner/bin/run_tests.rs @@ -784,13 +784,19 @@ fn run_test( std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()), ) .arg("test"); + if std::env::var("CARGO_NET_OFFLINE").as_deref() == Ok("true") { + cmd.arg("--offline"); + } if let Some(package) = config.package { cmd.arg("-p").arg(package); } if let Some(test) = config.test { cmd.arg(format!("'{}'", test)); } - cmd.arg("--").arg("--test-threads=1").arg("--nocapture"); + let test_threads = std::env::var("CARGO_TEST_THREADS").unwrap_or_else(|_| "1".to_string()); + cmd.arg("--") + .arg(format!("--test-threads={}", test_threads)) + .arg("--nocapture"); cmd.current_dir(manifest_dir.clone()); Teepee::new(cmd).output() } diff --git a/test-integration/test-tools/src/toml_to_args.rs b/test-integration/test-tools/src/toml_to_args.rs index 2d9025db8..8bb81cb9d 100644 --- a/test-integration/test-tools/src/toml_to_args.rs +++ b/test-integration/test-tools/src/toml_to_args.rs @@ -80,9 +80,42 @@ pub fn config_to_args( args.push(program.id); - let resolved_full_config_path = - config_dir.join(&program.path).canonicalize().unwrap(); - args.push(resolved_full_config_path.to_str().unwrap().to_string()); + let full_path_to_resolve = config_dir.join(&program.path); + match fs::canonicalize(&full_path_to_resolve) { + Ok(path) => { + args.push(path.to_str().unwrap().to_string()); + } + Err(e) => { + let abs_config_dir = fs::canonicalize(config_dir).unwrap_or(config_dir.to_path_buf()); + eprintln!( + "Error: Failed to resolve program path.\n\ + Config Dir: {:?}\n\ + Relative Path: {:?}\n\ + Resolution Attempt: {:?}\n\ + OS Error: {:?}", + abs_config_dir, + program.path, + full_path_to_resolve, + e + ); + + // List directory contents to aid debugging in CI environments + if let Some(parent) = full_path_to_resolve.parent() { + eprintln!("Directory contents of {:?}:", parent); + if let Ok(entries) = fs::read_dir(parent) { + for entry in entries { + if let Ok(entry) = entry { + eprintln!(" - {:?}", entry.file_name()); + } + } + } else { + eprintln!(" (Unable to read directory)"); + } + } + + panic!("Program file not found: {:?}", full_path_to_resolve); + } + } if program_loader == ProgramLoader::UpgradeableProgram { if let Some(auth) = program.auth { diff --git a/test-integration/test-tools/src/validator.rs b/test-integration/test-tools/src/validator.rs index 6b9080a22..c70eff3e3 100644 --- a/test-integration/test-tools/src/validator.rs +++ b/test-integration/test-tools/src/validator.rs @@ -32,26 +32,37 @@ pub fn start_magic_block_validator_with_config( } = test_runner_paths; let port = rpc_port_from_config(config_path); - - // First build so that the validator can start fast - let mut command = process::Command::new("cargo"); let keypair_base58 = loaded_chain_accounts.validator_authority_base58(); - command.arg("build"); - let build_res = command.current_dir(root_dir.clone()).output(); - if build_res.is_ok_and(|output| !output.status.success()) { - eprintln!("Failed to build validator"); - return None; - } + // Check if we should use a prebuilt binary (faster in CI) + let workspace_dir = resolve_workspace_dir(); + let target_dir = std::env::var("CARGO_TARGET_DIR") + .map(PathBuf::from) + .unwrap_or_else(|_| workspace_dir.join("../target")); + + let validator_bin = target_dir.join("debug/magicblock-validator"); + + let mut command = if validator_bin.exists() { + eprintln!("Using prebuilt validator binary: {:?}", validator_bin); + let mut cmd = process::Command::new(validator_bin); + cmd.arg(config_path); + cmd + } else { + eprintln!("Prebuilt validator not found at {:?}, falling back to cargo run", validator_bin); + // Fallback for local development + let mut cmd = process::Command::new("cargo"); + cmd.arg("run") + .arg("-p") + .arg("magicblock-validator") + .arg("--") + .arg(config_path); + cmd + }; - // Start validator via `cargo run -- ` - let mut command = process::Command::new("cargo"); - command.arg("run"); let rust_log_style = std::env::var("RUST_LOG_STYLE").unwrap_or(log_suffix.to_string()); + command - .arg("--") - .arg(config_path) .env("RUST_LOG_STYLE", rust_log_style) .env("MBV_VALIDATOR__KEYPAIR", keypair_base58.clone()) .current_dir(root_dir);