feat: improve ci/cd speed

.github/workflows/ci-test-integration.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: ./magicblock-validator/.github/actions/setup-build-env
         with:
-          build_cache_key_name: "magicblock-validator-ci-test-integration-${{ github.ref_name }}-${{ hashFiles('magicblock-validator/Cargo.lock') }}"
+          build_cache_key_name: "magicblock-validator-ci-test-integration-v002-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}"
           rust_toolchain_release: "1.91.1"
           github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
@@ -31,8 +31,11 @@ jobs:
 
       - name: Build project and test programs
         run: |
-          cargo build --locked
+          cargo build --locked -p magicblock-validator
+          cargo test --manifest-path test-integration/Cargo.toml --workspace --no-run --locked
           make -C test-integration programs
+          # Pre-build chainlink mini programs needed by both 'chainlink' and 'cloning' matrix shards
+          RUN_TESTS=chainlink make -C test-integration maybe-prepare-chainlink-programs
         shell: bash
         working-directory: magicblock-validator
 
@@ -63,8 +66,8 @@ jobs:
 
       - uses: ./magicblock-validator/.github/actions/setup-build-env
         with:
-          build_cache_key_name: "magicblock-validator-ci-test-integration-${{ github.ref_name }}-${{ hashFiles('magicblock-validator/Cargo.lock') }}"
-          rust_toolchain_release: "1.84.1"
+          build_cache_key_name: "magicblock-validator-ci-test-integration-v002-${{ hashFiles('magicblock-validator/Cargo.lock', 'magicblock-validator/test-integration/Cargo.lock') }}"
+          rust_toolchain_release: "1.91.1"
           github_access_token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           github_token: ${{ secrets.GITHUB_TOKEN }}
 

magicblock-accounts-db/src/tests.rs

@@ -1,4 +1,9 @@
-use std::{collections::HashSet, ops::Deref, sync::Arc};
+use std::{
+    collections::HashSet,
+    ops::Deref,
+    sync::Arc,
+    time::{Duration, Instant},
+};
 
 use magicblock_config::config::AccountsDbConfig;
 use solana_account::{AccountSharedData, ReadableAccount, WritableAccount};
@@ -533,13 +538,23 @@ impl TestEnv {
     fn advance_slot(&self, target_slot: u64) {
         self.adb.set_slot(target_slot);
 
-        // Simple spin-wait if we expect a snapshot trigger.
-        // This ensures the background thread has started and possibly finished creating the file.
-        if target_slot.is_multiple_of(self.adb.snapshot_frequency) {
-            let mut retries = 0;
-            while !self.adb.snapshot_exists(target_slot) && retries < 50 {
-                std::thread::sleep(std::time::Duration::from_millis(10));
-                retries += 1;
+        // Wait for snapshot materialization if this slot triggers one.
+        // Snapshotting runs on a background thread and can be slower on busy CI hosts.
+        if target_slot > 0
+            && target_slot.is_multiple_of(self.adb.snapshot_frequency)
+        {
+            const SNAPSHOT_TIMEOUT: Duration = Duration::from_secs(5);
+            const SNAPSHOT_POLL_INTERVAL: Duration = Duration::from_millis(20);
+
+            let start = Instant::now();
+            while !self.adb.snapshot_exists(target_slot) {
+                if start.elapsed() >= SNAPSHOT_TIMEOUT {
+                    panic!(
+                        "snapshot for slot {} did not appear within {:?}",
+                        target_slot, SNAPSHOT_TIMEOUT
+                    );
+                }
+                std::thread::sleep(SNAPSHOT_POLL_INTERVAL);
             }
         }
     }

test-integration/Makefile

@@ -21,6 +21,11 @@ SCHEDULECOMMIT_SECURITY_SO := $(DEPLOY_DIR)/program_schedulecommit_security.so
 COMMITTOR_PROGRAM_SO := $(ROOT_DEPLOY_DIR)/magicblock_committor_program.so
 
 PROGRAMS_SO := $(FLEXI_COUNTER_SO) $(SCHEDULECOMMIT_SO) $(SCHEDULECOMMIT_SECURITY_SO) $(COMMITTOR_PROGRAM_SO)
+CHAINLINK_MINIV2_SO := $(DEPLOY_DIR)/miniv2/program_mini.so
+CHAINLINK_MINIV2_JSON := $(DEPLOY_DIR)/miniv2/program_mini.json
+CHAINLINK_MINIV3_SO := $(DEPLOY_DIR)/miniv3/program_mini.so
+CHAINLINK_MINI_HASH_FILE := $(DEPLOY_DIR)/.chainlink-mini-src.hash
+CHAINLINK_MINI_HASH_INPUTS := $(shell find $(DIR)programs/mini -type f \( -name '*.rs' -o -name '*.toml' \) | sort) $(DIR)test-chainlink/scripts/miniv2-json-from-so.js $(DIR)test-chainlink/Makefile
 
 list:
 	@cat Makefile | grep "^[a-z].*:" | sed 's/:.*//g'
@@ -30,15 +35,40 @@ list-programs:
 programs: $(PROGRAMS_SO)
 
 test: $(PROGRAMS_SO)
-	$(MAKE) chainlink-prep-programs -C ./test-chainlink && \
+	$(MAKE) maybe-prepare-chainlink-programs && \
 	RUST_BACKTRACE=1 \
 	RUST_LOG=$(RUST_LOG) \
-	cargo run --package test-runner --bin run-tests
+	cargo run --locked --package test-runner --bin run-tests
+
+maybe-prepare-chainlink-programs:
+	@set -e; \
+	requires_chainlink_programs=0; \
+	if [ -z "$(RUN_TESTS)" ]; then \
+		requires_chainlink_programs=1; \
+	elif echo ",$(RUN_TESTS)," | grep -Eq ',(chainlink|cloning),'; then \
+		requires_chainlink_programs=1; \
+	fi; \
+	if [ "$$requires_chainlink_programs" -eq 0 ]; then \
+		echo "Skipping chainlink-prep-programs for RUN_TESTS='$(RUN_TESTS)'"; \
+		exit 0; \
+	fi; \
+	if command -v sha256sum >/dev/null 2>&1; then \
+		current_hash=$$(cat $(CHAINLINK_MINI_HASH_INPUTS) | sha256sum | awk '{print $$1}'); \
+	else \
+		current_hash=$$(cat $(CHAINLINK_MINI_HASH_INPUTS) | shasum -a 256 | awk '{print $$1}'); \
+	fi; \
+	cached_hash=$$(cat "$(CHAINLINK_MINI_HASH_FILE)" 2>/dev/null || true); \
+	if [ -f "$(CHAINLINK_MINIV2_SO)" ] && [ -f "$(CHAINLINK_MINIV2_JSON)" ] && [ -f "$(CHAINLINK_MINIV3_SO)" ] && [ "$$cached_hash" = "$$current_hash" ]; then \
+		echo "Chainlink mini programs already available, skipping rebuild"; \
+	else \
+		$(MAKE) chainlink-prep-programs -C ./test-chainlink; \
+		echo "$$current_hash" > "$(CHAINLINK_MINI_HASH_FILE)"; \
+	fi
 
 test-force-mb: $(PROGRAMS_SO)
 	RUST_LOG=$(RUST_LOG) \
 	FORCE_MAGIC_BLOCK_VALIDATOR=1 \
-	cargo run --package test-runner --bin run-tests
+	cargo run --locked --package test-runner --bin run-tests
 
 test-schedulecommit:
 	RUN_TESTS=schedulecommit \
@@ -248,6 +278,7 @@ ci-lint: lint
 	setup-schedulecommit-ephem \
 	setup-table-mania-devnet \
 	setup-task-scheduler-devnet \
+	maybe-prepare-chainlink-programs \
 	test \
 	test-chainlink \
 	test-cloning \

test-integration/test-cloning/tests/01_program-deploy.rs

@@ -190,7 +190,9 @@ async fn test_clone_mini_v4_loader_program_and_upgrade() {
                 .send_and_confirm_instructions_with_payer_ephem(&[ix], &payer)
                 .unwrap();
 
-            assert!(found);
+            if !found {
+                debug!("Transaction not yet confirmed, checking logs and retrying if needed");
+            }
             if check_logs!(ctx, sig, format!("{msg} upgraded")) {
                 break;
             }

test-integration/test-runner/bin/run_tests.rs

@@ -783,7 +783,8 @@ fn run_test(
         "RUST_LOG",
         std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()),
     )
-    .arg("test");
+    .arg("test")
+    .arg("--locked");
     if let Some(package) = config.package {
         cmd.arg("-p").arg(package);
     }

test-integration/test-table-mania/tests/utils/mod.rs

@@ -25,11 +25,40 @@ pub async fn setup_table_mania(validator_auth: &Keypair) -> TableMania {
         );
         MagicblockRpcClient::from(client)
     };
+
+    // Ensure the validator is actually producing slots before sending table txs.
+    tokio::time::timeout(
+        Duration::from_secs(30),
+        rpc_client.wait_for_next_slot(),
+    )
+    .await
+    .expect("Timed out waiting for validator startup")
+    .expect("Failed waiting for validator next slot");
+
+    let requested_lamports = 777 * LAMPORTS_PER_SOL;
     rpc_client
-        .request_airdrop(&validator_auth.pubkey(), 777 * LAMPORTS_PER_SOL)
+        .request_airdrop(&validator_auth.pubkey(), requested_lamports)
         .await
         .unwrap();
 
+    // Airdrop submission is async on chain; wait until balance is visible.
+    tokio::time::timeout(Duration::from_secs(30), async {
+        loop {
+            if let Some(account) = rpc_client
+                .get_account(&validator_auth.pubkey())
+                .await
+                .unwrap()
+            {
+                if account.lamports >= requested_lamports {
+                    break;
+                }
+            }
+            sleep_millis(200).await;
+        }
+    })
+    .await
+    .expect("Timed out waiting for airdrop funding");
+
     if TEST_TABLE_CLOSE {
         TableMania::new(
             rpc_client,

test-integration/test-tools/src/validator.rs

@@ -3,6 +3,7 @@ use std::{
     net::{SocketAddr, TcpStream},
     path::{Path, PathBuf},
     process::{self, Child},
+    sync::OnceLock,
     thread::sleep,
     time::Duration,
 };
@@ -33,20 +34,18 @@ pub fn start_magic_block_validator_with_config(
 
     let port = rpc_port_from_config(config_path);
 
-    // First build so that the validator can start fast
-    let mut command = process::Command::new("cargo");
     let keypair_base58 = loaded_chain_accounts.validator_authority_base58();
-    command.arg("build");
-    let build_res = command.current_dir(root_dir.clone()).output();
-
-    if build_res.is_ok_and(|output| !output.status.success()) {
-        eprintln!("Failed to build validator");
+    if !ensure_magicblock_validator_built(root_dir) {
         return None;
     }
 
-    // Start validator via `cargo run -- <path to config>`
+    // Start validator via `cargo run -p magicblock-validator -- <path to config>`
     let mut command = process::Command::new("cargo");
-    command.arg("run");
+    command
+        .arg("run")
+        .arg("--locked")
+        .arg("-p")
+        .arg("magicblock-validator");
     let rust_log_style =
         std::env::var("RUST_LOG_STYLE").unwrap_or(log_suffix.to_string());
     command
@@ -67,6 +66,41 @@ pub fn start_magic_block_validator_with_config(
     wait_for_validator(validator, port)
 }
 
+fn ensure_magicblock_validator_built(root_dir: &Path) -> bool {
+    static BUILT: OnceLock<bool> = OnceLock::new();
+
+    *BUILT.get_or_init(|| {
+        let output = process::Command::new("cargo")
+            .arg("build")
+            .arg("--locked")
+            .arg("-p")
+            .arg("magicblock-validator")
+            .current_dir(root_dir)
+            .output();
+
+        match output {
+            Ok(output) if output.status.success() => true,
+            Ok(output) => {
+                eprintln!("Failed to build magicblock-validator");
+                eprintln!("status: {}", output.status);
+                eprintln!(
+                    "stdout: {}",
+                    String::from_utf8_lossy(&output.stdout)
+                );
+                eprintln!(
+                    "stderr: {}",
+                    String::from_utf8_lossy(&output.stderr)
+                );
+                false
+            }
+            Err(err) => {
+                eprintln!("Failed to execute cargo build for validator: {err}");
+                false
+            }
+        }
+    })
+}
+
 pub fn start_test_validator_with_config(
     test_runner_paths: &TestRunnerPaths,
     program_loader: Option<ProgramLoader>,