diff --git a/Cargo.lock b/Cargo.lock index e61e2d104..f20f34d93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2989,6 +2989,7 @@ dependencies = [ "thiserror 1.0.69", "tracing", "tracing-subscriber", + "twox-hash", ] [[package]] @@ -5501,7 +5502,7 @@ dependencies = [ [[package]] name = "solana-account" version = "2.2.1" -source = "git+https://github.com/magicblock-labs/solana-account.git?rev=6eae52b#6eae52bde25e90b3c79d4935ce2b267e35338945" +source = "git+https://github.com/magicblock-labs/solana-account.git?rev=3ff1b2ea#3ff1b2eaf16fd8b82dd8e625673df9798b07db22" dependencies = [ "bincode", "qualifier_attr", @@ -8930,6 +8931,12 @@ dependencies = [ "webpki-roots 0.24.0", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" version = "1.19.0" diff --git a/Cargo.toml b/Cargo.toml index 8d47b19ba..36a7e1aaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -140,7 +140,7 @@ serde_json = "1.0" serde_with = "3.16" serial_test = "3.2" sha3 = "0.10.8" -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" } solana-account-decoder = { version = "2.2" } solana-account-decoder-client-types = { version = "2.2" } solana-account-info = { version = "2.2" } @@ -211,6 +211,10 @@ tonic-build = "0.9.2" tracing = "0.1" tracing-log = { version = "0.2", features = ["log-tracer"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } +twox-hash = { version = "2.1", default-features = false, features = [ + "xxhash3_64", + "alloc", +] } url = "2.5.0" # SPL Token crates used across the workspace @@ -231,7 +235,7 @@ version = "0.22.0" # some solana dependencies have solana-storage-proto as dependency # we need to patch them with our version, because they use protobuf-src v1.1.0 # and we use protobuf-src v2.1.1. Otherwise compilation fails -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" } solana-storage-proto = { path = "./storage-proto" } solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "bdbaac86" } # Fork is used to enable `disable_manual_compaction` usage diff --git a/magicblock-accounts-db/Cargo.toml b/magicblock-accounts-db/Cargo.toml index 981ef200f..2a854cd11 100644 --- a/magicblock-accounts-db/Cargo.toml +++ b/magicblock-accounts-db/Cargo.toml @@ -23,6 +23,9 @@ thiserror = { workspace = true } tracing = { workspace = true } magicblock-config = { workspace = true } +# misc +twox-hash = { workspace = true } + [dev-dependencies] tempfile = { workspace = true } tracing-subscriber = { workspace = true } diff --git a/magicblock-accounts-db/src/lib.rs b/magicblock-accounts-db/src/lib.rs index b1437a24a..f2826bc80 100644 --- a/magicblock-accounts-db/src/lib.rs +++ b/magicblock-accounts-db/src/lib.rs @@ -1,4 +1,4 @@ -use std::{fs, path::Path, sync::Arc, thread}; +use std::{fs, hash::Hasher, path::Path, sync::Arc, thread}; use error::{AccountsDbError, LogErr}; use index::{ @@ -13,6 +13,7 @@ use solana_account::{ use solana_pubkey::Pubkey; use storage::AccountsStorage; use tracing::{error, info, warn}; +use twox_hash::xxhash3_64; use crate::{snapshot::SnapshotManager, traits::AccountsBank}; @@ -374,6 +375,26 @@ impl AccountsDb { pub fn write_lock(&self) -> GlobalSyncLock { self.write_lock.clone() } + + /// Computes a deterministic checksum of all active accounts. + /// + /// Iterates all accounts in key-sorted order (via LMDB) and hashes both + /// pubkey and serialized account data using xxHash3. Returns a 64-bit hash + /// suitable for verifying state consistency across nodes. + /// + /// Acquires the write lock to ensure a consistent snapshot of the state. + pub fn checksum(&self) -> u64 { + let _locked = self.write_lock.write(); + let mut hasher = xxhash3_64::Hasher::new(); + for (pubkey, acc) in self.iter_all() { + let Some(borrowed) = acc.as_borrowed() else { + continue; + }; + hasher.write(pubkey.as_ref()); + hasher.write(borrowed.buffer()); + } + hasher.finish() + } } impl AccountsBank for AccountsDb { diff --git a/magicblock-accounts-db/src/storage.rs b/magicblock-accounts-db/src/storage.rs index 69dce7db3..9a541a9e8 100644 --- a/magicblock-accounts-db/src/storage.rs +++ b/magicblock-accounts-db/src/storage.rs @@ -371,10 +371,9 @@ impl AccountsStorage { Ok(()) } - /// Returns the total expected size of the file in bytes (Header + Data). + /// Returns the total occupied size of the storage file in bytes. pub(crate) fn size_bytes(&self) -> u64 { - (self.header().capacity_blocks as u64 * self.block_size as u64) - + METADATA_STORAGE_SIZE as u64 + self.active_segment().len() as u64 } pub(crate) fn block_size(&self) -> usize { diff --git a/magicblock-accounts-db/src/tests.rs b/magicblock-accounts-db/src/tests.rs index aba17258f..49d5e0303 100644 --- a/magicblock-accounts-db/src/tests.rs +++ b/magicblock-accounts-db/src/tests.rs @@ -464,6 +464,69 @@ fn test_database_reset() { assert_eq!(adb_reset.account_count(), 0); } +#[test] +fn test_checksum_deterministic_across_dbs() { + // Two independent DBs with identical accounts must produce identical checksums + let dir1 = tempfile::tempdir().unwrap(); + let dir2 = tempfile::tempdir().unwrap(); + let config = AccountsDbConfig::default(); + + let db1 = AccountsDb::new(&config, dir1.path(), 0).unwrap(); + let db2 = AccountsDb::new(&config, dir2.path(), 0).unwrap(); + + // Insert same accounts into both DBs + for i in 0..50 { + let pubkey = Pubkey::new_unique(); + let mut account = AccountSharedData::new(LAMPORTS, SPACE, &OWNER); + account.data_as_mut_slice()[..8] + .copy_from_slice(&(i as u64).to_le_bytes()); + db1.insert_account(&pubkey, &account).unwrap(); + db2.insert_account(&pubkey, &account).unwrap(); + } + + assert_eq!( + db1.checksum(), + db2.checksum(), + "checksums must match for identical state" + ); +} + +#[test] +fn test_checksum_detects_state_change() { + let env = TestEnv::new(); + + // Create initial state + let mut accounts: Vec<_> = (0..20) + .map(|_| { + let acc = env.create_and_insert_account(); + (acc.pubkey, acc.account) + }) + .collect(); + + let original_checksum = env.checksum(); + + // Modify a single account's data + accounts[5].1.data_as_mut_slice()[0] ^= 0xFF; + env.insert_account(&accounts[5].0, &accounts[5].1).unwrap(); + + assert_ne!( + env.checksum(), + original_checksum, + "checksum must detect single account modification" + ); + + // Modify lamports on a different account + accounts[10].1.set_lamports(1_000_000); + env.insert_account(&accounts[10].0, &accounts[10].1) + .unwrap(); + + assert_ne!( + env.checksum(), + original_checksum, + "checksum must detect lamport change" + ); +} + // ============================================================== // TEST UTILITIES // ============================================================== diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index e372ed370..95ffb18d7 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -2251,10 +2251,10 @@ dependencies = [ [[package]] name = "guinea" -version = "0.7.0" +version = "0.8.0" dependencies = [ "bincode", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "serde", "solana-program", ] @@ -3273,7 +3273,7 @@ dependencies = [ [[package]] name = "magicblock-account-cloner" -version = "0.7.0" +version = "0.8.0" dependencies = [ "async-trait", "bincode", @@ -3283,7 +3283,7 @@ dependencies = [ "magicblock-config", "magicblock-core", "magicblock-ledger", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "magicblock-program", "magicblock-rpc-client", "rand 0.9.2", @@ -3305,7 +3305,7 @@ dependencies = [ [[package]] name = "magicblock-accounts" -version = "0.7.0" +version = "0.8.0" dependencies = [ "async-trait", "magicblock-account-cloner", @@ -3327,7 +3327,7 @@ dependencies = [ [[package]] name = "magicblock-accounts-db" -version = "0.7.0" +version = "0.8.0" dependencies = [ "lmdb-rkv", "magicblock-config", @@ -3338,11 +3338,12 @@ dependencies = [ "solana-pubkey", "thiserror 1.0.69", "tracing", + "twox-hash", ] [[package]] name = "magicblock-aperture" -version = "0.7.0" +version = "0.8.0" dependencies = [ "agave-geyser-plugin-interface", "arc-swap", @@ -3390,7 +3391,7 @@ dependencies = [ [[package]] name = "magicblock-api" -version = "0.7.0" +version = "0.8.0" dependencies = [ "anyhow", "borsh 1.6.0", @@ -3405,7 +3406,7 @@ dependencies = [ "magicblock-config", "magicblock-core", "magicblock-ledger", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "magicblock-metrics", "magicblock-processor", "magicblock-program", @@ -3445,7 +3446,7 @@ dependencies = [ [[package]] name = "magicblock-chainlink" -version = "0.7.0" +version = "0.8.0" dependencies = [ "arc-swap", "async-trait", @@ -3457,7 +3458,7 @@ dependencies = [ "magicblock-config", "magicblock-core", "magicblock-delegation-program 1.1.3 (git+https://github.com/magicblock-labs/delegation-program.git?rev=2cb491032f372)", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "magicblock-metrics", "parking_lot", "scc", @@ -3499,7 +3500,7 @@ dependencies = [ [[package]] name = "magicblock-committor-program" -version = "0.7.0" +version = "0.8.0" dependencies = [ "borsh 1.6.0", "paste", @@ -3511,7 +3512,7 @@ dependencies = [ [[package]] name = "magicblock-committor-service" -version = "0.7.0" +version = "0.8.0" dependencies = [ "async-trait", "base64 0.21.7", @@ -3555,7 +3556,7 @@ dependencies = [ [[package]] name = "magicblock-config" -version = "0.7.0" +version = "0.8.0" dependencies = [ "clap", "derive_more", @@ -3573,10 +3574,10 @@ dependencies = [ [[package]] name = "magicblock-core" -version = "0.7.0" +version = "0.8.0" dependencies = [ "flume", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "solana-account", "solana-account-decoder", "solana-hash", @@ -3628,12 +3629,12 @@ dependencies = [ "solana-security-txt", "static_assertions", "strum 0.27.2", - "thiserror 1.0.69", + "thiserror 2.0.17", ] [[package]] name = "magicblock-ledger" -version = "0.7.0" +version = "0.8.0" dependencies = [ "arc-swap", "bincode", @@ -3684,7 +3685,7 @@ dependencies = [ [[package]] name = "magicblock-magic-program-api" -version = "0.7.0" +version = "0.8.0" dependencies = [ "bincode", "serde", @@ -3693,7 +3694,7 @@ dependencies = [ [[package]] name = "magicblock-metrics" -version = "0.7.0" +version = "0.8.0" dependencies = [ "http-body-util", "hyper 1.8.1", @@ -3707,7 +3708,7 @@ dependencies = [ [[package]] name = "magicblock-processor" -version = "0.7.0" +version = "0.8.0" dependencies = [ "bincode", "magicblock-accounts-db", @@ -3743,12 +3744,12 @@ dependencies = [ [[package]] name = "magicblock-program" -version = "0.7.0" +version = "0.8.0" dependencies = [ "bincode", "lazy_static", "magicblock-core", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "num-derive", "num-traits", "parking_lot", @@ -3760,6 +3761,8 @@ dependencies = [ "solana-hash", "solana-instruction", "solana-keypair", + "solana-loader-v3-interface 3.0.0", + "solana-loader-v4-interface", "solana-log-collector", "solana-program-runtime", "solana-pubkey", @@ -3775,7 +3778,7 @@ dependencies = [ [[package]] name = "magicblock-rpc-client" -version = "0.7.0" +version = "0.8.0" dependencies = [ "solana-account", "solana-account-decoder-client-types", @@ -3797,7 +3800,7 @@ dependencies = [ [[package]] name = "magicblock-table-mania" -version = "0.7.0" +version = "0.8.0" dependencies = [ "ed25519-dalek", "magicblock-metrics", @@ -3823,7 +3826,7 @@ dependencies = [ [[package]] name = "magicblock-task-scheduler" -version = "0.7.0" +version = "0.8.0" dependencies = [ "bincode", "chrono", @@ -3849,7 +3852,7 @@ dependencies = [ [[package]] name = "magicblock-validator-admin" -version = "0.7.0" +version = "0.8.0" dependencies = [ "magicblock-delegation-program 1.1.3 (git+https://github.com/magicblock-labs/delegation-program.git?rev=2cb491032f372)", "magicblock-program", @@ -3866,7 +3869,7 @@ dependencies = [ [[package]] name = "magicblock-version" -version = "0.7.0" +version = "0.8.0" dependencies = [ "git-version", "rustc_version", @@ -4766,7 +4769,7 @@ dependencies = [ "bincode", "borsh 1.6.0", "ephemeral-rollups-sdk", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "serde", "solana-program", ] @@ -4790,7 +4793,7 @@ dependencies = [ "borsh 1.6.0", "ephemeral-rollups-sdk", "magicblock-delegation-program 1.1.3 (git+https://github.com/magicblock-labs/delegation-program.git?rev=2cb491032f372)", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "rkyv 0.7.45", "solana-program", "static_assertions", @@ -5824,7 +5827,7 @@ dependencies = [ "ephemeral-rollups-sdk", "integration-test-tools", "magicblock-core", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "program-schedulecommit", "rand 0.8.5", "schedulecommit-client", @@ -5842,7 +5845,7 @@ version = "0.0.0" dependencies = [ "integration-test-tools", "magicblock-core", - "magicblock-magic-program-api 0.7.0", + "magicblock-magic-program-api 0.8.0", "program-schedulecommit", "program-schedulecommit-security", "schedulecommit-client", @@ -6243,7 +6246,7 @@ dependencies = [ [[package]] name = "solana-account" version = "2.2.1" -source = "git+https://github.com/magicblock-labs/solana-account.git?rev=6eae52b#6eae52bde25e90b3c79d4935ce2b267e35338945" +source = "git+https://github.com/magicblock-labs/solana-account.git?rev=3ff1b2ea#3ff1b2eaf16fd8b82dd8e625673df9798b07db22" dependencies = [ "bincode", "qualifier_attr", @@ -8561,7 +8564,7 @@ dependencies = [ [[package]] name = "solana-storage-proto" -version = "0.7.0" +version = "0.8.0" dependencies = [ "bincode", "bs58", @@ -10062,7 +10065,7 @@ dependencies = [ [[package]] name = "test-kit" -version = "0.7.0" +version = "0.8.0" dependencies = [ "guinea", "magicblock-accounts-db", @@ -10750,6 +10753,12 @@ dependencies = [ "webpki-roots 0.24.0", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" version = "1.19.0" diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index 0690ffee4..578353048 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -78,7 +78,7 @@ rkyv = "0.7.45" schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" serial_test = "3.2.0" -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" } solana-loader-v2-interface = "2.2" solana-loader-v3-interface = "4.0" solana-loader-v4-interface = "2.1" @@ -114,4 +114,4 @@ url = "2.5.0" solana-storage-proto = { path = "../storage-proto" } # same reason as above rocksdb = { git = "https://github.com/magicblock-labs/rust-rocksdb.git", rev = "6d975197" } -solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "6eae52b" } +solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "3ff1b2ea" }