diff --git a/Cargo.lock b/Cargo.lock index 5285f84d8..09df25098 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -54,6 +54,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -513,6 +524,18 @@ dependencies = [ "serde", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.8.2" @@ -659,6 +682,28 @@ dependencies = [ "serde", ] +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta 0.1.4", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytemuck" version = "1.23.1" @@ -1345,7 +1390,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6503af7917fea18ffef8f7e8553fb8dff89e2e6837e94e09dd7fb069c82d62c" dependencies = [ "bytes", - "rkyv", + "rkyv 0.8.11", "serde", "simdutf8", ] @@ -1492,6 +1537,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.31" @@ -1741,6 +1792,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -1748,7 +1802,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash", + "ahash 0.8.12", ] [[package]] @@ -2927,21 +2981,23 @@ dependencies = [ [[package]] name = "magicblock-delegation-program" -version = "1.1.0" -source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=aa1de56d90c#aa1de56d90c8a242377accd59899f272f0131f8c" +version = "1.1.3" +source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=3edb41022#3edb41022afb0e813607ffd4e79d1548131eab2b" dependencies = [ "bincode", "borsh 1.5.7", "bytemuck", "num_enum", - "paste", "pinocchio", "pinocchio-log", "pinocchio-pubkey", "pinocchio-system", + "rkyv 0.7.45", "solana-curve25519", "solana-program", "solana-security-txt", + "static_assertions", + "strum", "thiserror 1.0.69", ] @@ -3996,13 +4052,33 @@ dependencies = [ "autotools", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive 0.1.4", +] + [[package]] name = "ptr_meta" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90" dependencies = [ - "ptr_meta_derive", + "ptr_meta_derive 0.3.0", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -4051,13 +4127,19 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rancor" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947" dependencies = [ - "ptr_meta", + "ptr_meta 0.3.0", ] [[package]] @@ -4245,6 +4327,15 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "rend" version = "0.5.2" @@ -4327,6 +4418,24 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta 0.1.4", + "rend 0.4.2", + "rkyv_derive 0.7.45", + "seahash", + "tinyvec", + "uuid", +] + [[package]] name = "rkyv" version = "0.8.11" @@ -4337,14 +4446,25 @@ dependencies = [ "hashbrown 0.15.4", "indexmap 2.10.0", "munge", - "ptr_meta", + "ptr_meta 0.3.0", "rancor", - "rend", - "rkyv_derive", + "rend 0.5.2", + "rkyv_derive 0.8.11", "tinyvec", "uuid", ] +[[package]] +name = "rkyv_derive" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rkyv_derive" version = "0.8.11" @@ -4539,6 +4659,12 @@ version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "2.11.1" @@ -5082,7 +5208,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4ee734c35b736e632aa3b1367f933d93ee7b4129dd1e20ca942205d4834054e" dependencies = [ - "ahash", + "ahash 0.8.12", "lazy_static", "log", "qualifier_attr", @@ -5395,7 +5521,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e1d3b52b4a014efeaaab67f14e40af3972a4be61c523d612860db8e3145529" dependencies = [ - "ahash", + "ahash 0.8.12", "lazy_static", "solana-epoch-schedule", "solana-hash", @@ -6627,7 +6753,7 @@ name = "solana-svm" version = "2.2.1" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=3e9456ec4#3e9456ec4d5798ad8281537501c1e777d6888ba3" dependencies = [ - "ahash", + "ahash 0.8.12", "log", "percentage", "qualifier_attr", @@ -7070,7 +7196,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd1adc42def3cb101f3ebef3cd2d642f9a21072bbcd4ec9423343ccaa6afa596" dependencies = [ - "ahash", + "ahash 0.8.12", "bumpalo", "bytes", "cfg-if", @@ -7509,6 +7635,27 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "subtle" version = "2.6.1" @@ -7590,6 +7737,12 @@ dependencies = [ "unicode-width 0.1.14", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "task-local-extensions" version = "0.1.4" @@ -8777,6 +8930,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index 05fe94cc5..e0673688f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,7 +95,7 @@ magicblock-committor-program = { path = "./magicblock-committor-program", featur magicblock-committor-service = { path = "./magicblock-committor-service" } magicblock-config = { path = "./magicblock-config" } magicblock-core = { path = "./magicblock-core" } -magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "aa1de56d90c", features = [ +magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "3edb41022" , features = [ "no-entrypoint", ] } magicblock-ledger = { path = "./magicblock-ledger" } diff --git a/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs b/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs index a278a01fd..be7ea5161 100644 --- a/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs +++ b/magicblock-committor-service/src/intent_executor/task_info_fetcher.rs @@ -10,6 +10,7 @@ use log::warn; use lru::LruCache; use magicblock_metrics::metrics; use magicblock_rpc_client::{MagicBlockRpcClientError, MagicblockRpcClient}; +use solana_account::Account; use solana_pubkey::Pubkey; use solana_signature::Signature; @@ -36,6 +37,11 @@ pub trait TaskInfoFetcher: Send + Sync + 'static { /// Resets cache for some or all accounts fn reset(&self, reset_type: ResetType); + + async fn get_base_accounts( + &self, + pubkeys: &[Pubkey], + ) -> TaskInfoFetcherResult>; } pub enum ResetType<'a> { @@ -263,6 +269,25 @@ impl TaskInfoFetcher for CacheTaskInfoFetcher { } } } + + async fn get_base_accounts( + &self, + pubkeys: &[Pubkey], + ) -> TaskInfoFetcherResult> { + self.rpc_client + .get_multiple_accounts(pubkeys, None) + .await + .map_err(|err| { + TaskInfoFetcherError::MagicBlockRpcClientError(Box::new(err)) + }) + .map(|accounts| { + pubkeys + .iter() + .zip(accounts.into_iter()) + .filter_map(|(key, value)| value.map(|value| (*key, value))) + .collect() + }) + } } #[derive(thiserror::Error, Debug)] diff --git a/magicblock-committor-service/src/tasks/args_task.rs b/magicblock-committor-service/src/tasks/args_task.rs index 8919999c0..1bd5f1e5d 100644 --- a/magicblock-committor-service/src/tasks/args_task.rs +++ b/magicblock-committor-service/src/tasks/args_task.rs @@ -1,5 +1,9 @@ -use dlp::args::{CallHandlerArgs, CommitStateArgs}; +use dlp::{ + args::{CallHandlerArgs, CommitDiffArgs, CommitStateArgs}, + compute_diff, +}; use magicblock_metrics::metrics::LabelValue; +use solana_account::ReadableAccount; use solana_instruction::{AccountMeta, Instruction}; use solana_pubkey::Pubkey; @@ -8,14 +12,15 @@ use crate::tasks::TaskStrategy; use crate::tasks::{ buffer_task::{BufferTask, BufferTaskType}, visitor::Visitor, - BaseActionTask, BaseTask, BaseTaskError, BaseTaskResult, CommitTask, - FinalizeTask, PreparationState, TaskType, UndelegateTask, + BaseActionTask, BaseTask, BaseTaskError, BaseTaskResult, CommitDiffTask, + CommitTask, FinalizeTask, PreparationState, TaskType, UndelegateTask, }; /// Task that will be executed on Base layer via arguments #[derive(Clone)] pub enum ArgsTaskType { Commit(CommitTask), + CommitDiff(CommitDiffTask), Finalize(FinalizeTask), Undelegate(UndelegateTask), // Special action really BaseAction(BaseActionTask), @@ -59,6 +64,25 @@ impl BaseTask for ArgsTask { args, ) } + ArgsTaskType::CommitDiff(value) => { + let args = CommitDiffArgs { + nonce: value.commit_id, + lamports: value.committed_account.account.lamports, + diff: compute_diff( + value.base_account.data(), + value.committed_account.account.data(), + ) + .to_vec(), + allow_undelegation: value.allow_undelegation, + }; + + dlp::instruction_builder::commit_diff( + *validator, + value.committed_account.pubkey, + value.committed_account.account.owner, + args, + ) + } ArgsTaskType::Finalize(value) => { dlp::instruction_builder::finalize( *validator, @@ -107,6 +131,19 @@ impl BaseTask for ArgsTask { BufferTaskType::Commit(value), ))) } + ArgsTaskType::CommitDiff(value) => { + // TODO (snawaz): Currently, we do not support executing CommitDiff + // as BufferTask, which is why we're forcing CommitDiffTask to become CommitTask + // before converting this task into BufferTask. Once CommitDiff is supported + // by BufferTask, we do not have to do this, as it's essentially a downgrade. + Ok(Box::new(BufferTask::new_preparation_required( + BufferTaskType::Commit(CommitTask { + commit_id: value.commit_id, + allow_undelegation: value.allow_undelegation, + committed_account: value.committed_account, + }), + ))) + } ArgsTaskType::BaseAction(_) | ArgsTaskType::Finalize(_) | ArgsTaskType::Undelegate(_) => Err(self), @@ -133,6 +170,7 @@ impl BaseTask for ArgsTask { fn compute_units(&self) -> u32 { match &self.task_type { ArgsTaskType::Commit(_) => 70_000, + ArgsTaskType::CommitDiff(_) => 70_000, ArgsTaskType::BaseAction(task) => task.action.compute_units, ArgsTaskType::Undelegate(_) => 70_000, ArgsTaskType::Finalize(_) => 70_000, @@ -147,6 +185,7 @@ impl BaseTask for ArgsTask { fn task_type(&self) -> TaskType { match &self.task_type { ArgsTaskType::Commit(_) => TaskType::Commit, + ArgsTaskType::CommitDiff(_) => TaskType::Commit, ArgsTaskType::BaseAction(_) => TaskType::Action, ArgsTaskType::Undelegate(_) => TaskType::Undelegate, ArgsTaskType::Finalize(_) => TaskType::Finalize, @@ -159,11 +198,17 @@ impl BaseTask for ArgsTask { } fn reset_commit_id(&mut self, commit_id: u64) { - let ArgsTaskType::Commit(commit_task) = &mut self.task_type else { - return; + match &mut self.task_type { + ArgsTaskType::Commit(task) => { + task.commit_id = commit_id; + } + ArgsTaskType::CommitDiff(task) => { + task.commit_id = commit_id; + } + ArgsTaskType::BaseAction(_) + | ArgsTaskType::Finalize(_) + | ArgsTaskType::Undelegate(_) => {} }; - - commit_task.commit_id = commit_id; } } @@ -171,6 +216,7 @@ impl LabelValue for ArgsTask { fn value(&self) -> &str { match self.task_type { ArgsTaskType::Commit(_) => "args_commit", + ArgsTaskType::CommitDiff(_) => "args_commit_diff", ArgsTaskType::BaseAction(_) => "args_action", ArgsTaskType::Finalize(_) => "args_finalize", ArgsTaskType::Undelegate(_) => "args_undelegate", diff --git a/magicblock-committor-service/src/tasks/buffer_task.rs b/magicblock-committor-service/src/tasks/buffer_task.rs index ccb54b715..7a1fb1890 100644 --- a/magicblock-committor-service/src/tasks/buffer_task.rs +++ b/magicblock-committor-service/src/tasks/buffer_task.rs @@ -4,6 +4,8 @@ use magicblock_metrics::metrics::LabelValue; use solana_instruction::Instruction; use solana_pubkey::Pubkey; +#[cfg(any(test, feature = "dev-context-only-utils"))] +use super::args_task::ArgsTaskType; #[cfg(test)] use crate::tasks::TaskStrategy; use crate::{ @@ -62,6 +64,17 @@ impl BufferTask { } } +#[cfg(any(test, feature = "dev-context-only-utils"))] +impl From for BufferTaskType { + fn from(value: ArgsTaskType) -> Self { + match value { + ArgsTaskType::Commit(task) => BufferTaskType::Commit(task), + ArgsTaskType::CommitDiff(_) => panic!("BufferTask doesn't support CommitDiff yet. Disable your tests (if any) temporarily till the next PR"), + _ => unimplemented!("Only commit task can be BufferTask currently. Fix your tests"), + } + } +} + impl BaseTask for BufferTask { fn instruction(&self, validator: &Pubkey) -> Instruction { let BufferTaskType::Commit(ref value) = self.task_type; diff --git a/magicblock-committor-service/src/tasks/mod.rs b/magicblock-committor-service/src/tasks/mod.rs index 43477e7ce..ea8b5c17a 100644 --- a/magicblock-committor-service/src/tasks/mod.rs +++ b/magicblock-committor-service/src/tasks/mod.rs @@ -14,6 +14,7 @@ use magicblock_metrics::metrics::LabelValue; use magicblock_program::magic_scheduled_base_intent::{ BaseAction, CommittedAccount, }; +use solana_account::Account; use solana_instruction::Instruction; use solana_pubkey::Pubkey; use thiserror::Error; @@ -28,6 +29,8 @@ pub(crate) mod task_visitors; pub mod utils; pub mod visitor; +pub use task_builder::TaskBuilderImpl; + #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub enum TaskType { Commit, @@ -105,6 +108,14 @@ pub struct CommitTask { pub committed_account: CommittedAccount, } +#[derive(Clone, Debug)] +pub struct CommitDiffTask { + pub commit_id: u64, + pub allow_undelegation: bool, + pub committed_account: CommittedAccount, + pub base_account: Account, +} + #[derive(Clone)] pub struct UndelegateTask { pub delegated_account: Pubkey, @@ -292,6 +303,7 @@ pub type BaseTaskResult = Result; #[cfg(test)] mod serialization_safety_test { + use magicblock_program::{ args::ShortAccountMeta, magic_scheduled_base_intent::ProgramArgs, }; diff --git a/magicblock-committor-service/src/tasks/task_builder.rs b/magicblock-committor-service/src/tasks/task_builder.rs index bc134c6ea..7a9787fe2 100644 --- a/magicblock-committor-service/src/tasks/task_builder.rs +++ b/magicblock-committor-service/src/tasks/task_builder.rs @@ -6,9 +6,11 @@ use magicblock_program::magic_scheduled_base_intent::{ CommitType, CommittedAccount, MagicBaseIntent, ScheduledBaseIntent, UndelegateType, }; +use solana_account::Account; use solana_pubkey::Pubkey; use solana_signature::Signature; +use super::{CommitDiffTask, CommitTask}; use crate::{ intent_executor::task_info_fetcher::{ TaskInfoFetcher, TaskInfoFetcherError, @@ -16,7 +18,7 @@ use crate::{ persist::IntentPersister, tasks::{ args_task::{ArgsTask, ArgsTaskType}, - BaseActionTask, BaseTask, CommitTask, FinalizeTask, UndelegateTask, + BaseActionTask, BaseTask, FinalizeTask, UndelegateTask, }, }; @@ -40,6 +42,46 @@ pub trait TasksBuilder { /// V1: Actions are part of finalize tx pub struct TaskBuilderImpl; +// Accounts larger than COMMIT_STATE_SIZE_THRESHOLD use CommitDiff to +// reduce instruction size. Below this threshold, the commit is sent +// as CommitState. The value (256) is chosen because it is sufficient +// for small accounts, which typically could hold up to 8 u32 fields or +// 4 u64 fields. These integers are expected to be on the hot path +// and updated continuously. +pub const COMMIT_STATE_SIZE_THRESHOLD: usize = 256; + +impl TaskBuilderImpl { + pub fn create_commit_task( + commit_id: u64, + allow_undelegation: bool, + account: CommittedAccount, + base_account: Option, + ) -> ArgsTask { + let base_account = + if account.account.data.len() > COMMIT_STATE_SIZE_THRESHOLD { + base_account + } else { + None + }; + + if let Some(base_account) = base_account { + ArgsTaskType::CommitDiff(CommitDiffTask { + commit_id, + allow_undelegation, + committed_account: account, + base_account, + }) + } else { + ArgsTaskType::Commit(CommitTask { + commit_id, + allow_undelegation, + committed_account: account, + }) + } + .into() + } +} + #[async_trait] impl TasksBuilder for TaskBuilderImpl { /// Returns [`Task`]s for Commit stage @@ -68,14 +110,37 @@ impl TasksBuilder for TaskBuilderImpl { } }; - let committed_pubkeys = accounts - .iter() - .map(|account| account.pubkey) - .collect::>(); - let commit_ids = commit_id_fetcher - .fetch_next_commit_ids(&committed_pubkeys) - .await - .map_err(TaskBuilderError::CommitTasksBuildError)?; + let (commit_ids, base_accounts) = { + let committed_pubkeys = accounts + .iter() + .map(|account| account.pubkey) + .collect::>(); + + let diffable_pubkeys = accounts + .iter() + .filter(|account| { + account.account.data.len() > COMMIT_STATE_SIZE_THRESHOLD + }) + .map(|account| account.pubkey) + .collect::>(); + + tokio::join!( + commit_id_fetcher.fetch_next_commit_ids(&committed_pubkeys), + commit_id_fetcher + .get_base_accounts(diffable_pubkeys.as_slice()) + ) + }; + + let commit_ids = + commit_ids.map_err(TaskBuilderError::CommitTasksBuildError)?; + + let base_accounts = match base_accounts { + Ok(map) => map, + Err(err) => { + log::warn!("Failed to fetch base accounts for CommitDiff (id={}): {}; falling back to CommitState", base_intent.id, err); + Default::default() + } + }; // Persist commit ids for commitees commit_ids @@ -90,15 +155,13 @@ impl TasksBuilder for TaskBuilderImpl { .iter() .map(|account| { let commit_id = *commit_ids.get(&account.pubkey).expect("CommitIdFetcher provide commit ids for all listed pubkeys, or errors!"); - let task = ArgsTaskType::Commit(CommitTask { - commit_id, - allow_undelegation, - committed_account: account.clone(), - }); - - Box::new(ArgsTask::new(task)) as Box - }) - .collect(); + // TODO (snawaz): if accounts do not have duplicate, then we can use remove + // instead: + // let base_account = base_accounts.remove(&account.pubkey); + let base_account = base_accounts.get(&account.pubkey).cloned(); + let task = Self::create_commit_task(commit_id, allow_undelegation, account.clone(), base_account); + Box::new(task) as Box + }).collect(); Ok(tasks) } diff --git a/magicblock-committor-service/src/tasks/task_strategist.rs b/magicblock-committor-service/src/tasks/task_strategist.rs index 2cf7e3e13..edbbd74d3 100644 --- a/magicblock-committor-service/src/tasks/task_strategist.rs +++ b/magicblock-committor-service/src/tasks/task_strategist.rs @@ -289,6 +289,7 @@ impl TaskStrategist { // Get initial transaction size let mut current_tx_length = calculate_tx_length(tasks)?; + if current_tx_length <= MAX_ENCODED_TRANSACTION_SIZE { return Ok(current_tx_length); } @@ -373,6 +374,7 @@ mod tests { BaseAction, CommittedAccount, ProgramArgs, }; use solana_account::Account; + use solana_program::system_program; use solana_pubkey::Pubkey; use solana_system_program::id as system_program_id; @@ -384,12 +386,15 @@ mod tests { }, persist::IntentPersisterImpl, tasks::{ - task_builder::{TaskBuilderImpl, TasksBuilder}, - BaseActionTask, CommitTask, TaskStrategy, UndelegateTask, + task_builder::{ + TaskBuilderImpl, TasksBuilder, COMMIT_STATE_SIZE_THRESHOLD, + }, + BaseActionTask, TaskStrategy, UndelegateTask, }, }; struct MockInfoFetcher; + #[async_trait::async_trait] impl TaskInfoFetcher for MockInfoFetcher { async fn fetch_next_commit_ids( @@ -411,24 +416,55 @@ mod tests { } fn reset(&self, _: ResetType) {} + + async fn get_base_accounts( + &self, + _pubkeys: &[Pubkey], + ) -> TaskInfoFetcherResult> { + Ok(Default::default()) + } } // Helper to create a simple commit task - fn create_test_commit_task(commit_id: u64, data_size: usize) -> ArgsTask { - ArgsTask::new(ArgsTaskType::Commit(CommitTask { - commit_id, - allow_undelegation: false, - committed_account: CommittedAccount { - pubkey: Pubkey::new_unique(), - account: Account { - lamports: 1000, - data: vec![1; data_size], - owner: system_program_id(), - executable: false, - rent_epoch: 0, - }, + fn create_test_commit_task( + commit_id: u64, + data_size: usize, + diff_len: usize, + ) -> ArgsTask { + let committed_account = CommittedAccount { + pubkey: Pubkey::new_unique(), + account: Account { + lamports: 1000, + data: vec![1; data_size], + owner: system_program::id(), + executable: false, + rent_epoch: 0, }, - })) + }; + + if diff_len == 0 { + TaskBuilderImpl::create_commit_task( + commit_id, + false, + committed_account, + None, + ) + } else { + let base_account = { + let mut acc = committed_account.account.clone(); + assert!(diff_len <= acc.data.len()); + for byte in &mut acc.data[..diff_len] { + *byte = byte.wrapping_add(1); + } + acc + }; + TaskBuilderImpl::create_commit_task( + commit_id, + false, + committed_account, + Some(base_account), + ) + } } // Helper to create a Base action task @@ -466,7 +502,7 @@ mod tests { #[test] fn test_build_strategy_with_single_small_task() { let validator = Pubkey::new_unique(); - let task = create_test_commit_task(1, 100); + let task = create_test_commit_task(1, 100, 0); let tasks = vec![Box::new(task) as Box]; let strategy = TaskStrategist::build_strategy( @@ -484,7 +520,7 @@ mod tests { fn test_build_strategy_optimizes_to_buffer_when_needed() { let validator = Pubkey::new_unique(); - let task = create_test_commit_task(1, 1000); // Large task + let task = create_test_commit_task(1, 1000, 0); // Large task let tasks = vec![Box::new(task) as Box]; let strategy = TaskStrategist::build_strategy( @@ -505,7 +541,7 @@ mod tests { fn test_build_strategy_optimizes_to_buffer_u16_exceeded() { let validator = Pubkey::new_unique(); - let task = create_test_commit_task(1, 66_000); // Large task + let task = create_test_commit_task(1, 66_000, 0); // Large task let tasks = vec![Box::new(task) as Box]; let strategy = TaskStrategist::build_strategy( @@ -522,6 +558,67 @@ mod tests { )); } + #[test] + fn test_build_strategy_does_not_optimize_large_account_but_small_diff() { + let validator = Pubkey::new_unique(); + + let task = + create_test_commit_task(1, 66_000, COMMIT_STATE_SIZE_THRESHOLD); // large account but small diff + let tasks = vec![Box::new(task) as Box]; + + let strategy = TaskStrategist::build_strategy( + tasks, + &validator, + &None::, + ) + .expect("Should build strategy with buffer optimization"); + + assert_eq!(strategy.optimized_tasks.len(), 1); + assert_eq!(strategy.optimized_tasks[0].strategy(), TaskStrategy::Args); + } + + #[test] + fn test_build_strategy_does_not_optimize_large_account_and_above_threshold_diff( + ) { + let validator = Pubkey::new_unique(); + + let task = + create_test_commit_task(1, 66_000, COMMIT_STATE_SIZE_THRESHOLD + 1); // large account but small diff + let tasks = vec![Box::new(task) as Box]; + + let strategy = TaskStrategist::build_strategy( + tasks, + &validator, + &None::, + ) + .expect("Should build strategy with buffer optimization"); + + assert_eq!(strategy.optimized_tasks.len(), 1); + assert_eq!(strategy.optimized_tasks[0].strategy(), TaskStrategy::Args); + } + + #[test] + fn test_build_strategy_does_optimize_large_account_and_large_diff() { + let validator = Pubkey::new_unique(); + + let task = + create_test_commit_task(1, 66_000, COMMIT_STATE_SIZE_THRESHOLD * 4); // large account but small diff + let tasks = vec![Box::new(task) as Box]; + + let strategy = TaskStrategist::build_strategy( + tasks, + &validator, + &None::, + ) + .expect("Should build strategy with buffer optimization"); + + assert_eq!(strategy.optimized_tasks.len(), 1); + assert_eq!( + strategy.optimized_tasks[0].strategy(), + TaskStrategy::Buffer + ); + } + #[test] fn test_build_strategy_creates_multiple_buffers() { // TODO: ALSO MAX NUM WITH PURE BUFFER commits, no alts @@ -531,7 +628,7 @@ mod tests { let tasks = (0..NUM_COMMITS) .map(|i| { - let task = create_test_commit_task(i, 500); // Large task + let task = create_test_commit_task(i, 500, 0); // Large task Box::new(task) as Box }) .collect(); @@ -559,7 +656,7 @@ mod tests { let tasks = (0..NUM_COMMITS) .map(|i| { // Large task - let task = create_test_commit_task(i, 10000); + let task = create_test_commit_task(i, 10000, 0); Box::new(task) as Box }) .collect(); @@ -586,7 +683,7 @@ mod tests { let tasks = (0..NUM_COMMITS) .map(|i| { // Large task - let task = create_test_commit_task(i, 1000); + let task = create_test_commit_task(i, 1000, 0); Box::new(task) as Box }) .collect(); @@ -602,9 +699,9 @@ mod tests { #[test] fn test_optimize_strategy_prioritizes_largest_tasks() { let mut tasks = [ - Box::new(create_test_commit_task(1, 100)) as Box, - Box::new(create_test_commit_task(2, 1000)) as Box, // Larger task - Box::new(create_test_commit_task(3, 1000)) as Box, // Larger task + Box::new(create_test_commit_task(1, 100, 0)) as Box, + Box::new(create_test_commit_task(2, 1000, 0)) as Box, // Larger task + Box::new(create_test_commit_task(3, 1000, 0)) as Box, // Larger task ]; let _ = TaskStrategist::optimize_strategy(&mut tasks); @@ -617,7 +714,7 @@ mod tests { fn test_mixed_task_types_with_optimization() { let validator = Pubkey::new_unique(); let tasks = vec![ - Box::new(create_test_commit_task(1, 1000)) as Box, + Box::new(create_test_commit_task(1, 1000, 0)) as Box, Box::new(create_test_finalize_task()) as Box, Box::new(create_test_base_action_task(500)) as Box, Box::new(create_test_undelegate_task()) as Box, diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index f2f0eea2a..c8490af54 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -79,6 +79,17 @@ dependencies = [ "solana-svm-transaction", ] +[[package]] +name = "ahash" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" +dependencies = [ + "getrandom 0.2.16", + "once_cell", + "version_check", +] + [[package]] name = "ahash" version = "0.8.12" @@ -591,6 +602,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.8.2" @@ -737,6 +760,28 @@ dependencies = [ "serde", ] +[[package]] +name = "bytecheck" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2" +dependencies = [ + "bytecheck_derive", + "ptr_meta 0.1.4", + "simdutf8", +] + +[[package]] +name = "bytecheck_derive" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "bytemuck" version = "1.23.1" @@ -1542,7 +1587,7 @@ dependencies = [ "ephemeral-rollups-sdk-attribute-commit", "ephemeral-rollups-sdk-attribute-delegate", "ephemeral-rollups-sdk-attribute-ephemeral", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.0", "magicblock-magic-program-api 0.2.1", "solana-program", ] @@ -1656,7 +1701,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6503af7917fea18ffef8f7e8553fb8dff89e2e6837e94e09dd7fb069c82d62c" dependencies = [ "bytes", - "rkyv", + "rkyv 0.8.11", "serde", "simdutf8", ] @@ -1840,6 +1885,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.3.31" @@ -2109,6 +2160,9 @@ name = "hashbrown" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +dependencies = [ + "ahash 0.7.8", +] [[package]] name = "hashbrown" @@ -2116,7 +2170,7 @@ version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash", + "ahash 0.8.12", ] [[package]] @@ -2619,7 +2673,7 @@ dependencies = [ "log", "magicblock-config", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "random-port", "rayon", "serde", @@ -3190,7 +3244,7 @@ dependencies = [ "lru", "magicblock-config", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "magicblock-magic-program-api 0.5.0", "magicblock-metrics", "solana-account", @@ -3249,7 +3303,7 @@ dependencies = [ "log", "lru", "magicblock-committor-program", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "magicblock-metrics", "magicblock-program", "magicblock-rpc-client", @@ -3335,6 +3389,28 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "magicblock-delegation-program" +version = "1.1.3" +source = "git+https://github.com/magicblock-labs/delegation-program.git?rev=3edb41022#3edb41022afb0e813607ffd4e79d1548131eab2b" +dependencies = [ + "bincode", + "borsh 1.5.7", + "bytemuck", + "num_enum", + "pinocchio", + "pinocchio-log", + "pinocchio-pubkey", + "pinocchio-system", + "rkyv 0.7.45", + "solana-curve25519", + "solana-program", + "solana-security-txt", + "static_assertions", + "strum 0.27.2", + "thiserror 1.0.69", +] + [[package]] name = "magicblock-ledger" version = "0.5.0" @@ -3553,7 +3629,7 @@ name = "magicblock-validator-admin" version = "0.5.0" dependencies = [ "log", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "magicblock-program", "magicblock-rpc-client", "solana-commitment-config", @@ -4426,9 +4502,11 @@ version = "0.0.0" dependencies = [ "borsh 1.5.7", "ephemeral-rollups-sdk", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "magicblock-magic-program-api 0.5.0", + "rkyv 0.7.45", "solana-program", + "static_assertions", ] [[package]] @@ -4525,13 +4603,33 @@ dependencies = [ "autotools", ] +[[package]] +name = "ptr_meta" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1" +dependencies = [ + "ptr_meta_derive 0.1.4", +] + [[package]] name = "ptr_meta" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe9e76f66d3f9606f44e45598d155cb13ecf09f4a28199e48daf8c8fc937ea90" dependencies = [ - "ptr_meta_derive", + "ptr_meta_derive 0.3.0", +] + +[[package]] +name = "ptr_meta_derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", ] [[package]] @@ -4652,13 +4750,19 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rancor" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "caf5f7161924b9d1cea0e4cabc97c372cea92b5f927fc13c6bca67157a0ad947" dependencies = [ - "ptr_meta", + "ptr_meta 0.3.0", ] [[package]] @@ -4880,6 +4984,15 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "rend" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c" +dependencies = [ + "bytecheck", +] + [[package]] name = "rend" version = "0.5.2" @@ -4959,6 +5072,24 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rkyv" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9008cd6385b9e161d8229e1f6549dd23c3d022f132a2ea37ac3a10ac4935779b" +dependencies = [ + "bitvec", + "bytecheck", + "bytes", + "hashbrown 0.12.3", + "ptr_meta 0.1.4", + "rend 0.4.2", + "rkyv_derive 0.7.45", + "seahash", + "tinyvec", + "uuid", +] + [[package]] name = "rkyv" version = "0.8.11" @@ -4969,14 +5100,25 @@ dependencies = [ "hashbrown 0.15.4", "indexmap 2.10.0", "munge", - "ptr_meta", + "ptr_meta 0.3.0", "rancor", - "rend", - "rkyv_derive", + "rend 0.5.2", + "rkyv_derive 0.8.11", "tinyvec", "uuid", ] +[[package]] +name = "rkyv_derive" +version = "0.7.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503d1d27590a2b0a3a4ca4c94755aa2875657196ecbf401a42eff41d7de532c0" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "rkyv_derive" version = "0.8.11" @@ -5236,7 +5378,7 @@ dependencies = [ "integration-test-tools", "log", "magicblock-core", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "program-schedulecommit", "solana-program", "solana-rpc-client", @@ -5254,11 +5396,12 @@ dependencies = [ "log", "magicblock-committor-program", "magicblock-committor-service", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "magicblock-program", "magicblock-rpc-client", "magicblock-table-mania", "program-flexi-counter", + "program-schedulecommit", "rand 0.8.5", "solana-account", "solana-pubkey", @@ -5273,12 +5416,14 @@ dependencies = [ name = "schedulecommit-test-scenarios" version = "0.0.0" dependencies = [ + "borsh 1.5.7", "ephemeral-rollups-sdk", "integration-test-tools", "log", "magicblock-core", "magicblock-magic-program-api 0.5.0", "program-schedulecommit", + "rand 0.8.5", "schedulecommit-client", "solana-program", "solana-rpc-client", @@ -5347,6 +5492,12 @@ version = "3.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "584e070911c7017da6cb2eb0788d09f43d789029b5877d3e5ecc8acf86ceee21" +[[package]] +name = "seahash" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" + [[package]] name = "security-framework" version = "3.2.0" @@ -5747,7 +5898,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d65a1a23a53cae19cb92bab2cbdd9e289e5210bb12175ce27642c94adf74b220" dependencies = [ - "ahash", + "ahash 0.8.12", "bincode", "blake3", "bv", @@ -6048,7 +6199,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4ee734c35b736e632aa3b1367f933d93ee7b4129dd1e20ca942205d4834054e" dependencies = [ - "ahash", + "ahash 0.8.12", "lazy_static", "log", "qualifier_attr", @@ -6274,7 +6425,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a675ead1473b32a7a5735801608b35cbd8d3f5057ca8dbafdd5976146bb7e9e4" dependencies = [ - "ahash", + "ahash 0.8.12", "lazy_static", "log", "solana-bincode", @@ -6460,7 +6611,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89e1d3b52b4a014efeaaab67f14e40af3972a4be61c523d612860db8e3145529" dependencies = [ - "ahash", + "ahash 0.8.12", "lazy_static", "solana-epoch-schedule", "solana-hash", @@ -6939,7 +7090,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f0962d3818fc942a888f7c2d530896aeaf6f2da2187592a67bbdc8cf8a54192" dependencies = [ - "ahash", + "ahash 0.8.12", "bincode", "bv", "caps", @@ -7494,7 +7645,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5335e7925f6dc8d2fdcdc6ead3b190aca65f191a11cef74709a7a6ab5d0d5877" dependencies = [ - "ahash", + "ahash 0.8.12", "aquamarine", "arrayref", "base64 0.22.1", @@ -7565,8 +7716,8 @@ dependencies = [ "solana-vote", "solana-vote-program", "static_assertions", - "strum", - "strum_macros", + "strum 0.24.1", + "strum_macros 0.24.3", "symlink", "tar", "tempfile", @@ -8046,7 +8197,7 @@ version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0850baf834aba4a94a7558fa6cf6ca93fad284abf0363dec5fb9cab173a11fc4" dependencies = [ - "ahash", + "ahash 0.8.12", "itertools 0.12.1", "log", "percentage", @@ -8090,7 +8241,7 @@ name = "solana-svm" version = "2.2.1" source = "git+https://github.com/magicblock-labs/magicblock-svm.git?rev=3e9456ec4#3e9456ec4d5798ad8281537501c1e777d6888ba3" dependencies = [ - "ahash", + "ahash 0.8.12", "log", "percentage", "qualifier_attr", @@ -8749,7 +8900,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd1adc42def3cb101f3ebef3cd2d642f9a21072bbcd4ec9423343ccaa6afa596" dependencies = [ - "ahash", + "ahash 0.8.12", "bumpalo", "bytes", "cfg-if", @@ -9183,7 +9334,16 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" dependencies = [ - "strum_macros", + "strum_macros 0.24.3", +] + +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" +dependencies = [ + "strum_macros 0.27.2", ] [[package]] @@ -9199,6 +9359,18 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.104", +] + [[package]] name = "subtle" version = "2.6.1" @@ -9290,6 +9462,12 @@ dependencies = [ "solana-program", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.44" @@ -9389,7 +9567,7 @@ dependencies = [ "log", "magicblock-chainlink", "magicblock-config", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "program-flexi-counter", "program-mini", "solana-account", @@ -9471,7 +9649,7 @@ dependencies = [ "log", "magicblock-accounts-db", "magicblock-config", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "program-flexi-counter", "solana-rpc-client", "solana-sdk", @@ -9492,7 +9670,7 @@ dependencies = [ "magic-domain-program", "magicblock-api", "magicblock-config", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "magicblock-program", "magicblock-validator-admin", "solana-rpc-client", @@ -9532,7 +9710,7 @@ version = "0.0.0" dependencies = [ "integration-test-tools", "log", - "magicblock-delegation-program", + "magicblock-delegation-program 1.1.3", "program-flexi-counter", "solana-rpc-client-api", "solana-sdk", @@ -10777,6 +10955,15 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x509-parser" version = "0.14.0" diff --git a/test-integration/Cargo.toml b/test-integration/Cargo.toml index 6dd2ea45e..06999a97d 100644 --- a/test-integration/Cargo.toml +++ b/test-integration/Cargo.toml @@ -36,8 +36,8 @@ chrono = "0.4" cleanass = "0.0.1" color-backtrace = { version = "0.7" } ctrlc = "3.4.7" -ephemeral-rollups-sdk = { git = "https://github.com/magicblock-labs/ephemeral-rollups-sdk.git", rev = "2d0f16b" } futures = "0.3.31" +ephemeral-rollups-sdk = { git = "https://github.com/magicblock-labs/ephemeral-rollups-sdk.git", rev = "2d0f16b" } integration-test-tools = { path = "test-tools" } isocountry = "0.3.2" lazy_static = "1.4.0" @@ -57,7 +57,7 @@ magicblock-config = { path = "../magicblock-config" } magicblock-core = { path = "../magicblock-core" } magic-domain-program = { git = "https://github.com/magicblock-labs/magic-domain-program.git", rev = "ea04d46", default-features = false } magicblock_magic_program_api = { package = "magicblock-magic-program-api", path = "../magicblock-magic-program-api" } -magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "aa1de56d90c", features = [ +magicblock-delegation-program = { git = "https://github.com/magicblock-labs/delegation-program.git", rev = "3edb41022" , features = [ "no-entrypoint", ] } magicblock-program = { path = "../programs/magicblock" } @@ -71,6 +71,7 @@ program-schedulecommit-security = { path = "programs/schedulecommit-security" } rand = "0.8.5" random-port = "0.1.1" rayon = "1.10.0" +rkyv = "0.7.45" schedulecommit-client = { path = "schedulecommit/client" } serde = "1.0.217" serial_test = "3.2.0" @@ -89,6 +90,7 @@ solana-sdk-ids = { version = "2.2" } solana-system-interface = "1.0" solana-transaction-status = "2.2" spl-memo-interface = "1.0" +static_assertions = "1.1.0" teepee = "0.0.1" tempfile = "3.10.1" test-chainlink = { path = "./test-chainlink" } diff --git a/test-integration/programs/schedulecommit/Cargo.toml b/test-integration/programs/schedulecommit/Cargo.toml index 9067d6327..272fab194 100644 --- a/test-integration/programs/schedulecommit/Cargo.toml +++ b/test-integration/programs/schedulecommit/Cargo.toml @@ -9,6 +9,8 @@ ephemeral-rollups-sdk = { workspace = true } solana-program = { workspace = true } magicblock-delegation-program = { workspace = true } magicblock_magic_program_api = { workspace = true } +rkyv = { workspace = true } +static_assertions = { workspace = true } [lib] crate-type = ["cdylib", "lib"] diff --git a/test-integration/programs/schedulecommit/src/api.rs b/test-integration/programs/schedulecommit/src/api.rs index adecc662b..8f359a56c 100644 --- a/test-integration/programs/schedulecommit/src/api.rs +++ b/test-integration/programs/schedulecommit/src/api.rs @@ -9,9 +9,25 @@ use solana_program::{ }; use crate::{ - DelegateCpiArgs, ScheduleCommitCpiArgs, ScheduleCommitInstruction, + BookUpdate, DelegateCpiArgs, DelegateOrderBookArgs, ScheduleCommitCpiArgs, + ScheduleCommitInstruction, }; +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum UserSeeds { + MagicScheduleCommit, + OrderBook, +} + +impl UserSeeds { + pub fn bytes(&self) -> &'static [u8] { + match self { + UserSeeds::MagicScheduleCommit => b"magic_schedule_commit", + UserSeeds::OrderBook => b"order_book", + } + } +} + pub fn init_account_instruction( payer: Pubkey, player: Pubkey, @@ -32,6 +48,47 @@ pub fn init_account_instruction( ) } +pub fn init_order_book_instruction( + payer: Pubkey, + book_manager: Pubkey, + order_book: Pubkey, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new_readonly(book_manager, true), + AccountMeta::new(order_book, false), + AccountMeta::new_readonly(system_program::id(), false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::InitOrderBook, + account_metas, + ) +} + +pub fn grow_order_book_instruction( + payer: Pubkey, + book_manager: Pubkey, + order_book: Pubkey, + additional_space: u64, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new_readonly(book_manager, false), + AccountMeta::new(order_book, false), + AccountMeta::new_readonly(system_program::id(), false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::GrowOrderBook(additional_space), + account_metas, + ) +} + pub fn init_payer_escrow(payer: Pubkey) -> [Instruction; 2] { let top_up_ix = dlp::instruction_builder::top_up_ephemeral_balance( payer, @@ -58,19 +115,17 @@ pub fn init_payer_escrow(payer: Pubkey) -> [Instruction; 2] { pub fn delegate_account_cpi_instruction( payer: Pubkey, validator: Option, - player: Pubkey, + player_or_book_manager: Pubkey, + user_seed: UserSeeds, ) -> Instruction { let program_id = crate::id(); - let (pda, _) = pda_and_bump(&player); - - let args = DelegateCpiArgs { - valid_until: i64::MAX, - commit_frequency_ms: 1_000_000_000, - validator, - player, - }; + let (pda, _) = Pubkey::find_program_address( + &[user_seed.bytes(), player_or_book_manager.as_ref()], + &crate::ID, + ); let delegate_accounts = DelegateAccounts::new(pda, program_id); + let delegate_metas = DelegateAccountMetas::from(delegate_accounts); let account_metas = vec![ AccountMeta::new(payer, true), @@ -85,7 +140,25 @@ pub fn delegate_account_cpi_instruction( Instruction::new_with_borsh( program_id, - &ScheduleCommitInstruction::DelegateCpi(args), + &match user_seed { + UserSeeds::MagicScheduleCommit => { + ScheduleCommitInstruction::DelegateCpi(DelegateCpiArgs { + valid_until: i64::MAX, + commit_frequency_ms: 1_000_000_000, + player: player_or_book_manager, + validator, + }) + } + UserSeeds::OrderBook => { + ScheduleCommitInstruction::DelegateOrderBook( + DelegateOrderBookArgs { + commit_frequency_ms: 1_000_000_000, + book_manager: player_or_book_manager, + validator, + }, + ) + } + }, account_metas, ) } @@ -121,6 +194,45 @@ pub fn schedule_commit_cpi_instruction( ) } +pub fn update_order_book_instruction( + payer: Pubkey, + order_book: Pubkey, + update: BookUpdate, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new(order_book, false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::UpdateOrderBook(update), + account_metas, + ) +} + +pub fn schedule_commit_diff_instruction_for_order_book( + payer: Pubkey, + order_book: Pubkey, + magic_program_id: Pubkey, + magic_context_id: Pubkey, +) -> Instruction { + let program_id = crate::id(); + let account_metas = vec![ + AccountMeta::new(payer, true), + AccountMeta::new(order_book, false), + AccountMeta::new(magic_context_id, false), + AccountMeta::new_readonly(magic_program_id, false), + ]; + + Instruction::new_with_borsh( + program_id, + &ScheduleCommitInstruction::ScheduleCommitForOrderBook, + account_metas, + ) +} + pub fn schedule_commit_with_payer_cpi_instruction( payer: Pubkey, magic_program_id: Pubkey, diff --git a/test-integration/programs/schedulecommit/src/lib.rs b/test-integration/programs/schedulecommit/src/lib.rs index cdd207408..32d2e39f5 100644 --- a/test-integration/programs/schedulecommit/src/lib.rs +++ b/test-integration/programs/schedulecommit/src/lib.rs @@ -15,9 +15,12 @@ use solana_program::{ entrypoint::{self, ProgramResult}, instruction::{AccountMeta, Instruction}, msg, - program::invoke_signed, + program::{invoke, invoke_signed}, program_error::ProgramError, pubkey::Pubkey, + rent::Rent, + system_instruction, + sysvar::Sysvar, }; use crate::{ @@ -30,9 +33,12 @@ use crate::{ pub mod api; pub mod magicblock_program; +mod order_book; mod utils; pub const FAIL_UNDELEGATION_COUNT: u64 = u64::MAX - 1; +use order_book::*; +pub use order_book::{BookUpdate, OrderBookOwned, OrderLevel}; declare_id!("9hgprgZiRWmy8KkfvUuaVkDGrqo9GzeXMohwq6BazgUY"); @@ -47,6 +53,13 @@ pub struct DelegateCpiArgs { validator: Option, } +#[derive(BorshSerialize, BorshDeserialize, Debug, Clone)] +pub struct DelegateOrderBookArgs { + commit_frequency_ms: u32, + book_manager: Pubkey, + validator: Option, +} + #[derive(BorshSerialize, BorshDeserialize, Debug, Clone)] pub struct ScheduleCommitCpiArgs { /// Pubkeys of players from which PDAs were derived @@ -132,6 +145,19 @@ pub enum ScheduleCommitInstruction { // // It is not part of this enum as it has a custom discriminator // Undelegate, + /// Initialize an OrderBook + InitOrderBook, + + GrowOrderBook(u64), // additional_space + + /// Delegate order book to ER nodes + DelegateOrderBook(DelegateOrderBookArgs), + + /// Update order book + UpdateOrderBook(BookUpdate), + + /// ScheduleCommitDiffCpi + ScheduleCommitForOrderBook, } pub fn process_instruction<'a>( @@ -139,6 +165,7 @@ pub fn process_instruction<'a>( accounts: &'a [AccountInfo<'a>], instruction_data: &[u8], ) -> ProgramResult { + msg!("process_instruction: {}", instruction_data[0]); // Undelegate Instruction if instruction_data.len() >= EXTERNAL_UNDELEGATE_DISCRIMINATOR.len() { let (disc, seeds_data) = @@ -155,6 +182,7 @@ pub fn process_instruction<'a>( msg!("ERROR: failed to parse instruction data {:?}", err); ProgramError::InvalidArgument })?; + use ScheduleCommitInstruction::*; match ix { Init => process_init(program_id, accounts), @@ -180,6 +208,15 @@ pub fn process_instruction<'a>( } IncreaseCount => process_increase_count(accounts), SetCount(value) => process_set_count(accounts, value), + InitOrderBook => process_init_order_book(accounts), + GrowOrderBook(additional_space) => { + process_grow_order_book(accounts, additional_space) + } + DelegateOrderBook(args) => process_delegate_order_book(accounts, args), + UpdateOrderBook(args) => process_update_order_book(accounts, args), + ScheduleCommitForOrderBook => { + process_schedulecommit_for_orderbook(accounts) + } } } @@ -267,6 +304,170 @@ fn process_init<'a>( Ok(()) } +// ----------------- +// InitOrderBook +// ----------------- +fn process_init_order_book<'a>( + accounts: &'a [AccountInfo<'a>], +) -> entrypoint::ProgramResult { + msg!("Init OrderBook account"); + let [payer, book_manager, order_book, _system_program] = accounts else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + assert_is_signer(payer, "payer")?; + + let (pda, bump) = Pubkey::find_program_address( + &[b"order_book", book_manager.key.as_ref()], + &crate::ID, + ); + + assert_keys_equal(order_book.key, &pda, || { + format!( + "PDA for the account ('{}') and for book_manager ('{}') is incorrect", + order_book.key, book_manager.key + ) + })?; + + allocate_account_and_assign_owner(AllocateAndAssignAccountArgs { + payer_info: payer, + account_info: order_book, + owner: &crate::ID, + signer_seeds: &[b"order_book", book_manager.key.as_ref(), &[bump]], + size: 10 * 1024, + })?; + + Ok(()) +} + +fn process_grow_order_book<'a>( + accounts: &'a [AccountInfo<'a>], + additional_space: u64, +) -> entrypoint::ProgramResult { + msg!("Grow OrderBook account"); + let [payer, book_manager, order_book, system_program] = accounts else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + assert_is_signer(payer, "payer")?; + + let (pda, _bump) = Pubkey::find_program_address( + &[b"order_book", book_manager.key.as_ref()], + &crate::ID, + ); + + assert_keys_equal(order_book.key, &pda, || { + format!( + "PDA for the account ('{}') and for book_manager ('{}') is incorrect", + order_book.key, payer.key + ) + })?; + + let new_size = order_book.data_len() + additional_space as usize; + + // Ideally, we should transfer some lamports from payer to order_book + // so that realloc could use it + + let rent = Rent::get()?; + let required = rent.minimum_balance(new_size); + let current = order_book.lamports(); + if current < required { + let diff = required - current; + invoke( + &system_instruction::transfer(payer.key, order_book.key, diff), + &[payer.clone(), order_book.clone(), system_program.clone()], + )?; + } + + order_book.realloc(new_size, true)?; + + Ok(()) +} + +// ----------------- +// Delegate OrderBook +// ----------------- +pub fn process_delegate_order_book( + accounts: &[AccountInfo], + args: DelegateOrderBookArgs, +) -> Result<(), ProgramError> { + msg!("Processing delegate_order_book instruction"); + + let [payer, order_book, owner_program, buffer, delegation_record, delegation_metadata, delegation_program, system_program] = + accounts + else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + let seeds_no_bump = [b"order_book", args.book_manager.as_ref()]; + + delegate_account( + DelegateAccounts { + payer, + pda: order_book, + buffer, + delegation_record, + delegation_metadata, + owner_program, + delegation_program, + system_program, + }, + &seeds_no_bump, + DelegateConfig { + commit_frequency_ms: args.commit_frequency_ms, + validator: args.validator, + }, + )?; + + Ok(()) +} + +// ----------------- +// UpdateOrderBook +// ----------------- +fn process_update_order_book<'a>( + accounts: &'a [AccountInfo<'a>], + updates: BookUpdate, +) -> entrypoint::ProgramResult { + msg!("Update orderbook"); + let account_info_iter = &mut accounts.iter(); + let payer_info = next_account_info(account_info_iter)?; + let order_book_account = next_account_info(account_info_iter)?; + + assert_is_signer(payer_info, "payer")?; + + let mut book_raw = order_book_account.try_borrow_mut_data()?; + + OrderBook::try_new(&mut book_raw)?.update_from(updates); + + Ok(()) +} + +// ----------------- +// Schedule Commit +// ----------------- +pub fn process_schedulecommit_for_orderbook( + accounts: &[AccountInfo], +) -> Result<(), ProgramError> { + msg!("Processing schedulecommit (for orderbook) instruction"); + + let [payer, order_book_account, magic_context, magic_program] = accounts + else { + return Err(ProgramError::NotEnoughAccountKeys); + }; + + assert_is_signer(payer, "payer")?; + + commit_and_undelegate_accounts( + payer, + vec![order_book_account], + magic_context, + magic_program, + )?; + + Ok(()) +} + // ----------------- // Delegate // ----------------- @@ -619,15 +820,23 @@ fn process_undelegate_request( )?; { - let data = delegated_account.try_borrow_data()?; - match MainAccount::try_from_slice(&data) { - Ok(counter) => { - msg!("counter: {:?}", counter); - if counter.count == FAIL_UNDELEGATION_COUNT { - return Err(ProgramError::Custom(111)); + let mut data = delegated_account.try_borrow_mut_data()?; + match data.len() { + MainAccount::SIZE => match MainAccount::try_from_slice(&data) { + Ok(counter) => { + msg!("counter: {:?}", counter); + if counter.count == FAIL_UNDELEGATION_COUNT { + return Err(ProgramError::Custom(111)); + } } - } - Err(err) => msg!("Failed to deserialize: {:?}", err), + Err(err) => { + msg!("Failed to deserialize MainAccount: {:?}", err) + } + }, + _ => match OrderBook::try_new(&mut data) { + Ok(_) => {} + Err(err) => msg!("Failed to deserialize OrderBook: {:?}", err), + }, } }; Ok(()) diff --git a/test-integration/programs/schedulecommit/src/order_book.rs b/test-integration/programs/schedulecommit/src/order_book.rs new file mode 100644 index 000000000..5881387e4 --- /dev/null +++ b/test-integration/programs/schedulecommit/src/order_book.rs @@ -0,0 +1,231 @@ +use std::{ + mem::{align_of, size_of}, + slice, +}; + +use borsh::{BorshDeserialize, BorshSerialize}; +use solana_program::{msg, program_error::ProgramError}; +use static_assertions::const_assert; + +#[repr(C)] +#[derive( + BorshSerialize, BorshDeserialize, Debug, Clone, Copy, Default, PartialEq, Eq, +)] +pub struct OrderLevel { + pub price: u64, // ideally both fields could be some decimal value + pub size: u64, +} + +const_assert!(align_of::() == align_of::()); +const_assert!(size_of::() == 16); + +#[derive(BorshSerialize, BorshDeserialize, Debug, Clone, Default)] +pub struct BookUpdate { + pub bids: Vec, + pub asks: Vec, +} + +#[repr(C)] +pub struct OrderBookHeader { + pub bids_len: u32, + pub asks_len: u32, +} + +const_assert!(align_of::() == align_of::()); +const_assert!(size_of::() == 8); + +const ORDER_LEVEL_SIZE: usize = std::mem::size_of::(); +const HEADER_SIZE: usize = std::mem::size_of::(); + +#[derive(Default, Debug, Clone, PartialEq, Eq)] +pub struct OrderBookOwned { + pub bids: Vec, + pub asks: Vec, +} + +impl From<&OrderBook<'_>> for OrderBookOwned { + fn from(book_ref: &OrderBook) -> Self { + let mut book = Self::default(); + book.bids.extend_from_slice(book_ref.bids()); + book.asks.extend(book_ref.asks_reversed().iter().rev()); + book + } +} + +impl borsh::de::BorshDeserialize for OrderBookOwned { + fn deserialize(buf: &mut &[u8]) -> Result { + let (book_bytes, rest) = buf.split_at(buf.len()); + *buf = rest; // rest is actually empty + + // I make a copy so that I can get mutable bytes in the unsafe block below. + // I could take mutable bytes from &[u8] as well and unsafe block will not + // stop me, but that would break aliasing rules and therefore would invoke UB. + // It's a test code, so copying should be OK. + let mut book_bytes = { + let mut aligned = rkyv::AlignedVec::with_capacity(book_bytes.len()); + aligned.extend_from_slice(book_bytes); + aligned + }; + + OrderBook::try_new(unsafe { + slice::from_raw_parts_mut(book_bytes.as_mut_ptr(), book_bytes.len()) + }) + .map(|book| OrderBookOwned::from(&book)) + .map_err(|_| borsh::io::Error::from(borsh::io::ErrorKind::InvalidData)) + } + + fn deserialize_reader( + _reader: &mut R, + ) -> ::core::result::Result { + unimplemented!("deserialize_reader() not implemented. Please use buffer version as it needs to know size of the buffer") + } +} +pub struct OrderBook<'a> { + header: &'a mut OrderBookHeader, + capacity: usize, + levels: *mut OrderLevel, +} + +impl<'a> OrderBook<'a> { + // + // ========= Zero-Copy Order Book ========== + // + // ----------------------------------------- + // | account data | + // ----------------------------------------- + // | header | levels | + // ----------------------------------------- + // | asks grows -> <- bids grows | + // ----------------------------------------- + // + // Note: + // + // - asks grows towards right + // - bids grows towards left + // + pub fn try_new(data: &'a mut [u8]) -> Result { + let (header_bytes, levels_bytes) = data.split_at_mut(HEADER_SIZE); + + if !(header_bytes + .as_ptr() + .align_offset(align_of::()) + == 0 + && levels_bytes.as_ptr().align_offset(align_of::()) + == 0) + { + msg!( + "data is not properly aligned for OrderBook to be constructed" + ); + return Err(ProgramError::InvalidAccountData); + } + + let capacity = levels_bytes.len() / ORDER_LEVEL_SIZE; + let header = + unsafe { &mut *(header_bytes.as_ptr() as *mut OrderBookHeader) }; + + // Ensure header lengths never exceed backing storage; if they do, treat it as corrupted + // data and panic rather than creating out-of-bounds slices in `bids()`/`asks_reversed()`. + let used = header.bids_len as usize + header.asks_len as usize; + + if used > capacity { + msg!("OrderBook header lengths (bids_len + asks_len) exceed capacity"); + return Err(ProgramError::InvalidAccountData); + } + + Ok(Self { + header, + capacity, + levels: levels_bytes.as_mut_ptr() as *mut OrderLevel, + }) + } + + pub fn update_from(&mut self, updates: BookUpdate) { + self.add_bids(&updates.bids); + self.add_asks(&updates.asks); + } + + pub fn add_bids( + &mut self, + bids: &[OrderLevel], + ) -> Option<&'a [OrderLevel]> { + if self.remaining_capacity() < bids.len() { + return None; + } + let new_bids_len = self.bids_len() + bids.len(); + let bids_space = + unsafe { self.bids_with_uninitialized_slots(new_bids_len) }; + + bids_space[self.bids_len()..].copy_from_slice(bids); + self.header.bids_len = new_bids_len as u32; + + Some(bids_space) + } + + pub fn add_asks( + &mut self, + asks: &[OrderLevel], + ) -> Option<&'a [OrderLevel]> { + if self.remaining_capacity() < asks.len() { + return None; + } + let new_asks_len = self.asks_len() + asks.len(); + let asks_space = + unsafe { self.asks_with_uninitialized_slots(new_asks_len) }; + + // copy in the reverse order + for (dst, src) in + asks_space[..asks.len()].iter_mut().zip(asks.iter().rev()) + { + *dst = *src; + } + self.header.asks_len = new_asks_len as u32; + + Some(asks_space) + } + + pub fn bids(&self) -> &'a [OrderLevel] { + unsafe { slice::from_raw_parts(self.levels, self.bids_len()) } + } + + /// Note that the returned slice is in reverse order, means the first entry is the latest + /// entry and the last entry is the oldest entry. + pub fn asks_reversed(&self) -> &'a [OrderLevel] { + unsafe { + slice::from_raw_parts( + self.levels.add(self.capacity - self.asks_len()), + self.asks_len(), + ) + } + } + + pub fn bids_len(&self) -> usize { + self.header.bids_len as usize + } + + pub fn asks_len(&self) -> usize { + self.header.asks_len as usize + } + + unsafe fn bids_with_uninitialized_slots( + &mut self, + bids_len: usize, + ) -> &'a mut [OrderLevel] { + slice::from_raw_parts_mut(self.levels, bids_len) + } + + unsafe fn asks_with_uninitialized_slots( + &mut self, + asks_len: usize, + ) -> &'a mut [OrderLevel] { + slice::from_raw_parts_mut( + self.levels.add(self.capacity - asks_len), + asks_len, + ) + } + + fn remaining_capacity(&self) -> usize { + self.capacity + .checked_sub((self.header.bids_len + self.header.asks_len) as usize) + .expect("remaining_capacity must exist") + } +} diff --git a/test-integration/programs/schedulecommit/src/utils/mod.rs b/test-integration/programs/schedulecommit/src/utils/mod.rs index 659e37842..6cd4f4c20 100644 --- a/test-integration/programs/schedulecommit/src/utils/mod.rs +++ b/test-integration/programs/schedulecommit/src/utils/mod.rs @@ -72,6 +72,12 @@ pub fn allocate_account_and_assign_owner( .max(1) .saturating_sub(account_info.lamports()); + msg!( + "required_lamports: {}, payer has {}", + required_lamports, + payer_info.lamports() + ); + // 1. Transfer the required rent to the account if required_lamports > 0 { transfer_lamports(payer_info, account_info, required_lamports)?; diff --git a/test-integration/schedulecommit/client/src/schedule_commit_context.rs b/test-integration/schedulecommit/client/src/schedule_commit_context.rs index ac30d9aa7..2fc1d1892 100644 --- a/test-integration/schedulecommit/client/src/schedule_commit_context.rs +++ b/test-integration/schedulecommit/client/src/schedule_commit_context.rs @@ -5,7 +5,7 @@ use integration_test_tools::IntegrationTestContext; use log::*; use program_schedulecommit::api::{ delegate_account_cpi_instruction, init_account_instruction, - init_payer_escrow, pda_and_bump, + init_order_book_instruction, init_payer_escrow, UserSeeds, }; use solana_rpc_client::rpc_client::{RpcClient, SerializableTransaction}; use solana_rpc_client_api::config::RpcSendTransactionConfig; @@ -13,6 +13,7 @@ use solana_rpc_client_api::config::RpcSendTransactionConfig; use solana_sdk::signer::SeedDerivable; use solana_sdk::{ commitment_config::CommitmentConfig, + compute_budget::ComputeBudgetInstruction, hash::Hash, native_token::LAMPORTS_PER_SOL, pubkey::Pubkey, @@ -30,6 +31,7 @@ pub struct ScheduleCommitTestContext { pub payer_ephem: Keypair, // The Payer keypairs along with its PDA pubkey which we'll commit pub committees: Vec<(Keypair, Pubkey)>, + user_seed: UserSeeds, common_ctx: IntegrationTestContext, } @@ -61,14 +63,21 @@ impl ScheduleCommitTestContext { // ----------------- // Init // ----------------- - pub fn try_new_random_keys(ncommittees: usize) -> Result { - Self::try_new_internal(ncommittees, true) + pub fn try_new_random_keys( + ncommittees: usize, + user_seed: UserSeeds, + ) -> Result { + Self::try_new_internal(ncommittees, true, user_seed) } - pub fn try_new(ncommittees: usize) -> Result { - Self::try_new_internal(ncommittees, false) + pub fn try_new(ncommittees: usize, user_seed: UserSeeds) -> Result { + Self::try_new_internal(ncommittees, false, user_seed) } - fn try_new_internal(ncommittees: usize, random_keys: bool) -> Result { + fn try_new_internal( + ncommittees: usize, + random_keys: bool, + user_seed: UserSeeds, + ) -> Result { let ictx = IntegrationTestContext::try_new()?; let payer_chain = if random_keys { @@ -103,7 +112,10 @@ impl ScheduleCommitTestContext { lamports, ) .unwrap(); - let (pda, _) = pda_and_bump(&payer_ephem.pubkey()); + let (pda, _bump) = Pubkey::find_program_address( + &[user_seed.bytes(), payer_ephem.pubkey().as_ref()], + &program_schedulecommit::ID, + ); (payer_ephem, pda) }) .collect::>(); @@ -143,6 +155,7 @@ impl ScheduleCommitTestContext { payer_ephem, committees, common_ctx: ictx, + user_seed, }) } @@ -150,17 +163,49 @@ impl ScheduleCommitTestContext { // Schedule Commit specific Transactions // ----------------- pub fn init_committees(&self) -> Result { - let ixs = self - .committees - .iter() - .map(|(player, committee)| { - init_account_instruction( - self.payer_chain.pubkey(), - player.pubkey(), - *committee, - ) - }) - .collect::>(); + let mut ixs = vec![ + ComputeBudgetInstruction::set_compute_unit_limit(1_400_000), + ComputeBudgetInstruction::set_compute_unit_price(10_000), + ]; + match self.user_seed { + UserSeeds::MagicScheduleCommit => { + ixs.extend(self.committees.iter().map( + |(player, committee)| { + init_account_instruction( + self.payer_chain.pubkey(), + player.pubkey(), + *committee, + ) + }, + )); + } + UserSeeds::OrderBook => { + ixs.extend(self.committees.iter().map( + |(book_manager, committee)| { + init_order_book_instruction( + self.payer_chain.pubkey(), + book_manager.pubkey(), + *committee, + ) + }, + )); + + //// TODO (snawaz): currently the size of delegatable-account cannot be + //// more than 10K, else delegation will fail. So Let's revisit this when + //// we relax the limit on the account size, then we can use larger + //// account, say even 10 MB, and execute CommitDiff. + // + // ixs.extend(self.committees.iter().flat_map( + // |(payer, committee)| { + // [grow_order_book_instruction( + // payer.pubkey(), + // *committee, + // 10 * 1024 + // )] + // }, + // )); + } + }; let mut signers = self .committees @@ -224,6 +269,7 @@ impl ScheduleCommitTestContext { self.payer_chain.pubkey(), self.ephem_validator_identity, player.pubkey(), + self.user_seed, ); ixs.push(ix); } diff --git a/test-integration/schedulecommit/client/src/verify.rs b/test-integration/schedulecommit/client/src/verify.rs index 11098f9c9..d383cbc2f 100644 --- a/test-integration/schedulecommit/client/src/verify.rs +++ b/test-integration/schedulecommit/client/src/verify.rs @@ -1,5 +1,5 @@ use integration_test_tools::scheduled_commits::ScheduledCommitResult; -use program_schedulecommit::MainAccount; +use program_schedulecommit::{MainAccount, OrderBookOwned}; use solana_sdk::signature::Signature; use crate::ScheduleCommitTestContext; @@ -12,3 +12,12 @@ pub fn fetch_and_verify_commit_result_from_logs( res.confirm_commit_transactions_on_chain(ctx).unwrap(); res } + +pub fn fetch_and_verify_order_book_commit_result_from_logs( + ctx: &ScheduleCommitTestContext, + sig: Signature, +) -> ScheduledCommitResult { + let res = ctx.fetch_schedule_commit_result(sig).unwrap(); + res.confirm_commit_transactions_on_chain(ctx).unwrap(); + res +} diff --git a/test-integration/schedulecommit/elfs/dlp.so b/test-integration/schedulecommit/elfs/dlp.so index f07df31f3..1a169270b 100755 Binary files a/test-integration/schedulecommit/elfs/dlp.so and b/test-integration/schedulecommit/elfs/dlp.so differ diff --git a/test-integration/schedulecommit/test-scenarios/Cargo.toml b/test-integration/schedulecommit/test-scenarios/Cargo.toml index 93d93863a..3cda55a8f 100644 --- a/test-integration/schedulecommit/test-scenarios/Cargo.toml +++ b/test-integration/schedulecommit/test-scenarios/Cargo.toml @@ -16,3 +16,5 @@ solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-sdk = { workspace = true } test-kit = { workspace = true } +rand = { workspace = true } +borsh = { workspace = true } diff --git a/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs b/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs index 8f07d51a7..4c7456b43 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/01_commits.rs @@ -5,7 +5,7 @@ use log::*; use program_schedulecommit::{ api::{ delegate_account_cpi_instruction, init_account_instruction, - pda_and_bump, schedule_commit_cpi_instruction, + pda_and_bump, schedule_commit_cpi_instruction, UserSeeds, }, ScheduleCommitCpiArgs, ScheduleCommitInstruction, }; @@ -45,7 +45,10 @@ mod utils; #[test] fn test_committing_one_account() { run_test!({ - let ctx = get_context_with_delegated_committees(1); + let ctx = get_context_with_delegated_committees( + 1, + UserSeeds::MagicScheduleCommit, + ); let ScheduleCommitTestContextFields { payer_ephem: payer, @@ -98,7 +101,10 @@ fn test_committing_one_account() { #[test] fn test_committing_two_accounts() { run_test!({ - let ctx = get_context_with_delegated_committees(2); + let ctx = get_context_with_delegated_committees( + 2, + UserSeeds::MagicScheduleCommit, + ); let ScheduleCommitTestContextFields { payer_ephem: payer, @@ -229,6 +235,7 @@ fn init_and_delegate_player( payer.pubkey(), validator, player.pubkey(), + UserSeeds::MagicScheduleCommit, ); // Send transaction diff --git a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs index 550cdd508..f2d9c7ba0 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/02_commit_and_undelegate.rs @@ -10,10 +10,12 @@ use program_schedulecommit::{ schedule_commit_and_undelegate_cpi_instruction, schedule_commit_and_undelegate_cpi_twice, schedule_commit_and_undelegate_cpi_with_mod_after_instruction, - set_count_instruction, + schedule_commit_diff_instruction_for_order_book, set_count_instruction, + update_order_book_instruction, UserSeeds, }, - FAIL_UNDELEGATION_COUNT, + BookUpdate, OrderLevel, FAIL_UNDELEGATION_COUNT, }; +use rand::{RngCore, SeedableRng}; use schedulecommit_client::{ verify, ScheduleCommitTestContext, ScheduleCommitTestContextFields, }; @@ -51,7 +53,10 @@ fn commit_and_undelegate_one_account( Signature, Result, ) { - let ctx = get_context_with_delegated_committees(1); + let ctx = get_context_with_delegated_committees( + 1, + UserSeeds::MagicScheduleCommit, + ); let ScheduleCommitTestContextFields { payer_ephem: payer, committees, @@ -105,6 +110,62 @@ fn commit_and_undelegate_one_account( (ctx, *sig, tx_res) } +fn commit_and_undelegate_order_book_account( + update: BookUpdate, +) -> ( + ScheduleCommitTestContext, + Signature, + Result, +) { + let ctx = get_context_with_delegated_committees(1, UserSeeds::OrderBook); + let ScheduleCommitTestContextFields { + payer_ephem, + committees, + commitment, + ephem_client, + .. + } = ctx.fields(); + + assert_eq!(committees.len(), 1); + + let ixs = [ + update_order_book_instruction( + payer_ephem.pubkey(), + committees[0].1, + update, + ), + schedule_commit_diff_instruction_for_order_book( + payer_ephem.pubkey(), + committees[0].1, + magicblock_magic_program_api::id(), + magicblock_magic_program_api::MAGIC_CONTEXT_PUBKEY, + ), + ]; + + let ephem_blockhash = ephem_client.get_latest_blockhash().unwrap(); + let tx = Transaction::new_signed_with_payer( + &ixs, + Some(&payer_ephem.pubkey()), + &[&payer_ephem], + ephem_blockhash, + ); + + let sig = tx.get_signature(); + let tx_res = ephem_client + .send_and_confirm_transaction_with_spinner_and_config( + &tx, + *commitment, + RpcSendTransactionConfig { + skip_preflight: true, + ..Default::default() + }, + ); + debug!("txhash (scheduled_commit): {:?}", tx_res); + + debug!("Commit and Undelegate Transaction result: '{:?}'", tx_res); + (ctx, *sig, tx_res) +} + fn commit_and_undelegate_two_accounts( modify_after: bool, ) -> ( @@ -112,7 +173,10 @@ fn commit_and_undelegate_two_accounts( Signature, Result, ) { - let ctx = get_context_with_delegated_committees(2); + let ctx = get_context_with_delegated_committees( + 2, + UserSeeds::MagicScheduleCommit, + ); let ScheduleCommitTestContextFields { payer_ephem: payer, committees, @@ -172,7 +236,10 @@ fn commit_and_undelegate_two_accounts_twice() -> ( Signature, Result, ) { - let ctx = get_context_with_delegated_committees(2); + let ctx = get_context_with_delegated_committees( + 2, + UserSeeds::MagicScheduleCommit, + ); let ScheduleCommitTestContextFields { payer_ephem: payer, committees, @@ -230,6 +297,70 @@ fn test_committing_and_undelegating_one_account() { }); } +#[test] +fn test_committing_and_undelegating_huge_order_book_account() { + run_test!({ + let (rng_seed, update) = { + use rand::{ + rngs::{OsRng, StdRng}, + Rng, + }; + let rng_seed = OsRng.next_u64(); + println!("Important: use {rng_seed} as seed to regenerate the random inputs in case of test failure"); + let mut random = StdRng::seed_from_u64(rng_seed); + let mut update = BookUpdate::default(); + update.bids.extend((0..random.gen_range(5..10)).map(|_| { + OrderLevel { + price: random.gen_range(75000..90000), + size: random.gen_range(1..10), + } + })); + update.asks.extend((0..random.gen_range(5..10)).map(|_| { + OrderLevel { + price: random.gen_range(125000..150000), + size: random.gen_range(1..10), + } + })); + (rng_seed, update) + }; + let (ctx, sig, tx_res) = + commit_and_undelegate_order_book_account(update.clone()); + info!("'{}' {:?}", sig, tx_res); + + let res = verify::fetch_and_verify_order_book_commit_result_from_logs( + &ctx, sig, + ); + + let book = res + .included + .values() + .next() + .expect("one order-book must exist"); + + assert_eq!( + book.bids.len(), + update.bids.len(), + "Use {rng_seed} to generate the input and investigate" + ); + assert_eq!( + book.asks.len(), + update.asks.len(), + "Use {rng_seed} to generate the input and investigate" + ); + assert_eq!( + book.bids, update.bids, + "Use {rng_seed} to generate the input and investigate" + ); + assert_eq!( + book.asks, update.asks, + "Use {rng_seed} to generate the input and investigate" + ); + + assert_one_committee_was_committed(&ctx, &res, true); + assert_one_committee_account_was_undelegated_on_chain(&ctx); + }); +} + #[test] fn test_committing_and_undelegating_two_accounts_success() { run_test!({ @@ -607,7 +738,10 @@ fn test_committing_and_undelegating_two_accounts_twice() { #[test] fn test_committing_after_failed_undelegation() { run_test!({ - let ctx = get_context_with_delegated_committees(1); + let ctx = get_context_with_delegated_committees( + 1, + UserSeeds::MagicScheduleCommit, + ); let ScheduleCommitTestContextFields { payer_ephem: payer, committees, diff --git a/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs b/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs index eb8f80dfb..342f94ed2 100644 --- a/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs +++ b/test-integration/schedulecommit/test-scenarios/tests/utils/mod.rs @@ -1,6 +1,6 @@ use ephemeral_rollups_sdk::consts::DELEGATION_PROGRAM_ID; use integration_test_tools::scheduled_commits::ScheduledCommitResult; -use program_schedulecommit::MainAccount; +use program_schedulecommit::{api::UserSeeds, MainAccount}; use schedulecommit_client::ScheduleCommitTestContext; use solana_rpc_client_api::client_error; use solana_sdk::{ @@ -15,16 +15,22 @@ use solana_sdk::{ // ----------------- pub fn get_context_with_delegated_committees( ncommittees: usize, + user_seed: UserSeeds, ) -> ScheduleCommitTestContext { let ctx = if std::env::var("FIXED_KP").is_ok() { - ScheduleCommitTestContext::try_new(ncommittees) + ScheduleCommitTestContext::try_new(ncommittees, user_seed) } else { - ScheduleCommitTestContext::try_new_random_keys(ncommittees) + ScheduleCommitTestContext::try_new_random_keys(ncommittees, user_seed) } .unwrap(); + println!("get_context_with_delegated_committees inside"); + + let txhash = ctx.init_committees().unwrap(); + println!("txhash (init_committees): {}", txhash); + + let txhash = ctx.delegate_committees().unwrap(); + println!("txhash (delegate_committees): {}", txhash); - ctx.init_committees().unwrap(); - ctx.delegate_committees().unwrap(); ctx } @@ -32,11 +38,13 @@ pub fn get_context_with_delegated_committees( // Asserts // ----------------- #[allow(dead_code)] // used in 02_commit_and_undelegate.rs -pub fn assert_one_committee_was_committed( +pub fn assert_one_committee_was_committed( ctx: &ScheduleCommitTestContext, - res: &ScheduledCommitResult, + res: &ScheduledCommitResult, is_single_stage: bool, -) { +) where + T: std::fmt::Debug + borsh::BorshDeserialize + PartialEq + Eq, +{ let pda = ctx.committees[0].1; assert_eq!(res.included.len(), 1, "includes 1 pda"); diff --git a/test-integration/schedulecommit/test-security/tests/01_invocations.rs b/test-integration/schedulecommit/test-security/tests/01_invocations.rs index b05168035..8fa8ed255 100644 --- a/test-integration/schedulecommit/test-security/tests/01_invocations.rs +++ b/test-integration/schedulecommit/test-security/tests/01_invocations.rs @@ -1,4 +1,4 @@ -use program_schedulecommit::api::schedule_commit_cpi_instruction; +use program_schedulecommit::api::{schedule_commit_cpi_instruction, UserSeeds}; use schedulecommit_client::{ ScheduleCommitTestContext, ScheduleCommitTestContextFields, }; @@ -27,9 +27,12 @@ const NEEDS_TO_BE_OWNED_BY_INVOKING_PROGRAM: &str = fn prepare_ctx_with_account_to_commit() -> ScheduleCommitTestContext { let ctx = if std::env::var("FIXED_KP").is_ok() { - ScheduleCommitTestContext::try_new(2) + ScheduleCommitTestContext::try_new(2, UserSeeds::MagicScheduleCommit) } else { - ScheduleCommitTestContext::try_new_random_keys(2) + ScheduleCommitTestContext::try_new_random_keys( + 2, + UserSeeds::MagicScheduleCommit, + ) } .unwrap(); ctx.init_committees().unwrap(); diff --git a/test-integration/test-committor-service/Cargo.toml b/test-integration/test-committor-service/Cargo.toml index 7d74150f1..c6b448d1e 100644 --- a/test-integration/test-committor-service/Cargo.toml +++ b/test-integration/test-committor-service/Cargo.toml @@ -23,6 +23,7 @@ magicblock-table-mania = { workspace = true, features = [ "randomize_lookup_table_slot", ] } program-flexi-counter = { workspace = true, features = ["no-entrypoint"] } +program-schedulecommit = { workspace = true, features = ["no-entrypoint"] } rand = { workspace = true } solana-account = { workspace = true } solana-pubkey = { workspace = true } diff --git a/test-integration/test-committor-service/tests/common.rs b/test-integration/test-committor-service/tests/common.rs index 9966ff56a..2c3e1b0e9 100644 --- a/test-integration/test-committor-service/tests/common.rs +++ b/test-integration/test-committor-service/tests/common.rs @@ -10,7 +10,8 @@ use async_trait::async_trait; use magicblock_committor_service::{ intent_executor::{ task_info_fetcher::{ - ResetType, TaskInfoFetcher, TaskInfoFetcherResult, + ResetType, TaskInfoFetcher, TaskInfoFetcherError, + TaskInfoFetcherResult, }, IntentExecutorImpl, }, @@ -104,17 +105,22 @@ impl TestFixture { ) -> IntentExecutorImpl { let transaction_preparator = self.create_transaction_preparator(); - let task_info_fetcher = Arc::new(MockTaskInfoFetcher); IntentExecutorImpl::new( self.rpc_client.clone(), transaction_preparator, - task_info_fetcher, + self.create_task_info_fetcher(), ) } + + #[allow(dead_code)] + pub fn create_task_info_fetcher(&self) -> Arc { + Arc::new(MockTaskInfoFetcher(self.rpc_client.clone())) + } } -pub struct MockTaskInfoFetcher; +pub struct MockTaskInfoFetcher(MagicblockRpcClient); + #[async_trait] impl TaskInfoFetcher for MockTaskInfoFetcher { async fn fetch_next_commit_ids( @@ -136,6 +142,25 @@ impl TaskInfoFetcher for MockTaskInfoFetcher { } fn reset(&self, _: ResetType) {} + + async fn get_base_accounts( + &self, + pubkeys: &[Pubkey], + ) -> TaskInfoFetcherResult> { + self.0 + .get_multiple_accounts(pubkeys, None) + .await + .map_err(|err| { + TaskInfoFetcherError::MagicBlockRpcClientError(Box::new(err)) + }) + .map(|accounts| { + pubkeys + .iter() + .zip(accounts.into_iter()) + .filter_map(|(key, value)| value.map(|value| (*key, value))) + .collect() + }) + } } #[allow(dead_code)] diff --git a/test-integration/test-committor-service/tests/test_delivery_preparator.rs b/test-integration/test-committor-service/tests/test_delivery_preparator.rs index 5c48fa22b..dded722ee 100644 --- a/test-integration/test-committor-service/tests/test_delivery_preparator.rs +++ b/test-integration/test-committor-service/tests/test_delivery_preparator.rs @@ -1,4 +1,5 @@ use borsh::BorshDeserialize; +use futures::future::join_all; use magicblock_committor_program::Chunks; use magicblock_committor_service::{ persist::IntentPersisterImpl, @@ -86,15 +87,12 @@ async fn test_prepare_multiple_buffers() { generate_random_bytes(10), generate_random_bytes(500 * 1024), ]; - let buffer_tasks = datas - .iter() - .map(|data| { - let task = - BufferTaskType::Commit(create_commit_task(data.as_slice())); - Box::new(BufferTask::new_preparation_required(task)) - as Box - }) - .collect(); + let buffer_tasks = join_all(datas.iter().map(|data| async { + let task = BufferTaskType::Commit(create_commit_task(data.as_slice())); + Box::new(BufferTask::new_preparation_required(task)) + as Box + })) + .await; let mut strategy = TransactionStrategy { optimized_tasks: buffer_tasks, lookup_tables_keys: vec![], @@ -165,14 +163,11 @@ async fn test_lookup_tables() { generate_random_bytes(20), generate_random_bytes(30), ]; - let tasks = datas - .iter() - .map(|data| { - let task = - ArgsTaskType::Commit(create_commit_task(data.as_slice())); - Box::::new(task.into()) as Box - }) - .collect::>(); + let tasks = join_all(datas.iter().map(|data| async { + let task = ArgsTaskType::Commit(create_commit_task(data.as_slice())); + Box::::new(task.into()) as Box + })) + .await; let lookup_tables_keys = TaskStrategist::collect_lookup_table_keys( &fixture.authority.pubkey(), diff --git a/test-integration/test-committor-service/tests/test_ix_commit_local.rs b/test-integration/test-committor-service/tests/test_ix_commit_local.rs index 16ea10441..6af953d47 100644 --- a/test-integration/test-committor-service/tests/test_ix_commit_local.rs +++ b/test-integration/test-committor-service/tests/test_ix_commit_local.rs @@ -31,6 +31,7 @@ use test_kit::init_logger; use tokio::task::JoinSet; use utils::transactions::tx_logs_contain; +use self::utils::transactions::init_and_delegate_order_book_on_chain; use crate::utils::{ ensure_validator_authority, transactions::{ @@ -63,6 +64,7 @@ fn expect_strategies( // ----------------- // Single Account Commits // ----------------- + #[tokio::test] async fn test_ix_commit_single_account_100_bytes() { commit_single_account(100, CommitStrategy::Args, false).await; @@ -75,22 +77,47 @@ async fn test_ix_commit_single_account_100_bytes_and_undelegate() { #[tokio::test] async fn test_ix_commit_single_account_800_bytes() { - commit_single_account(800, CommitStrategy::FromBuffer, false).await; + commit_single_account(800, CommitStrategy::Args, false).await; } #[tokio::test] async fn test_ix_commit_single_account_800_bytes_and_undelegate() { - commit_single_account(800, CommitStrategy::FromBuffer, true).await; + commit_single_account(800, CommitStrategy::Args, true).await; } #[tokio::test] async fn test_ix_commit_single_account_one_kb() { - commit_single_account(1024, CommitStrategy::FromBuffer, false).await; + commit_single_account(1024, CommitStrategy::Args, false).await; } #[tokio::test] async fn test_ix_commit_single_account_ten_kb() { - commit_single_account(10 * 1024, CommitStrategy::FromBuffer, false).await; + commit_single_account(10 * 1024, CommitStrategy::Args, false).await; +} + +#[tokio::test] +async fn test_ix_commit_order_book_change_100_bytes() { + commit_book_order_account(100, CommitStrategy::Args, false).await; +} + +#[tokio::test] +async fn test_ix_commit_order_book_change_679_bytes() { + commit_book_order_account(679, CommitStrategy::Args, false).await; +} + +#[tokio::test] +async fn test_ix_commit_order_book_change_680_bytes() { + // We cannot use 680 as changed_len because that both 679 and 680 produce encoded tx + // of size 1644 (which is the max limit), but while the size of raw bytes for 679 is within + // 1232 limit, the size for 680 execeds by 1 (1233). That is why we used + // 681 as changed_len where CommitStrategy goes from Args to FromBuffer. + commit_book_order_account(681, CommitStrategy::FromBuffer, false).await; +} + +#[tokio::test] +async fn test_ix_commit_order_book_change_10k_bytes() { + commit_book_order_account(10 * 1024, CommitStrategy::FromBuffer, false) + .await; } async fn commit_single_account( @@ -158,12 +185,78 @@ async fn commit_single_account( .await; } -// TODO(thlorenz): once delegation program supports larger commits +async fn commit_book_order_account( + changed_len: usize, + expected_strategy: CommitStrategy, + undelegate: bool, +) { + init_logger!(); + + let validator_auth = ensure_validator_authority(); + fund_validator_auth_and_ensure_validator_fees_vault(&validator_auth).await; + + // Run each test with and without finalizing + let service = CommittorService::try_start( + validator_auth.insecure_clone(), + ":memory:", + ChainConfig::local(ComputeBudgetConfig::new(1_000_000)), + ) + .unwrap(); + let service = CommittorServiceExt::new(Arc::new(service)); + + let payer = Keypair::new(); + let (order_book_pk, mut order_book_ac) = + init_and_delegate_order_book_on_chain(&payer).await; + + // Modify bytes so that a diff is produced and is sent to DLP + let data = &mut order_book_ac.data; + assert!(changed_len <= data.len()); + for byte in &mut order_book_ac.data[..changed_len] { + *byte = byte.wrapping_add(1); + } + order_book_ac.owner = program_schedulecommit::id(); + + // We should always be able to Commit & Finalize 1 account either with Args or Buffers + let account = CommittedAccount { + pubkey: order_book_pk, + account: order_book_ac, + }; + let base_intent = if undelegate { + MagicBaseIntent::CommitAndUndelegate(CommitAndUndelegate { + commit_action: CommitType::Standalone(vec![account]), + undelegate_action: UndelegateType::Standalone, + }) + } else { + MagicBaseIntent::Commit(CommitType::Standalone(vec![account])) + }; + + let intent = ScheduledBaseIntentWrapper { + trigger_type: TriggerType::OnChain, + inner: ScheduledBaseIntent { + id: 0, + slot: 10, + blockhash: Hash::new_unique(), + action_sent_transaction: Transaction::default(), + payer: payer.pubkey(), + base_intent, + }, + }; + + ix_commit_local( + service, + vec![intent], + expect_strategies(&[(expected_strategy, 1)]), + ) + .await; +} + +// TODO(thlorenz/snawaz): once delegation program supports larger commits // add 1MB and 10MB tests // ----------------- // Multiple Account Commits // ----------------- + #[tokio::test] async fn test_ix_commit_two_accounts_1kb_2kb() { init_logger!(); @@ -171,7 +264,7 @@ async fn test_ix_commit_two_accounts_1kb_2kb() { &[1024, 2048], 1, false, - expect_strategies(&[(CommitStrategy::FromBuffer, 2)]), + expect_strategies(&[(CommitStrategy::Args, 2)]), ) .await; } @@ -219,25 +312,22 @@ async fn test_ix_commit_four_accounts_1kb_2kb_5kb_10kb_single_bundle() { &[1024, 2 * 1024, 5 * 1024, 10 * 1024], 1, false, - expect_strategies(&[(CommitStrategy::FromBuffer, 4)]), + expect_strategies(&[(CommitStrategy::Args, 4)]), ) .await; } #[tokio::test] async fn test_commit_20_accounts_1kb_bundle_size_2() { - commit_20_accounts_1kb( - 2, - expect_strategies(&[(CommitStrategy::FromBuffer, 20)]), - ) - .await; + commit_20_accounts_1kb(2, expect_strategies(&[(CommitStrategy::Args, 20)])) + .await; } #[tokio::test] async fn test_commit_5_accounts_1kb_bundle_size_3() { commit_5_accounts_1kb( 3, - expect_strategies(&[(CommitStrategy::FromBuffer, 5)]), + expect_strategies(&[(CommitStrategy::Args, 5)]), false, ) .await; @@ -247,7 +337,11 @@ async fn test_commit_5_accounts_1kb_bundle_size_3() { async fn test_commit_5_accounts_1kb_bundle_size_3_undelegate_all() { commit_5_accounts_1kb( 3, - expect_strategies(&[(CommitStrategy::FromBuffer, 5)]), + expect_strategies(&[ + // Intent fits in 1 TX only with ALT, see IntentExecutorImpl::try_unite_tasks + (CommitStrategy::FromBufferWithLookupTable, 3), + (CommitStrategy::Args, 2), + ]), true, ) .await; @@ -258,7 +352,7 @@ async fn test_commit_5_accounts_1kb_bundle_size_4() { commit_5_accounts_1kb( 4, expect_strategies(&[ - (CommitStrategy::FromBuffer, 1), + (CommitStrategy::Args, 1), (CommitStrategy::FromBufferWithLookupTable, 4), ]), false, @@ -271,7 +365,7 @@ async fn test_commit_5_accounts_1kb_bundle_size_4_undelegate_all() { commit_5_accounts_1kb( 4, expect_strategies(&[ - (CommitStrategy::FromBuffer, 1), + (CommitStrategy::Args, 1), (CommitStrategy::FromBufferWithLookupTable, 4), ]), true, @@ -291,11 +385,8 @@ async fn test_commit_5_accounts_1kb_bundle_size_5_undelegate_all() { #[tokio::test] async fn test_commit_20_accounts_1kb_bundle_size_3() { - commit_20_accounts_1kb( - 3, - expect_strategies(&[(CommitStrategy::FromBuffer, 20)]), - ) - .await; + commit_20_accounts_1kb(3, expect_strategies(&[(CommitStrategy::Args, 20)])) + .await; } #[tokio::test] @@ -314,7 +405,7 @@ async fn test_commit_20_accounts_1kb_bundle_size_6() { expect_strategies(&[ (CommitStrategy::FromBufferWithLookupTable, 18), // Two accounts don't make it into the bundles of size 6 - (CommitStrategy::FromBuffer, 2), + (CommitStrategy::Args, 2), ]), ) .await; @@ -476,7 +567,7 @@ async fn commit_multiple_accounts( ix_commit_local(service, intents, expected_strategies).await; } -// TODO(thlorenz): once delegation program supports larger commits add the following +// TODO(thlorenz/snawaz): once delegation program supports larger commits add the following // tests // // ## Scenario 1 diff --git a/test-integration/test-committor-service/tests/test_transaction_preparator.rs b/test-integration/test-committor-service/tests/test_transaction_preparator.rs index de2f4ea57..b82aa3aa5 100644 --- a/test-integration/test-committor-service/tests/test_transaction_preparator.rs +++ b/test-integration/test-committor-service/tests/test_transaction_preparator.rs @@ -4,11 +4,11 @@ use magicblock_committor_service::{ persist::IntentPersisterImpl, tasks::{ args_task::{ArgsTask, ArgsTaskType}, - buffer_task::{BufferTask, BufferTaskType}, + buffer_task::BufferTask, task_strategist::{TaskStrategist, TransactionStrategy}, utils::TransactionUtils, - BaseActionTask, BaseTask, CommitTask, FinalizeTask, PreparationState, - UndelegateTask, + BaseActionTask, BaseTask, FinalizeTask, PreparationState, + TaskBuilderImpl, UndelegateTask, }, transaction_preparator::TransactionPreparator, }; @@ -35,11 +35,12 @@ async fn test_prepare_commit_tx_with_single_account() { let committed_account = create_committed_account(&account_data); let tasks = vec![ - Box::new(ArgsTask::new(ArgsTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account.clone(), - allow_undelegation: true, - }))) as Box, + Box::new(TaskBuilderImpl::create_commit_task( + 1, + true, + committed_account.clone(), + None, + )) as Box, Box::new(ArgsTask::new(ArgsTaskType::Finalize(FinalizeTask { delegated_account: committed_account.pubkey, }))), @@ -90,20 +91,24 @@ async fn test_prepare_commit_tx_with_multiple_accounts() { let committed_account2 = create_committed_account(&account2_data); let buffer_commit_task = BufferTask::new_preparation_required( - BufferTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account2.clone(), - allow_undelegation: true, - }), + TaskBuilderImpl::create_commit_task( + 1, + true, + committed_account2.clone(), + None, + ) + .task_type + .into(), ); // Create test data let tasks = vec![ // account 1 - Box::new(ArgsTask::new(ArgsTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account1.clone(), - allow_undelegation: true, - }))) as Box, + Box::new(TaskBuilderImpl::create_commit_task( + 1, + true, + committed_account1.clone(), + None, + )) as Box, // account 2 Box::new(buffer_commit_task), // finalize account 1 @@ -187,11 +192,14 @@ async fn test_prepare_commit_tx_with_base_actions() { }; let buffer_commit_task = BufferTask::new_preparation_required( - BufferTaskType::Commit(CommitTask { - commit_id: 1, - committed_account: committed_account.clone(), - allow_undelegation: true, - }), + TaskBuilderImpl::create_commit_task( + 1, + true, + committed_account.clone(), + None, + ) + .task_type + .into(), ); let tasks = vec![ // commit account diff --git a/test-integration/test-committor-service/tests/utils/instructions.rs b/test-integration/test-committor-service/tests/utils/instructions.rs index 9b7e3ffbd..0fcb37f97 100644 --- a/test-integration/test-committor-service/tests/utils/instructions.rs +++ b/test-integration/test-committor-service/tests/utils/instructions.rs @@ -1,5 +1,6 @@ use solana_pubkey::Pubkey; -use solana_sdk::{instruction::Instruction, rent::Rent}; +use solana_sdk::{instruction::Instruction, rent::Rent, signature::Keypair}; +use test_kit::Signer; pub fn init_validator_fees_vault_ix(validator_auth: Pubkey) -> Instruction { dlp::instruction_builder::init_validator_fees_vault( @@ -49,3 +50,45 @@ pub fn init_account_and_delegate_ixs( rent_excempt: rent_exempt, } } + +pub struct InitOrderBookAndDelegateIxs { + pub init: Instruction, + pub delegate: Instruction, + pub book_manager: Keypair, + pub order_book: Pubkey, +} + +pub fn init_order_book_account_and_delegate_ixs( + payer: Pubkey, +) -> InitOrderBookAndDelegateIxs { + use program_schedulecommit::{api, ID}; + + let book_manager = Keypair::new(); + + println!("schedulecommit ID: {}", ID); + + let (order_book, _bump) = Pubkey::find_program_address( + &[b"order_book", book_manager.pubkey().as_ref()], + &ID, + ); + + let init_ix = api::init_order_book_instruction( + payer, + book_manager.pubkey(), + order_book, + ); + + let delegate_ix = api::delegate_account_cpi_instruction( + payer, + None, + book_manager.pubkey(), + api::UserSeeds::OrderBook, + ); + + InitOrderBookAndDelegateIxs { + init: init_ix, + delegate: delegate_ix, + book_manager, + order_book, + } +} diff --git a/test-integration/test-committor-service/tests/utils/transactions.rs b/test-integration/test-committor-service/tests/utils/transactions.rs index 851be7e39..166850104 100644 --- a/test-integration/test-committor-service/tests/utils/transactions.rs +++ b/test-integration/test-committor-service/tests/utils/transactions.rs @@ -13,8 +13,9 @@ use solana_sdk::{ }; use crate::utils::instructions::{ - init_account_and_delegate_ixs, init_validator_fees_vault_ix, - InitAccountAndDelegateIxs, + init_account_and_delegate_ixs, init_order_book_account_and_delegate_ixs, + init_validator_fees_vault_ix, InitAccountAndDelegateIxs, + InitOrderBookAndDelegateIxs, }; #[macro_export] @@ -120,10 +121,21 @@ pub async fn tx_logs_contain( .log_messages .clone() .unwrap_or_else(Vec::new); - logs.iter().any(|log| log.contains(needle)) + logs.iter().any(|log| { + // Lots of existing tests pass "CommitState" as needle argument to this function, but since now CommitTask + // could invoke CommitState or CommitDiff depending on the size of the account, we also look for "CommitDiff" + // in the logs when needle == CommitState. It's easier to make this little adjustment here than computing + // the decision and passing either CommitState or CommitDiff from the tests themselves. + if needle == "CommitState" { + log.contains(needle) || log.contains("CommitDiff") + } else { + log.contains(needle) + } + }) } /// This needs to be run for each test that required a new counter to be delegated +#[allow(dead_code)] pub async fn init_and_delegate_account_on_chain( counter_auth: &Keypair, bytes: u64, @@ -223,6 +235,68 @@ pub async fn init_and_delegate_account_on_chain( (pda, pda_acc) } +/// This needs to be run for each test that required a new order_book to be delegated +#[allow(dead_code)] +pub async fn init_and_delegate_order_book_on_chain( + payer: &Keypair, +) -> (Pubkey, Account) { + let rpc_client = RpcClient::new("http://localhost:7799".to_string()); + + rpc_client + .request_airdrop(&payer.pubkey(), 777 * LAMPORTS_PER_SOL) + .await + .unwrap(); + + let InitOrderBookAndDelegateIxs { + init, + delegate, + book_manager, + order_book, + } = init_order_book_account_and_delegate_ixs(payer.pubkey()); + + let latest_block_hash = rpc_client.get_latest_blockhash().await.unwrap(); + + // Init account + rpc_client + .send_and_confirm_transaction_with_spinner_and_config( + &Transaction::new_signed_with_payer( + &[init], + Some(&payer.pubkey()), + &[payer, &book_manager], + latest_block_hash, + ), + CommitmentConfig::confirmed(), + RpcSendTransactionConfig { + skip_preflight: true, + ..Default::default() + }, + ) + .await + .expect("Failed to init account"); + + // Delegate account + rpc_client + .send_and_confirm_transaction_with_spinner_and_config( + &Transaction::new_signed_with_payer( + &[delegate], + Some(&payer.pubkey()), + &[&payer], + latest_block_hash, + ), + CommitmentConfig::confirmed(), + RpcSendTransactionConfig { + skip_preflight: true, + ..Default::default() + }, + ) + .await + .expect("Failed to delegate"); + + let order_book_acc = get_account!(rpc_client, order_book, "order_book"); + + (order_book, order_book_acc) +} + /// This needs to be run once for all tests pub async fn fund_validator_auth_and_ensure_validator_fees_vault( validator_auth: &Keypair, diff --git a/test-integration/test-ledger-restore/tests/08_commit_update.rs b/test-integration/test-ledger-restore/tests/08_commit_update.rs index 699a7d03d..da24d2af2 100644 --- a/test-integration/test-ledger-restore/tests/08_commit_update.rs +++ b/test-integration/test-ledger-restore/tests/08_commit_update.rs @@ -52,7 +52,7 @@ fn test_restore_ledger_committed_and_updated_account() { fn write(ledger_path: &Path, payer: &Keypair) -> (Child, u64) { let programs = get_programs_with_flexi_counter(); - let (_, mut validator, ctx) = setup_validator_with_local_remote( + let (_tmpdir, mut validator, ctx) = setup_validator_with_local_remote( ledger_path, Some(programs), true, @@ -167,7 +167,7 @@ fn read(ledger_path: &Path, payer_kp: &Keypair) -> Child { let payer = &payer_kp.pubkey(); let programs = get_programs_with_flexi_counter(); - let (_, mut validator, ctx) = setup_validator_with_local_remote( + let (_tmpdir, mut validator, ctx) = setup_validator_with_local_remote( ledger_path, Some(programs), false, diff --git a/test-integration/test-tools/src/integration_test_context.rs b/test-integration/test-tools/src/integration_test_context.rs index fcfc346e5..d1678ee10 100644 --- a/test-integration/test-tools/src/integration_test_context.rs +++ b/test-integration/test-tools/src/integration_test_context.rs @@ -157,7 +157,9 @@ impl IntegrationTestContext { rpc_client: Option<&RpcClient>, label: &str, ) -> Option> { - let rpc_client = rpc_client.or(self.chain_client.as_ref())?; + let rpc_client = rpc_client.unwrap_or_else(|| { + panic!("rpc_client for [{label}] does not exist") + }); // Try this up to 50 times since devnet here returns the version response instead of // the EncodedConfirmedTransactionWithStatusMeta at times @@ -166,6 +168,13 @@ impl IntegrationTestContext { &sig, RpcTransactionConfig { commitment: Some(self.commitment), + max_supported_transaction_version: if label == "chain" { + // base chain cluster requires explicit v0 support, + // while ephemeral uses default version handling + Some(0) + } else { + None + }, ..Default::default() }, ) { diff --git a/test-integration/test-tools/src/scheduled_commits.rs b/test-integration/test-tools/src/scheduled_commits.rs index 038326a06..0b64cb7c8 100644 --- a/test-integration/test-tools/src/scheduled_commits.rs +++ b/test-integration/test-tools/src/scheduled_commits.rs @@ -179,7 +179,7 @@ impl IntegrationTestContext { { // 1. Find scheduled commit sent signature via // ScheduledCommitSent signature: - let (ephem_logs, scheduled_commmit_sent_sig) = { + let (ephem_logs_l1, scheduled_commmit_sent_sig) = { let logs = self.fetch_ephemeral_logs(sig).with_context(|| { format!( "Scheduled commit sent logs not found for sig {:?}", @@ -196,17 +196,17 @@ impl IntegrationTestContext { }; // 2. Find chain commit signatures - let chain_logs = self + let ephem_logs_l2 = self .fetch_ephemeral_logs(scheduled_commmit_sent_sig) .with_context(|| { format!( "Logs {:#?}\nScheduled commit sent sig {:?}", - ephem_logs, scheduled_commmit_sent_sig + ephem_logs_l1, scheduled_commmit_sent_sig ) })?; let (included, excluded, feepayers, sigs) = - extract_sent_commit_info_from_logs(&chain_logs); + extract_sent_commit_info_from_logs(&ephem_logs_l2); let mut committed_accounts = HashMap::new(); for pubkey in included { @@ -226,6 +226,10 @@ impl IntegrationTestContext { }; } + for sig in sigs.iter() { + self.dump_chain_logs(*sig); + } + Ok(ScheduledCommitResult { included: committed_accounts, excluded,