diff --git a/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs b/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs index da91ffed7..8b94864d8 100644 --- a/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs +++ b/magicblock-chainlink/src/chainlink/fetch_cloner/mod.rs @@ -464,7 +464,7 @@ where Ok(Err(err)) => { error!( pubkey = %pubkey, - error = %err, + error = ?err, "Failed to fetch delegation record" ); (None, None) @@ -472,7 +472,7 @@ where Err(err) => { error!( pubkey = %pubkey, - error = %err, + error = ?err, "Failed to fetch delegation record" ); (None, None) @@ -744,6 +744,7 @@ where mark_empty_if_not_found, fetch_origin, program_ids, + None, ) .await?; diff --git a/magicblock-chainlink/src/remote_account_provider/errors.rs b/magicblock-chainlink/src/remote_account_provider/errors.rs index 15d7415e2..affadc858 100644 --- a/magicblock-chainlink/src/remote_account_provider/errors.rs +++ b/magicblock-chainlink/src/remote_account_provider/errors.rs @@ -62,11 +62,11 @@ pub enum RemoteAccountProviderError { #[error("Failed to resolve account ({0}) to track slots")] ClockAccountCouldNotBeResolved(String), - #[error("Failed to resolve accounts to same slot ({0}) to track slots")] - SlotsDidNotMatch(String, Vec), + #[error("Failed to resolve accounts to same slot ({0}) to track slots hit limit: {2}")] + SlotsDidNotMatch(String, Vec, String), - #[error("Accounts matched same slot ({0}), but it's less than min required context slot {2} ")] - MatchingSlotsNotSatisfyingMinContextSlot(String, Vec, u64), + #[error("Accounts matched same slot ({0}), but it's less than min required context slot {2} hit limit: {3}")] + MatchingSlotsNotSatisfyingMinContextSlot(String, Vec, u64, String), #[error("LRU capacity must be greater than 0")] InvalidLruCapacity, diff --git a/magicblock-chainlink/src/remote_account_provider/mod.rs b/magicblock-chainlink/src/remote_account_provider/mod.rs index d219a17d0..91bd27fae 100644 --- a/magicblock-chainlink/src/remote_account_provider/mod.rs +++ b/magicblock-chainlink/src/remote_account_provider/mod.rs @@ -550,7 +550,7 @@ impl RemoteAccountProvider { pubkey: Pubkey, fetch_origin: AccountFetchOrigin, ) -> RemoteAccountProviderResult { - self.try_get_multi(&[pubkey], None, fetch_origin, None) + self.try_get_multi(&[pubkey], None, fetch_origin, None, None) .await // SAFETY: we are guaranteed to have a single result here as // otherwise we would have gotten an error @@ -569,7 +569,7 @@ impl RemoteAccountProvider { // 1. Fetch the _normal_ way and hope the slots match and if required // the min_context_slot is met let remote_accounts = self - .try_get_multi(pubkeys, None, fetch_origin, None) + .try_get_multi(pubkeys, None, fetch_origin, None, None) .await?; if let Match = slots_match_and_meet_min_context( &remote_accounts, @@ -604,6 +604,10 @@ impl RemoteAccountProvider { // 3. Wait for the slots to match const MAX_TOTAL_TIME: Duration = Duration::from_secs(10); + // NOTE: we capture the fetch start slot here and reuse it across retries as otherwise + // we never get a valid result if the RPC node we fetch from is just slightly behind and + // cannot serve us any data with the absolute latest min context slot + let fetch_start_slot = self.chain_slot.load(); let start = std::time::Instant::now(); let mut retries = 0; loop { @@ -622,7 +626,13 @@ impl RemoteAccountProvider { ); } let remote_accounts = self - .try_get_multi(pubkeys, None, fetch_origin, None) + .try_get_multi( + pubkeys, + None, + fetch_origin, + None, + Some(fetch_start_slot), + ) .await?; let slots_match_result = slots_match_and_meet_min_context( &remote_accounts, @@ -632,20 +642,19 @@ impl RemoteAccountProvider { return Ok(remote_accounts); } - if start.elapsed() > MAX_TOTAL_TIME { - return Err(RemoteAccountProviderError::SlotsDidNotMatch( - format!( - "Timeout after {}s waiting for slots to match", - MAX_TOTAL_TIME.as_secs_f64() - ), - vec![], - )); - } - retries += 1; - if retries == config.max_retries { + let hit_max_retry_limit = retries == config.max_retries; + if hit_max_retry_limit || start.elapsed() > MAX_TOTAL_TIME { let remote_accounts = remote_accounts.into_iter().map(|a| a.slot()).collect(); + let limit = if hit_max_retry_limit { + format!("max retries {}", config.max_retries) + } else { + format!( + "max total time of {} seconds", + MAX_TOTAL_TIME.as_secs() + ) + }; match slots_match_result { // SAFETY: Match case is already handled and returns Match => unreachable!("we would have returned above"), @@ -654,15 +663,18 @@ impl RemoteAccountProvider { RemoteAccountProviderError::SlotsDidNotMatch( pubkeys_str(pubkeys), remote_accounts, + limit, ), ); } MatchButBelowMinContextSlot(slot) => { return Err( RemoteAccountProviderError::MatchingSlotsNotSatisfyingMinContextSlot( - pubkeys_str(pubkeys), - remote_accounts, - slot) + pubkeys_str(pubkeys), + remote_accounts, + slot, + limit + ) ); } } @@ -686,6 +698,7 @@ impl RemoteAccountProvider { mark_empty_if_not_found: Option<&[Pubkey]>, fetch_origin: AccountFetchOrigin, program_ids: Option<&[Pubkey]>, + fetch_start_slot: Option, ) -> RemoteAccountProviderResult> { if pubkeys.is_empty() { return Ok(vec![]); @@ -697,7 +710,8 @@ impl RemoteAccountProvider { // Create channels for potential subscription updates to override fetch results let mut subscription_overrides = vec![]; - let fetch_start_slot = self.chain_slot.load(); + let fetch_start_slot = + fetch_start_slot.unwrap_or_else(|| self.chain_slot.load()); { let mut fetching = self.fetching_accounts.lock().unwrap(); @@ -1552,7 +1566,8 @@ mod test { RemoteAccountProviderError::MatchingSlotsNotSatisfyingMinContextSlot( _pubkeys, _slots, - slot + slot, + _ ) if slot == CURRENT_SLOT + 1 )); } diff --git a/test-integration/test-chainlink/tests/ix_remote_account_provider.rs b/test-integration/test-chainlink/tests/ix_remote_account_provider.rs index 4099bd002..2da58425a 100644 --- a/test-integration/test-chainlink/tests/ix_remote_account_provider.rs +++ b/test-integration/test-chainlink/tests/ix_remote_account_provider.rs @@ -162,6 +162,7 @@ async fn ixtest_get_multiple_accounts_for_valid_slot() { None, AccountFetchOrigin::GetAccount, None, + None, ) .await .unwrap(); @@ -194,6 +195,7 @@ async fn ixtest_get_multiple_accounts_for_valid_slot() { None, AccountFetchOrigin::GetAccount, None, + None, ) .await .unwrap(); @@ -228,6 +230,7 @@ async fn ixtest_get_multiple_accounts_for_valid_slot() { None, AccountFetchOrigin::GetAccount, None, + None, ) .await .unwrap();