feat(drive-abci): debugging tool to replay abci requests

.github/actions/docker/action.yaml

@@ -50,6 +50,9 @@ inputs:
   cache_to_name:
     description: "Save cache to name manifest (should be used only on default branch)"
     default: "false"
+  additional_features:
+    description: Extra Cargo features to enable (comma-separated)
+    default: ""
 outputs:
   digest:
     value: ${{ steps.docker_build.outputs.digest }}
@@ -192,6 +195,7 @@ runs:
           AWS=${{ env.HOME }}/.aws/credentials
         build-args: |
           CARGO_BUILD_PROFILE=${{ inputs.cargo_profile }}
+          ADDITIONAL_FEATURES=${{ inputs.additional_features }}
           ${{ steps.sccache.outputs.env_vars }}
         cache-from: ${{ steps.layer_cache_settings.outputs.cache_from }}
         cache-to: ${{ steps.layer_cache_settings.outputs.cache_to }}

.github/workflows/release-docker-image.yml

@@ -25,6 +25,10 @@ on:
         type: string
         description: Cargo profile. i.e. release, dev
         default: release
+      additional_features:
+        type: string
+        description: Extra Cargo features to enable for Drive builds (comma-separated)
+        default: ""
 env:
   DIGEST_NAME: digests-${{ inputs.image_org }}-${{ inputs.image_name }}-${{ inputs.tag }}-${{ inputs.cargo_profile }}-${{ github.sha }}
   DIGEST_DIR_PATH: /tmp/digests
@@ -66,6 +70,7 @@ jobs:
           cache_secret_access_key: ${{ secrets.CACHE_SECRET_KEY }}
           # On release, we generate a new "base" image, so we need to save cache to name manifest, like '.../drive'
           cache_to_name: ${{ github.event_name == 'release' && 'true' || 'false' }}
+          additional_features: ${{ inputs.additional_features }}
 
       - name: Export digest
         run: |

.github/workflows/release.yml

@@ -178,6 +178,7 @@ jobs:
       target: drive-abci
       cargo_profile: dev
       tag: ${{ inputs.tag || github.event.release.tag_name }}-debug
+      additional_features: console,grovedbg,replay
 
   release-rs-dapi-image:
     name: Release RS-DAPI image

Dockerfile

@@ -417,6 +417,7 @@ FROM deps AS build-drive-abci
 # This is only for testing purpose and should be used only for
 # local development environment
 ARG SDK_TEST_DATA
+ARG ADDITIONAL_FEATURES=""
 
 SHELL ["/bin/bash", "-o", "pipefail","-e", "-x", "-c"]
 
@@ -431,10 +432,13 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM
     --mount=type=secret,id=AWS \
     set -ex; \
     source /root/env && \
+    export FEATURES_FLAG=""; \
+    ADDITIONAL_FEATURES_TRIMMED="$(echo "${ADDITIONAL_FEATURES}" | tr -d '[:space:]')"; \
     if  [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \
     mv .cargo/config-release.toml .cargo/config.toml; \
-    else \
-    export FEATURES_FLAG="--features=console,grovedbg"; \
+    fi && \
+    if [[ -n "${ADDITIONAL_FEATURES_TRIMMED}" ]]; then \
+    export FEATURES_FLAG="--features=${ADDITIONAL_FEATURES_TRIMMED}"; \
     fi && \
     if [ "${SDK_TEST_DATA}" == "true" ]; then \
     mv .cargo/config-test-sdk-data.toml .cargo/config.toml; \
@@ -504,13 +508,17 @@ RUN --mount=type=cache,sharing=shared,id=cargo_registry_index,target=${CARGO_HOM
     --mount=type=secret,id=AWS \
     set -ex; \
     source /root/env && \
+    export FEATURES_FLAG=""; \
+    ADDITIONAL_FEATURES_TRIMMED="$(echo "${ADDITIONAL_FEATURES}" | tr -d '[:space:]')"; \
     if  [[ "${CARGO_BUILD_PROFILE}" == "release" ]] ; then \
     mv .cargo/config-release.toml .cargo/config.toml; \
     export OUT_DIRECTORY=release; \
     else \
-    export FEATURES_FLAG="--features=console,grovedbg"; \
     export OUT_DIRECTORY=debug; \
     fi && \
+    if [[ -n "${ADDITIONAL_FEATURES_TRIMMED}" ]]; then \
+    export FEATURES_FLAG="--features=${ADDITIONAL_FEATURES_TRIMMED}"; \
+    fi && \
     if [ "${SDK_TEST_DATA}" == "true" ]; then \
     mv .cargo/config-test-sdk-data.toml .cargo/config.toml; \
     fi && \

packages/rs-drive-abci/.env.testnet

@@ -33,21 +33,21 @@ CORE_CHECK_TX_JSON_RPC_PASSWORD=password
 INITIAL_CORE_CHAINLOCKED_HEIGHT=1243
 
 # https://github.com/dashevo/dashcore-lib/blob/286c33a9d29d33f05d874c47a9b33764a0be0cf1/lib/constants/index.js#L42-L57
-VALIDATOR_SET_QUORUM_TYPE=llmq_25_67
-VALIDATOR_SET_QUORUM_SIZE=25
+VALIDATOR_SET_QUORUM_TYPE=6
+VALIDATOR_SET_QUORUM_SIZE=100
 VALIDATOR_SET_QUORUM_WINDOW=24
 VALIDATOR_SET_QUORUM_ACTIVE_SIGNERS=24
 VALIDATOR_SET_QUORUM_ROTATION=false
 VALIDATOR_SET_ROTATION_BLOCK_COUNT=64
 
-CHAIN_LOCK_QUORUM_TYPE=llmq_50_60
-CHAIN_LOCK_QUORUM_SIZE=50
+CHAIN_LOCK_QUORUM_TYPE=1
+CHAIN_LOCK_QUORUM_SIZE=400
 CHAIN_LOCK_QUORUM_WINDOW=24
 CHAIN_LOCK_QUORUM_ACTIVE_SIGNERS=24
 CHAIN_LOCK_QUORUM_ROTATION=false
 
-INSTANT_LOCK_QUORUM_TYPE=llmq_60_75
-INSTANT_LOCK_QUORUM_SIZE=50
+INSTANT_LOCK_QUORUM_TYPE=5
+INSTANT_LOCK_QUORUM_SIZE=60
 INSTANT_LOCK_QUORUM_WINDOW=288
 INSTANT_LOCK_QUORUM_ACTIVE_SIGNERS=32
 INSTANT_LOCK_QUORUM_ROTATION=true
@@ -77,9 +77,9 @@ MASTERNODE_REWARD_SHARES_SECOND_PUBLIC_KEY=02bf55f97f189895da29824781053140ee66b
 WITHDRAWALS_MASTER_PUBLIC_KEY=027057cdf58628635ef7b75e6b6c90dd996a16929cd68130e16b9328d429e5e03a
 WITHDRAWALS_SECOND_PUBLIC_KEY=022084d827fea4823a69aa7c8d3e02fe780eaa0ef1e5e9841af395ba7e40465ab6
 
-EPOCH_TIME_LENGTH_S=788400
+EPOCH_TIME_LENGTH_S=3600
 
-CHAIN_ID=devnet
+CHAIN_ID=dash-testnet-51
 BLOCK_SPACING_MS=5000
 
 TOKIO_CONSOLE_ENABLED=false

packages/rs-drive-abci/Cargo.toml

@@ -54,6 +54,11 @@ tracing-subscriber = { version = "0.3.16", default-features = false, features =
 tenderdash-abci = { git = "https://github.com/dashpay/rs-tenderdash-abci", tag = "v1.5.0-dev.2", features = [
   "grpc",
 ] }
+time = { version = "0.3", optional = true, features = [
+  "macros",
+  "formatting",
+  "serde-human-readable",
+] }
 
 lazy_static = "1.4.0"
 itertools = { version = "0.13" }
@@ -79,8 +84,6 @@ console-subscriber = { version = "0.4", optional = true }
 bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "0842b17583888e8f46c252a4ee84cdfd58e0546f", optional = true }
 
 [dev-dependencies]
-bs58 = { version = "0.5.0" }
-base64 = "0.22.1"
 platform-version = { path = "../rs-platform-version", features = [
   "mock-versions",
 ] }
@@ -104,7 +107,7 @@ bls-signatures = { git = "https://github.com/dashpay/bls-signatures", rev = "084
 mockall = { version = "0.13" }
 
 # For tests of grovedb verify
-rocksdb = { version = "0.23.0" }
+rocksdb = { version = "0.24.0" }
 integer-encoding = { version = "4.0.0" }
 
 [features]
@@ -113,7 +116,8 @@ mocks = ["mockall", "drive/fixtures-and-mocks", "bls-signatures"]
 console = ["console-subscriber", "tokio/tracing"]
 testing-config = []
 grovedbg = ["drive/grovedbg"]
-
+# `abci-server replay` command
+replay = ["dep:time", "tenderdash-abci/serde"]
 [[bin]]
 name = "drive-abci"
 path = "src/main.rs"

packages/rs-drive-abci/src/abci/handler/process_proposal.rs

@@ -33,12 +33,11 @@ where
     if let Some(block_execution_context) = block_execution_context_guard.as_mut() {
         // We are already in a block, or in init chain.
         // This only makes sense if we were the proposer unless we are at a future round
-        if block_execution_context.block_state_info().round() != (request.round as u32) {
+        let block_state_info = block_execution_context.block_state_info();
+        if block_state_info.round() != (request.round as u32) {
             // We were not the proposer, and we should process something new
             drop_block_execution_context = true;
-        } else if let Some(current_block_hash) =
-            block_execution_context.block_state_info().block_hash()
-        {
+        } else if let Some(current_block_hash) = block_state_info.block_hash() {
             // There is also the possibility that this block already came in, but tenderdash crashed
             // Now tenderdash is sending it again
             if let Some(proposal_info) = block_execution_context.proposer_results() {
@@ -69,6 +68,15 @@ where
             } else {
                 // We are getting a different block hash for a block of the same round
                 // This is a terrible issue
+                tracing::error!(
+                    method = "process_proposal",
+                    block_state_info = ?block_state_info,
+                    "received a process proposal request twice with different hash for height {}/round {}: existing hash {:?}, new hash {:?}",
+                    request.height,
+                    request.round,
+                    current_block_hash,
+                    request.hash,
+                );
                 Err(Error::Abci(AbciError::BadRequest(
                     "received a process proposal request twice with different hash".to_string(),
                 )))?;

packages/rs-drive-abci/src/execution/engine/run_block_proposal/v0/mod.rs

@@ -399,6 +399,7 @@ where
             tracing::trace!(
                 method = "run_block_proposal_v0",
                 app_hash = hex::encode(root_hash),
+                block_hash = hex::encode(block_proposal.block_hash.unwrap_or_default()),
                 platform_state_fingerprint = hex::encode(
                     block_execution_context
                         .block_platform_state()

packages/rs-drive-abci/src/execution/storage/fetch_platform_state/mod.rs

@@ -1,3 +1,5 @@
+//! Fetches execution state from grovedb storage
+
 use crate::error::execution::ExecutionError;
 use crate::error::Error;
 use crate::platform_types::platform::Platform;

packages/rs-drive-abci/src/execution/storage/mod.rs

@@ -1,2 +1,2 @@
-mod fetch_platform_state;
+pub mod fetch_platform_state;
 mod store_platform_state;

packages/rs-drive-abci/src/lib.rs

@@ -50,5 +50,10 @@ pub mod query;
 /// Various utils
 pub mod utils;
 
+/// Replay captured ABCI requests against drive-abci
+#[cfg(feature = "replay")]
+pub mod replay;
 /// Drive server
 pub mod server;
+/// Verification helpers
+pub mod verify;

packages/rs-drive-abci/src/main.rs

@@ -1,13 +1,15 @@
 //! Main server process for RS-Drive-ABCI
 //!
 //! RS-Drive-ABCI server starts a single-threaded server and listens to connections from Tenderdash.
+#[cfg(feature = "replay")]
+use drive_abci::replay::{self, ReplayArgs};
+use drive_abci::verify::verify_grovedb;
 
 use clap::{Parser, Subcommand};
 use dapi_grpc::platform::v0::get_status_request;
 use dapi_grpc::platform::v0::get_status_request::GetStatusRequestV0;
 use dapi_grpc::platform::v0::platform_client::PlatformClient;
 use dapi_grpc::tonic::transport::Uri;
-use dpp::version::PlatformVersion;
 use drive_abci::config::{FromEnv, PlatformConfig};
 use drive_abci::core::wait_for_core_to_sync::v0::wait_for_core_to_sync_v0;
 use drive_abci::logging::{LogBuilder, LogConfig, LogDestination, Loggers};
@@ -16,7 +18,6 @@ use drive_abci::platform_types::platform::Platform;
 use drive_abci::rpc::core::DefaultCoreRPC;
 use drive_abci::{logging, server};
 use itertools::Itertools;
-use std::fs::remove_file;
 #[cfg(all(tokio_unstable, feature = "console"))]
 use std::net::SocketAddr;
 use std::path::PathBuf;
@@ -63,6 +64,11 @@ enum Commands {
     /// Print current software version
     #[command()]
     Version,
+
+    /// Replay ABCI requests captured from drive-abci logs.
+    #[cfg(feature = "replay")]
+    #[command()]
+    Replay(ReplayArgs),
 }
 
 /// Server that accepts connections from Tenderdash, and
@@ -151,8 +157,13 @@ impl Cli {
             }
             Commands::Config => dump_config(&config)?,
             Commands::Status => runtime.block_on(check_status(&config))?,
-            Commands::Verify => verify_grovedb(&config.db_path, true)?,
+            Commands::Verify => drive_abci::verify::run(&config, true)?,
             Commands::Version => print_version(),
+            #[cfg(feature = "replay")]
+            Commands::Replay(args) => {
+                replay::run(config, args, cancel.clone()).map_err(|e| e.to_string())?;
+                return Ok(());
+            }
         };
 
         Ok(())
@@ -331,62 +342,6 @@ async fn check_status(config: &PlatformConfig) -> Result<(), String> {
         .map_err(|e| format!("can't request status: {e}"))
 }
 
-/// Verify GroveDB integrity.
-///
-/// This function will execute GroveDB integrity checks if one of the following conditions is met:
-/// - `force` is `true`
-/// - file `.fsck` in `config.db_path` exists
-///
-/// After successful verification, .fsck file is removed.
-fn verify_grovedb(db_path: &PathBuf, force: bool) -> Result<(), String> {
-    let fsck = PathBuf::from(db_path).join(".fsck");
-
-    if !force {
-        if !fsck.exists() {
-            return Ok(());
-        }
-        tracing::info!(
-            "found {} file, starting grovedb verification",
-            fsck.display()
-        );
-    }
-
-    let grovedb = drive::grovedb::GroveDb::open(db_path).expect("open grovedb");
-    //todo: get platform version instead of taking latest
-    let result = grovedb
-        .visualize_verify_grovedb(
-            None,
-            true,
-            true,
-            &PlatformVersion::latest().drive.grove_version,
-        )
-        .map_err(|e| e.to_string());
-
-    match result {
-        Ok(data) => {
-            for result in data {
-                tracing::warn!(?result, "grovedb verification")
-            }
-            tracing::info!("grovedb verification finished");
-
-            if fsck.exists() {
-                if let Err(e) = remove_file(&fsck) {
-                    tracing::warn!(
-                        error = ?e,
-                        path  =fsck.display().to_string(),
-                        "grovedb verification: cannot remove .fsck file: please remove it manually to avoid running verification again",
-                    );
-                }
-            }
-            Ok(())
-        }
-        Err(e) => {
-            tracing::error!("grovedb verification failed: {}", e);
-            Err(e)
-        }
-    }
-}
-
 /// Print current software version.
 fn print_version() {
     println!("{}", env!("CARGO_PKG_VERSION"));