POC: Implement Merkle tree synchronization protocols #1759

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open

xilosada wants to merge 99 commits into master from test/tree_sync

.cursor/worktrees.json

-Original file line number
+Diff line change
@@ -0,0 +1,5 @@
+    {
+      "setup-worktree": [
+        "npm install"
+      ]
+    }

Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Cargo.toml

-Original file line number
+Diff line change
@@ Expand Up / @@ -87,6 +87,7 @@ members = [ @@
         "./apps/abi_conformance",
         "./apps/state-schema-conformance",
         "./apps/xcall-example",
+        "./apps/sync-test",
         "./tools/calimero-abi",
         "./tools/merodb",
@@ Expand Down Expand Up / @@ -143,6 +144,7 @@ jsonschema = "0.17" @@
     jsonwebtoken = "9.3.0"
     lazy_static = "1.4"
     libp2p = "0.56.0"
+    lz4_flex = "0.11"
     libp2p-identity = "0.2.12"
     libp2p-metrics = "0.17.0"
     libp2p-stream = "0.4.0-alpha"
@@ Expand Down @@

apps/abi_conformance/src/lib.rs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -3,6 +3,7 @@ use std::collections::BTreeMap;
  
    use calimero_sdk::app;

    use calimero_sdk::borsh::{BorshDeserialize, BorshSerialize};

    use calimero_sdk::serde::{Deserialize, Serialize};

    use calimero_storage::collections::{LwwRegister, UnorderedMap, Vector};

    use thiserror::Error;

    // Test multi-file ABI generation

    @@ -106,22 +107,22 @@ pub enum Event {
  
    // State

    #[app::state(emits = Event)]

    #[derive(Debug, PartialEq, Eq, PartialOrd, BorshSerialize, BorshDeserialize)]

    #[derive(Debug, BorshSerialize, BorshDeserialize)]

    #[borsh(crate = "calimero_sdk::borsh")]

    pub struct AbiState {

        counters: BTreeMap<String, u32>, // map<string,u32>

        users: Vec<UserId32>,            // list<UserId32>

        counters: UnorderedMap<String, LwwRegister<u32>>, // map<string,u32> - CRDT with LWW values

        users: Vector<LwwRegister<UserId32>>,             // list<UserId32> - CRDT with LWW values

    }

    // Implementation

    #[app::logic]

    impl AbiState {

        #[app::init]

        #[must_use]

        pub const fn init() -> Self {

        pub fn init() -> Self {

            Self {

                counters: BTreeMap::new(),

                users: Vec::new(),

                counters: UnorderedMap::new(),

                users: Vector::new(),

            }

        }

apps/blobs/src/lib.rs

            
                      Original file line number
                      Diff line number
                      Diff line change
                  
    @@ -9,7 +9,7 @@
  
    use calimero_sdk::borsh::{BorshDeserialize, BorshSerialize};

    use calimero_sdk::serde::Serialize;

    use calimero_sdk::{app, env};

    use calimero_storage::collections::UnorderedMap;

    use calimero_storage::collections::{Counter, LwwRegister, UnorderedMap};

    // === CONSTANTS ===

    @@ -146,15 +146,15 @@ impl calimero_storage::collections::Mergeable for FileRecord {
  
    pub struct FileShareState {

        /// Context owner's identity as base58-encoded public key

        /// Set during initialization from `env::executor_id()`

        pub owner: String,

        pub owner: LwwRegister<String>,

        /// Map of file ID to file metadata records

        /// Key: file ID (e.g., "file_0"), Value: FileRecord

        pub files: UnorderedMap<String, FileRecord>,

        /// Counter for generating unique file IDs

        /// Incremented on each file upload

        pub file_counter: u64,

        /// Incremented on each file upload (CRDT G-Counter for distributed safety)

        pub file_counter: Counter,

    }

    /// Events emitted by the application

    @@ -195,9 +195,9 @@ impl FileShareState {
  
            app::log!("Initializing file sharing app for owner: {}", owner);

            FileShareState {

                owner,

                owner: LwwRegister::new(owner),

                files: UnorderedMap::new(),

                file_counter: 0,

                file_counter: Counter::new(),

            }

        }

    @@ -225,8 +225,15 @@ impl FileShareState {
  
        ) -> Result<String, String> {

            let blob_id = parse_blob_id_base58(&blob_id_str)?;

            let file_id = format!("file_{}", self.file_counter);

            self.file_counter += 1;

            // Get current counter value for file ID, then increment

            let counter_value = self

                .file_counter

                .value()

                .map_err(|e| format!("Failed to get counter: {e:?}"))?;

            let file_id = format!("file_{}", counter_value);

            self.file_counter

                .increment()

                .map_err(|e| format!("Failed to increment counter: {e:?}"))?;

            let uploader_id = env::executor_id();

            let uploader = encode_blob_id_base58(&uploader_id);

    @@ -436,7 +443,10 @@ impl FileShareState {
  
                 - Total files: {}\n\

                 - Total storage: {:.2} MB ({} bytes)\n\

                 - Owner: {}",

                file_count, total_mb, total_size, self.owner

                file_count,

                total_mb,

                total_size,

                self.owner.get()

            ))

        }

    }

apps/kv-store/src/lib.rs

-Original file line number
+Diff line change
@@ Expand Up / @@ -37,7 +37,8 @@ impl KvStore { @@
         #[app::init]
         pub fn init() -> KvStore {
             KvStore {
-                items: UnorderedMap::new(),
+                // Use deterministic ID based on field name for sync compatibility
+                items: UnorderedMap::new_with_field_name("items"),
             }
         }
@@ Expand Down @@

apps/sync-test/Cargo.toml

-Original file line number
+Diff line change
@@ -0,0 +1,24 @@
+    [package]
+    name = "sync-test"
+    version = "0.1.0"
+    edition = "2021"
+    publish = false
+    [lib]
+    crate-type = ["cdylib"]
+    [dependencies]
+    calimero-sdk = { path = "../../crates/sdk" }
+    calimero-storage = { path = "../../crates/storage" }
+    calimero-storage-macros = { path = "../../crates/storage-macros" }
+    hex = "0.4"
+    thiserror = "2.0"
+    [profile.app-release]
+    inherits = "release"
+    codegen-units = 1
+    opt-level = "z"
+    lto = true
+    debug = false
+    panic = "abort"
+    overflow-checks = true

apps/sync-test/README.md

-Original file line number
+Diff line change
@@ -0,0 +1,96 @@
+    # Sync Test Application
+    Comprehensive test application for validating Calimero's synchronization protocol.
+    ## Purpose
+    This application exercises ALL storage spaces and CRDT types to ensure proper synchronization:
+    - **Public Storage**: Shared state across all nodes
+    - **User Storage**: Per-user isolated state
+    - **Frozen Storage**: Content-addressed immutable data
+    ## CRDT Types Tested
+    | Type | Description | Merge Semantics |
+    |------|-------------|-----------------|
+    | `LwwRegister<T>` | Last-Write-Wins register | Latest timestamp wins |
+    | `Counter` | PN-Counter | Positive/negative increments merge |
+    | `UnorderedMap<K, V>` | Key-value map | Merge by key, delegate to value CRDT |
+    | `UserStorage<T>` | Per-user data | Isolated by user public key |
+    | `FrozenStorage<T>` | Immutable blobs | Content-addressed (no merge needed) |
+    ## Operations
+    ### Public Key-Value
+    - `set(key, value)` - Set a key-value pair
+    - `get(key)` - Get a value
+    - `delete(key)` - Delete a key (creates tombstone)
+    - `batch_set(pairs)` - Batch set multiple pairs
+    - `entries()` - Get all entries
+    - `len()` - Get count of entries
+    ### Public Counters
+    - `counter_inc(name)` - Increment a named counter
+    - `counter_dec(name)` - Decrement a named counter
+    - `counter_get(name)` - Get counter value
+    ### Public Stats (Nested CRDT)
+    - `stats_inc(entity)` - Record increment
+    - `stats_dec(entity)` - Record decrement
+    - `stats_get(entity)` - Get (increments, decrements)
+    ### User Storage
+    - `user_set_simple(value)` - Set current user's value
+    - `user_get_simple()` - Get current user's value
+    - `user_set_kv(key, value)` - Set in user's private store
+    - `user_get_kv(key)` - Get from user's private store
+    - `user_delete_kv(key)` - Delete from user's private store
+    - `user_counter_inc()` - Increment user's counter
+    - `user_counter_get()` - Get user's counter
+    - `user_get_simple_for(user_key)` - Read another user's value
+    ### Frozen Storage
+    - `frozen_add(data)` - Add immutable data, returns hash
+    - `frozen_get(hash_hex)` - Get by hash
+    ### Verification
+    - `snapshot()` - Get deterministic state snapshot
+    - `verify(expected)` - Verify state matches expected
+    - `get_operation_count()` - Total operations performed
+    - `get_deleted_count()` - Count of deleted keys
+    - `was_deleted(key)` - Check if key was deleted
+    ### Bulk Operations (Benchmarking)
+    - `bulk_write(prefix, count, value_size)` - Write N keys
+    - `bulk_delete(prefix, count)` - Delete N keys
+    - `bulk_counter_inc(name, count)` - Increment N times
+    ## Building
+    ```bash
+    ./build.sh
+    ```
+    Output: `res/sync_test.wasm`
+    ## Testing with merobox
+    See `workflows/` for example test workflows.
+    ## Deterministic Verification
+    The `snapshot()` method returns a deterministic representation of state that can be compared across nodes:
+    ```json
+    {
+      "public_kv_count": 10,
+      "public_kv_entries": {"key1": "value1", ...},
+      "public_counter_values": {"counter1": 5, ...},
+      "deleted_keys_count": 2,
+      "frozen_count": 1,
+      "operation_count": 15
+    }
+    ```
+    After sync convergence, all nodes should return identical snapshots.

apps/sync-test/build.sh

-Original file line number
+Diff line change
@@ -0,0 +1,20 @@
+    #!/bin/bash
+    set -e
+    cd "$(dirname $0)"
+    TARGET="${CARGO_TARGET_DIR:-../../target}"
+    cargo build --target wasm32-unknown-unknown --profile app-release
+    mkdir -p res
+    cp "$TARGET/wasm32-unknown-unknown/app-release/sync_test.wasm" res/sync_test.wasm
+    # Skip wasm-opt for now - it requires --enable-bulk-memory-opt
+    # if command -v wasm-opt >/dev/null 2>&1; then
+    #     wasm-opt -Oz --enable-bulk-memory res/sync_test.wasm -o res/sync_test.wasm
+    # fi
+    ls -la res/sync_test.wasm

apps/sync-test/res/sync_test.wasm

Binary file not shown.

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

POC: Implement Merkle tree synchronization protocols #1759

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!

POC: Implement Merkle tree synchronization protocols #1759

Are you sure you want to change the base?

Uh oh!

POC: Implement Merkle tree synchronization protocols #1759

Uh oh!

Uh oh!

Diff view

Diff view

There are no files selected for viewing

Uh oh!

Uh oh!