diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..e59cd50 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,29 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '[BUG] ' +labels: bug +assignees: '' +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Run command '...' +2. With parameters '...' +3. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Environment:** + - OS: [e.g. Ubuntu 22.04] + - Rust version: [e.g. 1.75.0] + - Python version: [e.g. 3.10] + - Zebrad version: [e.g. 1.5.0] + - Lightwalletd version: [e.g. 0.4.14] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..b0a0adb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,19 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '[FEATURE] ' +labels: enhancement +assignees: '' +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml new file mode 100644 index 0000000..3082561 --- /dev/null +++ b/.github/workflows/python-ci.yml @@ -0,0 +1,36 @@ +name: Python CI + +on: + push: + branches: [ main, develop ] + pull_request: + branches: [ main ] + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install dependencies + working-directory: ./visualization + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install flake8 black + + - name: Lint with flake8 + working-directory: ./visualization + run: | + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --filename visualize.py + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --filename visualize.py + + - name: Check formatting with black + working-directory: ./visualization + run: black --check . diff --git a/.github/workflows/rust-ci.yml b/.github/workflows/rust-ci.yml new file mode 100644 index 0000000..088e93f --- /dev/null +++ b/.github/workflows/rust-ci.yml @@ -0,0 +1,26 @@ +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Install protobuf compiler + run: sudo apt-get update && sudo apt-get install -y protobuf-compiler + + - name: Fetch proto files + run: ./scripts/fetch_protos.sh + + - name: Build + working-directory: ./analyzer + run: cargo build --release --verbose + + - name: Run tests + working-directory: ./analyzer + run: cargo test --verbose diff --git a/.gitignore b/.gitignore index 99daa86..5b493b3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,40 @@ -/target -results.csv +# Rust +/analyzer/target/ +/analyzer/Cargo.lock +**/*.rs.bk +target/ +# Proto files +/analyzer/proto/*.proto + +# Python +__pycache__/ +*.py[cod] +*$py.class +.Python +venv/ +env/ +*.egg-info/ + +# Results +/results/* +!/results/.gitkeep +!/results/README.md + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Build artifacts +*.so +*.dylib +*.dll + +# Logs +*.log diff --git a/Cargo.lock b/Cargo.lock deleted file mode 100644 index 23877a9..0000000 --- a/Cargo.lock +++ /dev/null @@ -1,1998 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "anyhow" -version = "1.0.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "axum" -version = "0.6.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" -dependencies = [ - "async-trait", - "axum-core", - "bitflags 1.3.2", - "bytes", - "futures-util", - "http", - "http-body", - "hyper", - "itoa", - "matchit", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower", - "tower-layer", - "tower-service", -] - -[[package]] -name = "axum-core" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http", - "http-body", - "mime", - "rustversion", - "tower-layer", - "tower-service", -] - -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", - "windows-link", -] - -[[package]] -name = "base64" -version = "0.21.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394" - -[[package]] -name = "bumpalo" -version = "3.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - -[[package]] -name = "bytes" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" - -[[package]] -name = "cc" -version = "1.2.40" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d05d92f4b1fd76aad469d46cdd858ca761576082cd37df81416691e50199fb" -dependencies = [ - "find-msvc-tools", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" - -[[package]] -name = "compact_block_analyzer" -version = "0.1.0" -dependencies = [ - "anyhow", - "csv", - "hex", - "prost", - "prost-types", - "reqwest", - "serde", - "serde_json", - "tokio", - "tonic", - "tonic-build", -] - -[[package]] -name = "core-foundation" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "csv" -version = "1.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acdc4883a9c96732e4733212c01447ebd805833b7275a73ca3ee080fd77afdaf" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" -dependencies = [ - "memchr", -] - -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "either" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" - -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "fastrand" -version = "2.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" - -[[package]] -name = "find-msvc-tools" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0399f9d26e5191ce32c498bebd31e7a3ceabc2745f0ac54af3f335126c3f24b3" - -[[package]] -name = "fixedbitset" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared", -] - -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "futures-channel" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" -dependencies = [ - "futures-core", -] - -[[package]] -name = "futures-core" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" - -[[package]] -name = "futures-sink" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" - -[[package]] -name = "futures-task" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" - -[[package]] -name = "futures-util" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" -dependencies = [ - "futures-core", - "futures-task", - "pin-project-lite", - "pin-utils", -] - -[[package]] -name = "getrandom" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" -dependencies = [ - "cfg-if", - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", -] - -[[package]] -name = "getrandom" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" -dependencies = [ - "cfg-if", - "libc", - "r-efi", - "wasi 0.14.7+wasi-0.2.4", -] - -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - -[[package]] -name = "h2" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 2.11.4", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - -[[package]] -name = "hashbrown" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyper-timeout" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" -dependencies = [ - "hyper", - "pin-project-lite", - "tokio", - "tokio-io-timeout", -] - -[[package]] -name = "hyper-tls" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" -dependencies = [ - "bytes", - "hyper", - "native-tls", - "tokio", - "tokio-native-tls", -] - -[[package]] -name = "icu_collections" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" -dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" - -[[package]] -name = "icu_properties" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" -dependencies = [ - "displaydoc", - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "potential_utf", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" - -[[package]] -name = "icu_provider" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" -dependencies = [ - "displaydoc", - "icu_locale_core", - "stable_deref_trait", - "tinystr", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - -[[package]] -name = "indexmap" -version = "2.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" -dependencies = [ - "equivalent", - "hashbrown 0.16.0", -] - -[[package]] -name = "io-uring" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "libc", -] - -[[package]] -name = "ipnet" -version = "2.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" - -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" - -[[package]] -name = "js-sys" -version = "0.3.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305" -dependencies = [ - "once_cell", - "wasm-bindgen", -] - -[[package]] -name = "libc" -version = "0.2.176" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174" - -[[package]] -name = "linux-raw-sys" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" - -[[package]] -name = "litemap" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" - -[[package]] -name = "matchit" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" - -[[package]] -name = "memchr" -version = "2.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" - -[[package]] -name = "mime" -version = "0.3.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" - -[[package]] -name = "miniz_oxide" -version = "0.8.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" -dependencies = [ - "adler2", -] - -[[package]] -name = "mio" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" -dependencies = [ - "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", -] - -[[package]] -name = "multimap" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" - -[[package]] -name = "native-tls" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87de3442987e9dbec73158d5c715e7ad9072fda936bb03d19d7fa10e00520f0e" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - -[[package]] -name = "once_cell" -version = "1.21.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - -[[package]] -name = "openssl" -version = "0.10.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" -dependencies = [ - "bitflags 2.9.4", - "cfg-if", - "foreign-types", - "libc", - "once_cell", - "openssl-macros", - "openssl-sys", -] - -[[package]] -name = "openssl-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "openssl-probe" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" - -[[package]] -name = "openssl-sys" -version = "0.9.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "petgraph" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" -dependencies = [ - "fixedbitset", - "indexmap 2.11.4", -] - -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" - -[[package]] -name = "potential_utf" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84df19adbe5b5a0782edcab45899906947ab039ccf4573713735ee7de1e6b08a" -dependencies = [ - "zerovec", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "prettyplease" -version = "0.2.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" -dependencies = [ - "proc-macro2", - "syn", -] - -[[package]] -name = "proc-macro2" -version = "1.0.101" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "prost" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" -dependencies = [ - "bytes", - "prost-derive", -] - -[[package]] -name = "prost-build" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" -dependencies = [ - "bytes", - "heck", - "itertools", - "log", - "multimap", - "once_cell", - "petgraph", - "prettyplease", - "prost", - "prost-types", - "regex", - "syn", - "tempfile", -] - -[[package]] -name = "prost-derive" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" -dependencies = [ - "anyhow", - "itertools", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "prost-types" -version = "0.12.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" -dependencies = [ - "prost", -] - -[[package]] -name = "quote" -version = "1.0.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.16", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags 2.9.4", -] - -[[package]] -name = "regex" -version = "1.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" - -[[package]] -name = "reqwest" -version = "0.11.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" -dependencies = [ - "base64", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", - "ipnet", - "js-sys", - "log", - "mime", - "native-tls", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls-pemfile", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "system-configuration", - "tokio", - "tokio-native-tls", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", - "winreg", -] - -[[package]] -name = "rustc-demangle" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" - -[[package]] -name = "rustix" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" -dependencies = [ - "bitflags 2.9.4", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls-pemfile" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" -dependencies = [ - "base64", -] - -[[package]] -name = "rustversion" -version = "1.0.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" - -[[package]] -name = "ryu" -version = "1.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" - -[[package]] -name = "schannel" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" -dependencies = [ - "bitflags 2.9.4", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "serde" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.228" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.145" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", - "serde_core", -] - -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - -[[package]] -name = "signal-hook-registry" -version = "1.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" -dependencies = [ - "libc", -] - -[[package]] -name = "slab" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" - -[[package]] -name = "smallvec" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - -[[package]] -name = "socket2" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" -dependencies = [ - "libc", - "windows-sys 0.59.0", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "syn" -version = "2.0.106" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "system-configuration" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" -dependencies = [ - "bitflags 1.3.2", - "core-foundation", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "tempfile" -version = "3.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" -dependencies = [ - "fastrand", - "getrandom 0.3.3", - "once_cell", - "rustix", - "windows-sys 0.61.2", -] - -[[package]] -name = "tinystr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tokio" -version = "1.47.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" -dependencies = [ - "backtrace", - "bytes", - "io-uring", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "slab", - "socket2 0.6.0", - "tokio-macros", - "windows-sys 0.59.0", -] - -[[package]] -name = "tokio-io-timeout" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" -dependencies = [ - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-macros" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tonic" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" -dependencies = [ - "async-stream", - "async-trait", - "axum", - "base64", - "bytes", - "h2", - "http", - "http-body", - "hyper", - "hyper-timeout", - "percent-encoding", - "pin-project", - "prost", - "tokio", - "tokio-stream", - "tower", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tonic-build" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" -dependencies = [ - "prettyplease", - "proc-macro2", - "prost-build", - "quote", - "syn", -] - -[[package]] -name = "tower" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" -dependencies = [ - "futures-core", - "futures-util", - "indexmap 1.9.3", - "pin-project", - "pin-project-lite", - "rand", - "slab", - "tokio", - "tokio-util", - "tower-layer", - "tower-service", - "tracing", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.41" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "tracing-core" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "unicode-ident" -version = "1.0.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d" - -[[package]] -name = "url" -version = "2.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasi" -version = "0.14.7+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c" -dependencies = [ - "wasip2", -] - -[[package]] -name = "wasip2" -version = "1.0.1+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.54" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" -dependencies = [ - "cfg-if", - "js-sys", - "once_cell", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "web-sys" -version = "0.3.81" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" -dependencies = [ - "windows-targets 0.48.5", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.59.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" -dependencies = [ - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" -dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm 0.52.6", - "windows_aarch64_msvc 0.52.6", - "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", - "windows_i686_msvc 0.52.6", - "windows_x86_64_gnu 0.52.6", - "windows_x86_64_gnullvm 0.52.6", - "windows_x86_64_msvc 0.52.6", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - -[[package]] -name = "wit-bindgen" -version = "0.46.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" - -[[package]] -name = "writeable" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" - -[[package]] -name = "yoke" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "zerofrom" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerotrie" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7aa2bd55086f1ab526693ecbe444205da57e25f4489879da80635a46d90e73b" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index 01f0abb..0000000 --- a/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "compact_block_analyzer" -version = "0.1.0" -edition = "2021" - -[dependencies] -tokio = { version = "1", features = ["full"] } -tonic = "0.11" -prost = "0.12" -prost-types = "0.12" -reqwest = { version = "0.11", features = ["json"] } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -hex = "0.4" -csv = "1.3" -anyhow = "1.0" - -[build-dependencies] -tonic-build = "0.11" diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2d59335 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Zcash Community + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/NETWORK_UPGRADES.md b/NETWORK_UPGRADES.md new file mode 100644 index 0000000..13e3eb6 --- /dev/null +++ b/NETWORK_UPGRADES.md @@ -0,0 +1,160 @@ +# Zcash Network Upgrades Reference + +This document provides context for the protocol eras used in sampling and analysis. + +## Network Upgrade Timeline + +| Era | Activation Block | Date | Key Features | +|-----|-----------------|------|--------------| +| **Sapling** | 419,200 - 653,599 | Oct 2018 - Dec 2019 | New shielded pool, improved performance | +| **Blossom** | 653,600 - 903,799 | Dec 2019 - Jul 2020 | Block time 150s → 75s | +| **Heartwood** | 903,800 - 1,046,399 | Jul 2020 - Nov 2020 | Shielded coinbase, FlyClient | +| **Canopy** | 1,046,400 - 1,687,103 | Nov 2020 - May 2022 | Dev fund activation | +| **NU5** | 1,687,104 - 2,726,399 | May 2022 - Apr 2024 | Orchard pool, unified addresses | +| **NU6** | 2,726,400 - current | Apr 2024 - present | Current protocol | + +## Why These Eras Matter for Analysis + +### Sapling (419,200 - 653,599) +**Characteristics:** +- New efficient shielded pool introduced +- Gradual adoption of shielded transactions +- Transparent still common but declining +- Block time: 150 seconds + +**Relevance to Analysis:** +- Transition period from transparent to shielded +- Mixed usage patterns +- Baseline for early shielded adoption + +### Blossom (653,600 - 903,799) +**Characteristics:** +- **Block time reduced to 75 seconds** +- Twice as many blocks per day (from ~576 to ~1152) +- Continued Sapling adoption +- More blocks = more overhead for initial sync + +**Relevance to Analysis:** +- **Critical**: Block time change affects bandwidth calculations +- Must account for 2x blocks per day after Blossom +- Same MB/block but different MB/day after this upgrade + +### Heartwood (903,800 - 1,046,399) +**Characteristics:** +- Shielded coinbase enabled (miners can receive directly to Sapling) +- FlyClient support for light clients +- ZIP 213 (shielded coinbase) + +**Relevance to Analysis:** +- Shielded coinbase reduces transparent coinbase outputs +- May show decrease in transparent transactions +- Short era but important transition + +### Canopy (1,046,400 - 1,687,103) +**Characteristics:** +- Dev fund activation (20% of block rewards) +- Mature Sapling usage +- Longest single-upgrade era +- Stable protocol period + +**Relevance to Analysis:** +- Good baseline for "normal" modern usage +- Large sample available +- Represents stable state before Orchard + +### NU5 (1,687,104 - 2,726,399) +**Characteristics:** +- **Orchard pool introduced** (new shielded pool) +- **Unified addresses** (single address for all pools) +- Halo 2 proving system +- Transaction version 5 + +**Relevance to Analysis:** +- Introduction of Orchard changes transaction mix +- Unified addresses may affect usage patterns +- Large era, represents recent "stable" state + +### NU6 (2,726,400 - current) +**Characteristics:** +- Current protocol version +- Most recent upgrade (April 2024) +- Active development and adoption + +**Relevance to Analysis:** +- **Most relevant for current decisions** +- Represents actual current usage +- Should be heavily sampled +- Ongoing changes as ecosystem adapts + +## Impact on Compact Block Analysis + +### Transparent Usage Trends + +``` +Sapling: ████████████░░░░░░░░ (High → Medium) +Blossom: ████████░░░░░░░░░░░░ (Medium) +Heartwood: ██████░░░░░░░░░░░░░░ (Medium → Low) +Canopy: ████░░░░░░░░░░░░░░░░ (Low) +NU5: ███░░░░░░░░░░░░░░░░░ (Low) +NU6: ███░░░░░░░░░░░░░░░░░ (Low, current) +``` + +### Block Frequency (Important!) + +**Before Blossom:** +- Block time: 150 seconds +- Blocks per day: ~576 +- Daily bandwidth = avg_block_size × 576 + +**After Blossom:** +- Block time: 75 seconds +- Blocks per day: ~1,152 +- Daily bandwidth = avg_block_size × 1,152 + +⚠️ **When calculating daily bandwidth, account for this doubling after block 653,600!** + +### Sampling Recommendations + +**For Current State Analysis:** +- Focus on NU6 (current) and NU5 (recent stable) +- 60-70% of samples from these eras + +**For Historical Context:** +- Sapling through Canopy shows evolution of usage patterns +- Important to understand how transparent usage declined + +**For Bandwidth Calculations:** +- Separate pre-Blossom and post-Blossom blocks +- Use correct blocks/day multiplier +- Daily sync = avg_size × (576 or 1,152) depending on era + +## Querying Specific Eras + +```bash +# Sapling (transition period) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 500000 501000 sapling.csv + +# Blossom (block time change) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 750000 751000 blossom.csv + +# Heartwood (shielded coinbase) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 950000 951000 heartwood.csv + +# Canopy (mature Sapling) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 1300000 1301000 canopy.csv + +# NU5 (Orchard introduction) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 2000000 2001000 nu5.csv + +# NU6 (current) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 2800000 2801000 nu6.csv +``` + +## References + +- [Zcash Network Upgrade Timeline](https://z.cash/upgrade/) +- [ZIP 200: Network Upgrade Mechanism](https://zips.z.cash/zip-0200) +- [ZIP 206: Deployment of the Blossom Network Upgrade](https://zips.z.cash/zip-0206) +- [ZIP 221: FlyClient - Consensus-Layer Changes](https://zips.z.cash/zip-0221) +- [ZIP 224: Orchard Shielded Protocol](https://zips.z.cash/zip-0224) +- [Zcash Protocol Specification](https://zips.z.cash/protocol/protocol.pdf) diff --git a/PROJECT_SUMMARY.md b/PROJECT_SUMMARY.md new file mode 100644 index 0000000..9cf3a95 --- /dev/null +++ b/PROJECT_SUMMARY.md @@ -0,0 +1,73 @@ +# Project Summary: Compact Block Analyzer + +## What We Built + +A complete statistical analysis tool for evaluating the bandwidth impact of adding transparent transaction data to Zcash's compact block protocol. + +## Components + +### 1. Rust Analyzer (`src/main.rs`) +**Purpose**: Data collection and size estimation + +**Features**: +- Multiple sampling strategies (6 different modes) +- Fetches real compact blocks from lightwalletd via gRPC +- Fetches full blocks from Zebrad via JSON-RPC +- Estimates protobuf overhead for new transparent fields +- Era tracking (pre-Sapling, Sapling, Canopy, NU5) +- Configurable via command-line +- Reproducible results (fixed random seed) + +**Sampling Strategies**: +- `quick`: Fast overview (~1500 blocks, 15min) +- `recommended`: Balanced hybrid approach (~5000 blocks, 30min) +- `thorough`: Comprehensive analysis (~11000 blocks, 2hr) +- `equal`: Equal samples per era +- `proportional`: Proportional to blockchain distribution +- `weighted`: Custom weights per era +- `range`: Analyze specific block range + +### 2. Python Visualizer (`visualize.py`) +**Purpose**: Statistical analysis and visualization + +**Generates 7 Types of Charts**: +1. **Distribution**: Histogram + KDE + box plot +2. **Time Series**: Overhead over blockchain height with era markers +3. **Era Comparison**: Box plots, violin plots, bar charts by era +4. **Correlations**: Overhead vs inputs/outputs/tx count +5. **Cumulative Distribution**: CDF with percentile markers +6. **Bandwidth Impact**: Daily sync, full sync, costs, time projections +7. **Heatmaps**: Overhead by era and transaction characteristics + +**Statistical Report Includes**: +- Summary statistics (mean, median, std dev, percentiles) +- Confidence intervals (95%) +- Statistics broken down by era +- Bandwidth impact calculations +- Correlation analysis +- Decision framework recommendations + +### 3. Documentation + +**README.md**: Complete user guide +- Installation instructions +- Usage examples +- Sampling strategy explanations +- Troubleshooting +- Decision criteria + +**QUICKSTART.md**: 5-minute getting started guide +- Step-by-step setup +- First analysis in minutes +- Common issues and solutions + +**AI_DISCLAIMER.md**: Transparency about AI assistance +- What was AI-generated +- Required validations +- Appropriate/inappropriate uses +- Contribution guidelines + +**.claude_project_context.md**: Technical reference +- Architecture decisions +- Proto structure details +- Common pitfalls diff --git a/QUICKSTART.md b/QUICKSTART.md new file mode 100644 index 0000000..8005589 --- /dev/null +++ b/QUICKSTART.md @@ -0,0 +1,239 @@ +# Quick Start Guide + +Get started analyzing Zcash compact block overhead in 5 minutes. + +## Prerequisites Check + +```bash +# Check Rust is installed +rustc --version + +# Check Python is installed +python --version # Should be 3.8+ + +# Check Zebrad is running +curl -X POST http://127.0.0.1:8232 -d '{"method":"getblockcount","params":[],"id":1}' + +# Check lightwalletd is running +grpcurl -plaintext localhost:9067 list +``` + +## Setup (5 minutes) + +### 1. Get the Project + +```bash +cargo new compact_block_analyzer +cd compact_block_analyzer +mkdir proto +``` + +### 2. Get Protocol Buffers + +```bash +# Clone the lightwallet-protocol repo +git clone https://github.com/zcash/lightwallet-protocol.git temp_proto +cd temp_proto +git fetch origin pull/1/head:pr-1 +git checkout pr-1 + +# Copy proto files +cp compact_formats.proto ../proto/ +cp service.proto ../proto/ + +cd .. +rm -rf temp_proto +``` + +### 3. Setup Build System + +Create `build.rs`: +```rust +fn main() -> Result<(), Box> { + tonic_build::configure() + .build_server(false) + .compile( + &["proto/service.proto", "proto/compact_formats.proto"], + &["proto/"], + )?; + Ok(()) +} +``` + +### 4. Copy Code + +Replace `Cargo.toml` and `src/main.rs` with the provided artifacts. + +### 5. Build + +```bash +cargo build --release +``` + +## Run Your First Analysis (2 minutes) + +### Quick Test (100 blocks) + +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + range 2400000 2400100 \ + test.csv +``` + +### Recommended Analysis (~30 minutes for 5000 blocks) + +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + recommended \ + results.csv +``` + +This will: +- Sample 5000 blocks strategically across all eras +- Take ~30 minutes (100ms delay between blocks) +- Provide statistically significant results + +## Visualize Results (1 minute) + +```bash +# Install Python dependencies +pip install pandas matplotlib seaborn numpy scipy + +# Generate all charts +python visualize.py results.csv +``` + +Open `charts/statistical_report.txt` to see the analysis! + +## Understanding the Output + +### Key Metrics to Look At + +1. **Median overhead** (in the report) + - This is the "typical" overhead + - Most important single number + +2. **95th percentile** (in the report) + - Worst-case planning + - Important for capacity planning + +3. **Distribution chart** (`charts/distribution.png`) + - Shows the spread of overhead values + - Look for the shape (normal, skewed, bimodal?) + +4. **Bandwidth impact chart** (`charts/bandwidth_impact.png`) + - Practical impact: MB per day, sync time, cost + - This makes the numbers concrete + +### Decision Framework + +The report will tell you: + +- **< 20% median overhead**: Low impact, consider making it default +- **20-50% median overhead**: Moderate impact, consider opt-in +- **> 50% median overhead**: High impact, needs separate method + +## Common Issues + +### "Connection refused" on port 9067 +```bash +# Restart lightwalletd +lightwalletd --grpc-bind-addr=127.0.0.1:9067 --zcash-conf-path=/path/to/zebra.conf +``` + +### "Connection refused" on port 8232 +```bash +# Check zebrad is running +ps aux | grep zebrad + +# Check zebra.toml has RPC enabled +# Should have [rpc] section with listen_addr = "127.0.0.1:8232" +``` + +### Build errors about proto files +```bash +# Ensure proto files exist +ls -la proto/ + +# Clean and rebuild +cargo clean +cargo build --release +``` + +### Python import errors +```bash +# Install all dependencies +pip install -r requirements.txt +``` + +## Next Steps + +### Experiment with Different Sampling + +```bash +# Quick analysis (1500 blocks, ~15 minutes) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 quick quick.csv + +# Thorough analysis (11000 blocks, ~2 hours) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 thorough thorough.csv + +# Equal samples per era +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 equal equal.csv +``` + +### Compare Different Eras + +```bash +# Pre-Sapling era (heavy transparent) +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 100000 101000 pre_sapling.csv + +# Recent blocks +cargo run --release -- http://127.0.0.1:9067 http://127.0.0.1:8232 range 2400000 2401000 recent.csv + +# Visualize both +python visualize.py pre_sapling.csv -o charts_pre_sapling +python visualize.py recent.csv -o charts_recent +``` + +### Dive Deeper + +- Read `statistical_report.txt` for detailed analysis +- Examine correlation charts to understand what drives overhead +- Look at heatmaps to see how overhead varies by transaction patterns +- Check time series to see trends over blockchain history + +## Tips for Best Results + +1. **Use 'recommended' mode** for most analyses + - Balanced, statistically sound + - Reasonable time (~30 minutes) + +2. **Run multiple samples** if results seem unexpected + - Sampling has some variance + - Multiple runs can confirm findings + +3. **Focus on recent blocks** for current decisions + - Old blocks less relevant for current usage + - But historical context helps understand trends + +4. **Look at the whole picture** + - Don't just look at mean/median + - Check the distribution, outliers, worst-case + +5. **Consider practical impact** + - MB per day matters more than abstract percentages + - Think about mobile users, limited bandwidth + +## Getting Help + +- Check `README.md` for full documentation +- Check `.claude_project_context.md` for technical details +- Check `AI_DISCLAIMER.md` for limitations +- Open an issue if you find bugs +- Validate results against your own calculations + +Happy analyzing! 🚀 diff --git a/README.md b/README.md index a398789..5a519a1 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,7 @@ zebrad start Verify RPC is accessible: ```bash curl -X POST http://127.0.0.1:8232 \ + -H "Content-Type: application/json" \ -d '{"method":"getblockcount","params":[],"id":1}' ``` @@ -120,77 +121,140 @@ grpcurl -plaintext localhost:9067 list ### Run Analysis +The tool supports multiple analysis modes: + +#### Quick Analysis (~1,500 blocks) ```bash cargo run --release -- \ - \ - \ - \ - \ - [output-csv] + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + quick \ + quick_results.csv ``` -**Example:** +#### Recommended Analysis (~5,000 blocks) - Best Balance ```bash -# Analyze 1000 recent blocks cargo run --release -- \ http://127.0.0.1:9067 \ http://127.0.0.1:8232 \ - 2400000 \ - 2401000 \ + recommended \ results.csv ``` -### Recommended Analysis Ranges +This uses a **hybrid sampling strategy**: +- 750 samples from each protocol era (pre-Sapling, Sapling, Canopy, NU5) +- 2,000 additional samples from recent blocks (last 100K) +- Provides balanced historical context with focus on current usage -**Recent blocks** (most relevant for current usage patterns): +#### Thorough Analysis (~11,000 blocks) ```bash -# Last ~1000 blocks cargo run --release -- \ http://127.0.0.1:9067 \ http://127.0.0.1:8232 \ - 2400000 \ - 2401000 \ - recent_blocks.csv + thorough \ + thorough_results.csv ``` -**Historical comparison** (different network usage patterns): +#### Equal Sampling Per Era (~4,000 blocks) ```bash -# Around Sapling activation (Oct 2018) cargo run --release -- \ http://127.0.0.1:9067 \ http://127.0.0.1:8232 \ - 419200 \ - 420200 \ - sapling_era.csv + equal \ + equal_results.csv +``` -# Around NU5 activation (May 2022) - Orchard introduction +#### Proportional Sampling (~5,000 blocks) +```bash +# Samples proportionally to era size cargo run --release -- \ http://127.0.0.1:9067 \ http://127.0.0.1:8232 \ - 1687104 \ - 1688104 \ - nu5_era.csv + proportional \ + proportional_results.csv ``` -## Output +#### Weighted Sampling (~5,000 blocks) +```bash +# Custom weights favoring recent blocks +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + weighted \ + weighted_results.csv +``` -### Console Output +#### Specific Range (Original Mode) +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + range 2400000 2401000 \ + range_results.csv +``` + +### Sampling Strategies Explained + +**Why use sampling?** The Zcash blockchain has 2.4M+ blocks. Analyzing every block would take days. Statistical sampling gives accurate results in minutes. + +| Strategy | Description | Best For | +|----------|-------------|----------| +| **Quick** | Fast overview with fewer samples | Initial exploration | +| **Recommended** | Balanced approach with recent focus | Most use cases | +| **Thorough** | Comprehensive coverage | Final analysis | +| **Equal** | Same samples per era | Era comparison | +| **Proportional** | Samples match blockchain distribution | Representing whole chain | +| **Weighted** | More recent, less historical | Current state focus | -The tool provides real-time progress and summary statistics: +### Visualize Results +After running the analysis, generate charts and statistics: + +```bash +# Install Python dependencies +pip install -r requirements.txt + +# Generate all visualizations +python visualize.py results.csv --output-dir ./charts +``` + +This creates: +- **distribution.png** - Histogram and box plot of overhead +- **time_series.png** - Overhead trends over blockchain height +- **by_era.png** - Comparison across protocol eras +- **correlations.png** - Relationship between overhead and transaction characteristics +- **cumulative.png** - Cumulative distribution functions +- **bandwidth_impact.png** - Practical bandwidth scenarios +- **heatmap.png** - Overhead by era and transaction patterns +- **statistical_report.txt** - Comprehensive statistical analysis + +### Example Output + +**Console output during analysis:** ``` -Analyzing blocks 2400000 to 2401000... -Block 2400000: current=15234 bytes, estimated=18456 bytes, delta=+3222 bytes (21.15%), tx=45, tin=12, tout=89 -Block 2400001: current=12890 bytes, estimated=14234 bytes, delta=+1344 bytes (10.43%), tx=32, tin=5, tout=43 +Current blockchain tip: 2450000 + +Sampling Strategy: HybridRecent +Total samples: 5000 + +Distribution by era: + pre_sapling: 750 samples (15.0% of total, 1 in 559 blocks) + sapling: 750 samples (15.0% of total, 1 in 836 blocks) + canopy: 750 samples (15.0% of total, 1 in 854 blocks) + nu5: 2750 samples (55.0% of total, 1 in 295 blocks) + +Analyzing 5000 blocks... +Progress: 0/5000 (0.0%) +Progress: 500/5000 (10.0%) ... === ANALYSIS SUMMARY === -Blocks analyzed: 1000 +Blocks analyzed: 5000 Current compact blocks: - Total: 15.23 MB + Total: 76.23 MB With transparent data: - Estimated total: 18.67 MB - Delta: +3.44 MB + Estimated total: 93.45 MB + Delta: +17.22 MB Overall increase: 22.58% Per-block statistics: @@ -205,17 +269,87 @@ Practical impact: Additional bandwidth per day: 9.89 MB ``` +**Statistical report snippet:** +``` +DECISION FRAMEWORK +-------------------------------------------------------------------------------- +Median overhead: 18.5% +95th percentile: 35.2% + +RECOMMENDATION: LOW IMPACT + The overhead is relatively small (<20%). Consider making transparent + data part of the default GetBlockRange method. This would: + - Simplify the API (single method) + - Provide feature parity with full nodes + - Have minimal bandwidth impact on users +``` + +## Output + +### Console Output + +The tool provides real-time progress and summary statistics during analysis. + ### CSV Output Detailed per-block analysis in CSV format: ```csv -height,current_compact_size,estimated_with_transparent,delta_bytes,delta_percent,tx_count,transparent_inputs,transparent_outputs -2400000,15234,18456,3222,21.15,45,12,89 -2400001,12890,14234,1344,10.43,32,5,43 +height,era,current_compact_size,estimated_with_transparent,delta_bytes,delta_percent,tx_count,transparent_inputs,transparent_outputs +2400000,nu5,15234,18456,3222,21.15,45,12,89 +2400001,nu5,12890,14234,1344,10.43,32,5,43 ... ``` +### Visualization Output + +The Python script generates comprehensive visualizations: + +1. **Distribution Analysis** + - Histogram with kernel density estimation + - Box plot showing quartiles and outliers + - Marked median and mean values + +2. **Time Series Analysis** + - Overhead percentage over blockchain height + - Absolute size increase over time + - Rolling averages to show trends + - Era boundaries marked + +3. **Era Comparison** + - Box plots comparing distributions across eras + - Violin plots showing density + - Bar charts with standard deviations + - Sample size distribution + +4. **Correlation Analysis** + - Overhead vs transparent inputs + - Overhead vs transparent outputs + - Overhead vs transaction count + - Overhead vs current block size + +5. **Cumulative Distribution** + - CDF of overhead percentages + - CDF of absolute byte increases + - Percentile markers (P50, P75, P90, P95, P99) + +6. **Bandwidth Impact** + - Daily sync bandwidth comparison + - Full chain sync comparison + - Mobile data cost estimates + - Sync time projections + +7. **Heatmaps** + - Overhead by era and transaction count + - Overhead by era and transparent I/O + +8. **Statistical Report** + - Summary statistics with confidence intervals + - Statistics broken down by era + - Practical bandwidth calculations + - Correlation coefficients + - Decision framework recommendations + ## How It Works 1. **Fetch real compact block** from lightwalletd via gRPC diff --git a/analyzer/Cargo.toml b/analyzer/Cargo.toml new file mode 100644 index 0000000..056975f --- /dev/null +++ b/analyzer/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "compact-block-analyzer" +version = "0.1.0" +edition = "2021" +authors = ["Your Name "] +license = "MIT OR Apache-2.0" +description = "Analyze bandwidth impact of transparent data in Zcash compact blocks" +repository = "https://github.com/yourusername/compact-block-analyzer" + +[dependencies] +tokio = { version = "1", features = ["full"] } +tonic = "0.11" +prost = "0.12" +prost-types = "0.12" +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +hex = "0.4" +csv = "1.3" +anyhow = "1.0" +rand = "0.8" +clap = { version = "4.0", features = ["derive"] } + +[build-dependencies] +tonic-build = "0.11" + +[dev-dependencies] +tempfile = "3.0" diff --git a/analyzer/README.md b/analyzer/README.md new file mode 100644 index 0000000..a66a3a2 --- /dev/null +++ b/analyzer/README.md @@ -0,0 +1,508 @@ +# Compact Block Analyzer for Zcash + +A tool to analyze the bandwidth impact of adding transparent transaction data (`CompactTxIn` and `TxOut`) to Zcash's compact block protocol. + +## Overview + +This analyzer helps evaluate the proposed changes to the [lightwallet-protocol](https://github.com/zcash/lightwallet-protocol/pull/1) by measuring: + +- Current compact block sizes from production lightwalletd +- Estimated sizes with transparent input/output data added +- Bandwidth impact on light clients syncing block ranges + +The tool fetches **real compact blocks** from lightwalletd and compares them against estimated sizes calculated from full block data in Zebrad, giving accurate projections of the bandwidth impact. + +## Background + +The Zcash light client protocol currently omits transparent transaction inputs and outputs from compact blocks. [PR #1](https://github.com/zcash/lightwallet-protocol/pull/1) proposes adding: + +- `CompactTxIn` - references to transparent inputs being spent +- `TxOut` - transparent outputs being created + +This analysis helps decide whether to: +- Make transparent data part of the default `GetBlockRange` RPC +- Create a separate opt-in method for clients that need it +- Use pool-based filtering (as implemented in [librustzcash PR #1781](https://github.com/zcash/librustzcash/pull/1781)) + +## Prerequisites + +- **Rust** 1.70+ ([install](https://rustup.rs/)) +- **Zebrad** - synced Zcash full node with RPC enabled +- **Lightwalletd** - connected to your Zebrad instance + +## Installation + +### 1. Clone and Setup + +```bash +# Create the project +cargo new compact_block_analyzer +cd compact_block_analyzer + +# Create proto directory +mkdir proto +``` + +### 2. Get Protocol Buffer Definitions + +```bash +# Clone the lightwallet-protocol repository +git clone https://github.com/zcash/lightwallet-protocol.git +cd lightwallet-protocol + +# Checkout the PR with transparent data additions +git fetch origin pull/1/head:pr-1 +git checkout pr-1 + +# Copy proto files to your project +cp compact_formats.proto ../compact_block_analyzer/proto/ +cp service.proto ../compact_block_analyzer/proto/ + +cd ../compact_block_analyzer +``` + +### 3. Configure Build + +Create `build.rs` in the project root: + +```rust +fn main() -> Result<(), Box> { + tonic_build::configure() + .build_server(false) // Client only + .compile( + &["proto/service.proto", "proto/compact_formats.proto"], + &["proto/"], + )?; + Ok(()) +} +``` + +### 4. Update Dependencies + +Replace `Cargo.toml` with the dependencies from the artifact, or copy `src/main.rs` from the artifact which includes the dependency list. + +### 5. Build + +```bash +cargo build --release +``` + +## Usage + +### Start Required Services + +#### Zebrad +```bash +# If not already running +zebrad start +``` + +Verify RPC is accessible: +```bash +curl -X POST http://127.0.0.1:8232 \ + -d '{"method":"getblockcount","params":[],"id":1}' +``` + +#### Lightwalletd +```bash +lightwalletd \ + --grpc-bind-addr=127.0.0.1:9067 \ + --zcash-conf-path=/path/to/zebra.conf \ + --log-file=/dev/stdout +``` + +Verify lightwalletd is running: +```bash +# With grpcurl installed +grpcurl -plaintext localhost:9067 list +# Should show: cash.z.wallet.sdk.rpc.CompactTxStreamer +``` + +### Run Analysis + +The tool supports multiple analysis modes: + +#### Quick Analysis (~1,500 blocks) +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + quick \ + quick_results.csv +``` + +#### Recommended Analysis (~5,000 blocks) - Best Balance +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + recommended \ + results.csv +``` + +This uses a **hybrid sampling strategy**: +- 750 samples from each protocol era (pre-Sapling, Sapling, Canopy, NU5) +- 2,000 additional samples from recent blocks (last 100K) +- Provides balanced historical context with focus on current usage + +#### Thorough Analysis (~11,000 blocks) +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + thorough \ + thorough_results.csv +``` + +#### Equal Sampling Per Era (~4,000 blocks) +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + equal \ + equal_results.csv +``` + +#### Proportional Sampling (~5,000 blocks) +```bash +# Samples proportionally to era size +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + proportional \ + proportional_results.csv +``` + +#### Weighted Sampling (~5,000 blocks) +```bash +# Custom weights favoring recent blocks +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + weighted \ + weighted_results.csv +``` + +#### Specific Range (Original Mode) +```bash +cargo run --release -- \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + range 2400000 2401000 \ + range_results.csv +``` + +### Sampling Strategies Explained + +**Why use sampling?** The Zcash blockchain has 2.4M+ blocks. Analyzing every block would take days. Statistical sampling gives accurate results in minutes. + +| Strategy | Description | Best For | +|----------|-------------|----------| +| **Quick** | Fast overview with fewer samples | Initial exploration | +| **Recommended** | Balanced approach with recent focus | Most use cases | +| **Thorough** | Comprehensive coverage | Final analysis | +| **Equal** | Same samples per era | Era comparison | +| **Proportional** | Samples match blockchain distribution | Representing whole chain | +| **Weighted** | More recent, less historical | Current state focus | + +### Visualize Results + +After running the analysis, generate charts and statistics: + +```bash +# Install Python dependencies +pip install -r requirements.txt + +# Generate all visualizations +python visualize.py results.csv --output-dir ./charts +``` + +This creates: +- **distribution.png** - Histogram and box plot of overhead +- **time_series.png** - Overhead trends over blockchain height +- **by_era.png** - Comparison across protocol eras +- **correlations.png** - Relationship between overhead and transaction characteristics +- **cumulative.png** - Cumulative distribution functions +- **bandwidth_impact.png** - Practical bandwidth scenarios +- **heatmap.png** - Overhead by era and transaction patterns +- **statistical_report.txt** - Comprehensive statistical analysis + +### Example Output + +**Console output during analysis:** +``` +Current blockchain tip: 2450000 + +Sampling Strategy: HybridRecent +Total samples: 5000 + +Distribution by era: + pre_sapling: 750 samples (15.0% of total, 1 in 559 blocks) + sapling: 750 samples (15.0% of total, 1 in 836 blocks) + canopy: 750 samples (15.0% of total, 1 in 854 blocks) + nu5: 2750 samples (55.0% of total, 1 in 295 blocks) + +Analyzing 5000 blocks... +Progress: 0/5000 (0.0%) +Progress: 500/5000 (10.0%) +... + +=== ANALYSIS SUMMARY === +Blocks analyzed: 5000 +Current compact blocks: + Total: 76.23 MB +With transparent data: + Estimated total: 93.45 MB + Delta: +17.22 MB + Overall increase: 22.58% + +Per-block statistics: + Median increase: 18.45% + 95th percentile: 35.21% + Min: 5.32% + Max: 47.83% + +Practical impact: + Current daily sync (~2880 blocks): 43.86 MB + With transparent: 53.75 MB + Additional bandwidth per day: 9.89 MB +``` + +**Statistical report snippet:** +``` +DECISION FRAMEWORK +-------------------------------------------------------------------------------- +Median overhead: 18.5% +95th percentile: 35.2% + +RECOMMENDATION: LOW IMPACT + The overhead is relatively small (<20%). Consider making transparent + data part of the default GetBlockRange method. This would: + - Simplify the API (single method) + - Provide feature parity with full nodes + - Have minimal bandwidth impact on users +``` + +## Output + +### Console Output + +The tool provides real-time progress and summary statistics during analysis. + +### CSV Output + +Detailed per-block analysis in CSV format: + +```csv +height,era,current_compact_size,estimated_with_transparent,delta_bytes,delta_percent,tx_count,transparent_inputs,transparent_outputs +2400000,nu5,15234,18456,3222,21.15,45,12,89 +2400001,nu5,12890,14234,1344,10.43,32,5,43 +... +``` + +### Visualization Output + +The Python script generates comprehensive visualizations: + +1. **Distribution Analysis** + - Histogram with kernel density estimation + - Box plot showing quartiles and outliers + - Marked median and mean values + +2. **Time Series Analysis** + - Overhead percentage over blockchain height + - Absolute size increase over time + - Rolling averages to show trends + - Era boundaries marked + +3. **Era Comparison** + - Box plots comparing distributions across eras + - Violin plots showing density + - Bar charts with standard deviations + - Sample size distribution + +4. **Correlation Analysis** + - Overhead vs transparent inputs + - Overhead vs transparent outputs + - Overhead vs transaction count + - Overhead vs current block size + +5. **Cumulative Distribution** + - CDF of overhead percentages + - CDF of absolute byte increases + - Percentile markers (P50, P75, P90, P95, P99) + +6. **Bandwidth Impact** + - Daily sync bandwidth comparison + - Full chain sync comparison + - Mobile data cost estimates + - Sync time projections + +7. **Heatmaps** + - Overhead by era and transaction count + - Overhead by era and transparent I/O + +8. **Statistical Report** + - Summary statistics with confidence intervals + - Statistics broken down by era + - Practical bandwidth calculations + - Correlation coefficients + - Decision framework recommendations + +## How It Works + +1. **Fetch real compact block** from lightwalletd via gRPC + - Gets the actual production compact block size + - Includes all current fields (Sapling outputs, Orchard actions, etc.) + +2. **Fetch full block** from Zebrad via RPC + - Gets transparent input/output data + - Provides transaction details needed for estimation + +3. **Calculate overhead** using protobuf encoding rules + - Estimates size of `CompactTxIn` messages (containing `OutPoint`) + - Estimates size of `TxOut` messages (value + scriptPubKey) + - Accounts for protobuf field tags, length prefixes, and nested messages + +4. **Compare and report** + - Current size vs. estimated size with transparent data + - Per-block and aggregate statistics + +## Protobuf Size Estimation + +The estimator calculates sizes based on the proposed proto definitions: + +```protobuf +message OutPoint { + bytes txid = 1; // 32 bytes + uint32 index = 2; // varint +} + +message CompactTxIn { + OutPoint prevout = 1; +} + +message TxOut { + uint32 value = 1; // varint + bytes scriptPubKey = 2; // variable length +} +``` + +Added to `CompactTx`: +```protobuf +repeated CompactTxIn vin = 7; +repeated TxOut vout = 8; +``` + +The calculation includes: +- Field tags (1 byte per field) +- Length prefixes for bytes and nested messages (varint) +- Actual data sizes +- Nested message overhead + +## Interpreting Results + +### Decision Guidelines + +- **< 20% increase**: Consider making transparent data default + - Minimal impact on bandwidth + - Simplifies API (single method) + - Better for light client feature parity + +- **20-50% increase**: Consider separate opt-in method + - Significant but manageable overhead + - Let clients choose based on their needs + - Pool filtering could help (librustzcash PR #1781) + +- **> 50% increase**: Likely needs separate method + - Major bandwidth impact + - Important for mobile/limited bandwidth users + - Clear opt-in for clients that need transparent data + +### Key Metrics to Examine + +1. **Median increase** - typical overhead +2. **95th percentile** - worst-case for active blocks +3. **Daily bandwidth impact** - practical cost for staying synced +4. **Initial sync impact** - multiply by ~2.4M blocks +5. **Correlation with transparent usage** - understand which blocks drive overhead + +## Troubleshooting + +### Connection Errors + +**Port 9067 (lightwalletd):** +```bash +# Check if running +ps aux | grep lightwalletd +netstat -tlnp | grep 9067 + +# Test connection +grpcurl -plaintext localhost:9067 list +``` + +**Port 8232 (zebrad):** +```bash +# Check if running +ps aux | grep zebrad +netstat -tlnp | grep 8232 + +# Test RPC +curl -X POST http://127.0.0.1:8232 \ + -d '{"method":"getblockcount","params":[],"id":1}' +``` + +### Build Errors + +**Proto compilation fails:** +```bash +# Ensure proto files exist +ls -la proto/ + +# Clean and rebuild +cargo clean +cargo build --release +``` + +**"Block not found" errors:** +- Check if block height exists on mainnet +- Verify Zebrad is fully synced +- Ensure lightwalletd has indexed the blocks + +### Rate Limiting + +The tool includes a 100ms delay between blocks to avoid overwhelming your node. For faster analysis: + +1. Reduce the delay in the code +2. Run multiple instances for different ranges +3. Use a more powerful machine for Zebrad + +## Contributing + +This tool is designed for protocol analysis. Contributions welcome: + +- Improved size estimation accuracy +- Additional output formats (JSON, charts) +- Statistical analysis enhancements +- Performance optimizations + +## Related Work + +- [lightwallet-protocol PR #1](https://github.com/zcash/lightwallet-protocol/pull/1) - Proto definitions +- [librustzcash PR #1781](https://github.com/zcash/librustzcash/pull/1781) - Pool filtering implementation +- [Zcash Protocol Specification](https://zips.z.cash/) - Full protocol details + +## License + +Same license as the Zcash lightwallet-protocol project. + +## Support + +For questions or issues: +- Open an issue in this repository +- Discuss on Zcash Community Forum +- Zcash R&D Discord + +## Acknowledgments + +Built to support analysis for improving Zcash light client protocol bandwidth efficiency. diff --git a/analyzer/build.rs b/analyzer/build.rs new file mode 100644 index 0000000..0c9ec23 --- /dev/null +++ b/analyzer/build.rs @@ -0,0 +1,7 @@ +fn main() -> Result<(), Box> { + tonic_build::configure().build_server(false).compile( + &["proto/service.proto", "proto/compact_formats.proto"], + &["proto/"], + )?; + Ok(()) +} diff --git a/analyzer/src/main.rs b/analyzer/src/main.rs new file mode 100644 index 0000000..fc73706 --- /dev/null +++ b/analyzer/src/main.rs @@ -0,0 +1,596 @@ +// Complete working implementation that fetches REAL data from Zebra and lightwalletd + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +// Include generated gRPC client code +pub mod cash { + pub mod z { + pub mod wallet { + pub mod sdk { + pub mod rpc { + tonic::include_proto!("cash.z.wallet.sdk.rpc"); + } + } + } + } +} + +use cash::z::wallet::sdk::rpc::{ + compact_tx_streamer_client::CompactTxStreamerClient, BlockId, BlockRange, +}; + +// Import sampling module +mod sampling; +use sampling::*; + +// Zebra RPC response structures +#[derive(Debug, Deserialize)] +struct ZebraBlock { + tx: Vec, +} + +#[derive(Debug, Deserialize)] +struct ZebraTransaction { + vin: Vec, + vout: Vec, +} + +#[derive(Debug, Deserialize)] +struct ZebraVin { + txid: Option, + vout: Option, +} + +#[derive(Debug, Deserialize)] +struct ZebraVout { + value: f64, + #[serde(rename = "scriptPubKey")] + script_pubkey: ScriptPubKey, +} + +#[derive(Debug, Deserialize)] +struct ScriptPubKey { + hex: String, +} + +// Analysis results +#[derive(Debug, Serialize)] +struct BlockAnalysis { + height: u64, + era: String, + current_compact_size: usize, + estimated_with_transparent: usize, + delta_bytes: i64, + delta_percent: f64, + tx_count: usize, + transparent_inputs: usize, + transparent_outputs: usize, +} + +struct TransparentEstimator { + zebra_rpc_url: String, + lightwalletd_url: String, + http_client: reqwest::Client, +} + +impl TransparentEstimator { + fn new(zebra_rpc_url: String, lightwalletd_url: String) -> Self { + Self { + zebra_rpc_url, + lightwalletd_url, + http_client: reqwest::Client::new(), + } + } + + async fn get_current_tip(&self) -> Result { + let request = serde_json::json!({ + "jsonrpc": "2.0", + "method": "getblockcount", + "params": [], + "id": 1 + }); + + let response: serde_json::Value = self + .http_client + .post(&self.zebra_rpc_url) + .json(&request) + .send() + .await? + .json() + .await?; + + let tip = response["result"] + .as_u64() + .ok_or_else(|| anyhow::anyhow!("Failed to get block count"))?; + + Ok(tip) + } + + async fn get_compact_block_from_lightwalletd(&self, height: u64) -> Result> { + // Connect to lightwalletd gRPC service + let mut client = CompactTxStreamerClient::connect(self.lightwalletd_url.clone()).await?; + + // Request a single block by creating a range with start=end=height + // NO pool_types field - using main branch proto + let request = tonic::Request::new(BlockRange { + start: Some(BlockId { + height, + hash: vec![], + }), + end: Some(BlockId { + height, + hash: vec![], + }), + }); + + // Get the block stream (will have just one block) + let mut stream = client.get_block_range(request).await?.into_inner(); + + // Get the single block from the stream + if let Some(compact_block) = stream.message().await? { + // Encode the CompactBlock to bytes to measure its actual size + use prost::Message; + let mut buf = Vec::new(); + compact_block.encode(&mut buf)?; + return Ok(buf); + } + + anyhow::bail!("Block {} not found in lightwalletd", height) + } + + async fn get_full_block_from_zebra(&self, height: u64) -> Result { + let request = serde_json::json!({ + "jsonrpc": "2.0", + "method": "getblock", + "params": [height.to_string(), 2], + "id": 1 + }); + + let response: serde_json::Value = self + .http_client + .post(&self.zebra_rpc_url) + .json(&request) + .send() + .await? + .json() + .await?; + + let block: ZebraBlock = serde_json::from_value(response["result"].clone())?; + Ok(block) + } + + fn estimate_transparent_overhead(&self, block: &ZebraBlock) -> (usize, usize, usize) { + // Estimate protobuf size for transparent data based on actual PR definitions: + // + // message OutPoint { + // bytes txid = 1; // 32 bytes + // uint32 index = 2; // varint + // } + // + // message CompactTxIn { + // OutPoint prevout = 1; // nested message + // } + // + // For coinbase, we'll estimate a fixed overhead since the .proto isn't finalized yet + // + // message TxOut { + // uint32 value = 1; // varint (note: uint32, not uint64!) + // bytes scriptPubKey = 2; // variable length + // } + // + // CompactTx gets: + // repeated CompactTxIn vin = 7; + // repeated TxOut vout = 8; + + let mut total_overhead = 0; + let mut input_count = 0; + let mut output_count = 0; + + for tx in &block.tx { + // Estimate CompactTxIn (repeated field in CompactTx) + for vin in &tx.vin { + if let (Some(_txid), Some(vout_idx)) = (&vin.txid, &vin.vout) { + // Regular transparent input (not a coinbase) + + // OutPoint message size: + let mut outpoint_size = 0; + + // Field 1: bytes txid = 32 bytes + // Tag (1 byte) + length varint (1 byte) + 32 bytes + outpoint_size += 1 + 1 + 32; + + // Field 2: uint32 index (varint) + // Tag (1 byte) + varint value (1-5 bytes, typically 1-2) + outpoint_size += 1 + Self::varint_size(*vout_idx as usize); + + // CompactTxIn wraps OutPoint as field 1 + let mut compact_txin_size = 0; + // Tag for field 1 (1 byte) + length of OutPoint + OutPoint data + compact_txin_size += 1 + Self::varint_size(outpoint_size) + outpoint_size; + + // This CompactTxIn is in a repeated field (vin = 7) in CompactTx + // Tag for repeated field (1 byte) + length + message + let vin_entry_size = + 1 + Self::varint_size(compact_txin_size) + compact_txin_size; + + total_overhead += vin_entry_size; + input_count += 1; + } else { + // Coinbase input + // Since the .proto isn't finalized, we estimate a fixed size + // Typical coinbase: ~40-100 bytes of data + sequence + // Conservative estimate for CompactTxIn with coinbase: + // - Field tag for coinbase data: 1 byte + // - Length prefix: 1 byte (for typical 40-100 byte coinbase) + // - Coinbase data: ~70 bytes average + // - Field tag for sequence: 1 byte + // - Sequence value: 4 bytes + // - Repeated field overhead: 1 byte tag + 1 byte length + // Total: ~79 bytes + + const COINBASE_COMPACT_TXIN_SIZE: usize = 79; + total_overhead += COINBASE_COMPACT_TXIN_SIZE; + input_count += 1; + } + } + + // Estimate TxOut (repeated field in CompactTx) + for vout in &tx.vout { + let mut txout_size = 0; + + // Field 1: uint32 value (varint) + let value_zatoshis = (vout.value * 100_000_000.0) as u64; + txout_size += 1 + Self::varint_size(value_zatoshis as usize); + + // Field 2: bytes scriptPubKey + let script_len = vout.script_pubkey.hex.len() / 2; // hex to bytes + txout_size += 1 + Self::varint_size(script_len) + script_len; + + // This TxOut is in a repeated field (vout = 8) in CompactTx + // Tag for repeated field (1 byte) + length + message + let vout_entry_size = 1 + Self::varint_size(txout_size) + txout_size; + + total_overhead += vout_entry_size; + output_count += 1; + } + } + + (total_overhead, input_count, output_count) + } + + fn varint_size(value: usize) -> usize { + // Protobuf varint encoding size + match value { + 0..=127 => 1, + 128..=16383 => 2, + 16384..=2097151 => 3, + 2097152..=268435455 => 4, + _ => 5, + } + } + + async fn analyze_block(&self, height: u64) -> Result { + // Get actual compact block from lightwalletd + let compact_block_bytes = self.get_compact_block_from_lightwalletd(height).await?; + let current_size = compact_block_bytes.len(); + + // Get full block from Zebra to calculate transparent overhead + let full_block = self.get_full_block_from_zebra(height).await?; + let (transparent_overhead, input_count, output_count) = + self.estimate_transparent_overhead(&full_block); + + let estimated_size = current_size + transparent_overhead; + let delta = estimated_size as i64 - current_size as i64; + let delta_percent = if current_size > 0 { + (delta as f64 / current_size as f64) * 100.0 + } else { + 0.0 + }; + + Ok(BlockAnalysis { + height, + era: String::new(), // Will be set by analyze_blocks + current_compact_size: current_size, + estimated_with_transparent: estimated_size, + delta_bytes: delta, + delta_percent, + tx_count: full_block.tx.len(), + transparent_inputs: input_count, + transparent_outputs: output_count, + }) + } + + async fn analyze_blocks(&self, heights: &[u64]) -> Result> { + let mut results = Vec::new(); + let total = heights.len(); + let eras = Era::zcash_eras(*heights.last().unwrap_or(&2_400_000)); + + for (i, &height) in heights.iter().enumerate() { + if i % 10 == 0 { + println!( + "Progress: {}/{} ({:.1}%)", + i, + total, + (i as f64 / total as f64) * 100.0 + ); + } + + match self.analyze_block(height).await { + Ok(mut analysis) => { + analysis.era = Era::get_era_for_height(height, &eras); + println!( + "Block {}: current={} bytes, estimated={} bytes, delta=+{} bytes ({:.2}%), era={}", + height, + analysis.current_compact_size, + analysis.estimated_with_transparent, + analysis.delta_bytes, + analysis.delta_percent, + analysis.era + ); + results.push(analysis); + } + Err(e) => { + eprintln!("Error analyzing block {}: {}", height, e); + } + } + + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + + println!("Progress: {}/{} (100.0%)", total, total); + Ok(results) + } + + fn write_csv(&self, results: &[BlockAnalysis], filename: &str) -> Result<()> { + let mut wtr = csv::Writer::from_path(filename)?; + for result in results { + wtr.serialize(result)?; + } + wtr.flush()?; + Ok(()) + } + + fn print_summary(&self, results: &[BlockAnalysis]) { + if results.is_empty() { + return; + } + + let total_current: usize = results.iter().map(|r| r.current_compact_size).sum(); + let total_estimated: usize = results.iter().map(|r| r.estimated_with_transparent).sum(); + let total_delta = total_estimated as i64 - total_current as i64; + + let mut deltas: Vec = results.iter().map(|r| r.delta_percent).collect(); + deltas.sort_by(|a, b| a.partial_cmp(b).unwrap()); + + let median = deltas[deltas.len() / 2]; + let p95_idx = ((deltas.len() as f64) * 0.95) as usize; + let p95 = deltas[p95_idx.min(deltas.len() - 1)]; + + println!("\n=== ANALYSIS SUMMARY ==="); + println!("Blocks analyzed: {}", results.len()); + println!("\nCurrent compact blocks:"); + println!( + " Total: {} bytes ({:.2} MB)", + total_current, + total_current as f64 / 1_000_000.0 + ); + println!("\nWith transparent data:"); + println!( + " Estimated total: {} bytes ({:.2} MB)", + total_estimated, + total_estimated as f64 / 1_000_000.0 + ); + println!( + " Delta: +{} bytes ({:.2} MB)", + total_delta, + total_delta as f64 / 1_000_000.0 + ); + println!( + " Overall increase: {:.2}%", + (total_delta as f64 / total_current as f64) * 100.0 + ); + println!("\nPer-block statistics:"); + println!(" Median increase: {:.2}%", median); + println!(" 95th percentile: {:.2}%", p95); + println!(" Min: {:.2}%", deltas[0]); + println!(" Max: {:.2}%", deltas[deltas.len() - 1]); + + // Practical impact examples + println!("\nPractical impact:"); + let blocks_per_day = 1152; // Post-Blossom (75s blocks) + let daily_current = (total_current as f64 / results.len() as f64) * blocks_per_day as f64; + let daily_estimated = + (total_estimated as f64 / results.len() as f64) * blocks_per_day as f64; + println!( + " Current daily sync (~{} blocks): {:.2} MB", + blocks_per_day, + daily_current / 1_000_000.0 + ); + println!( + " With transparent: {:.2} MB", + daily_estimated / 1_000_000.0 + ); + println!( + " Additional bandwidth per day: {:.2} MB", + (daily_estimated - daily_current) / 1_000_000.0 + ); + } +} + +#[tokio::main] +async fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + + if args.len() < 4 { + eprintln!( + "Usage: {} [output.csv]", + args[0] + ); + eprintln!(); + eprintln!("Modes:"); + eprintln!(" range - Analyze specific block range"); + eprintln!(" quick - Quick sampling (~1500 blocks)"); + eprintln!(" recommended - Balanced sampling (~5000 blocks)"); + eprintln!(" thorough - Thorough sampling (~11000 blocks)"); + eprintln!(" equal - Equal samples per era (~4000 blocks)"); + eprintln!(" proportional - Proportional to era size (~5000 blocks)"); + eprintln!(" weighted - Weighted toward recent (~5000 blocks)"); + eprintln!(" density - Every Nth block across all eras"); + eprintln!(" complete - Analyze every block in range (no sampling)"); + eprintln!(); + eprintln!("Examples:"); + eprintln!( + " {} http://127.0.0.1:9067 http://127.0.0.1:8232 range 2400000 2401000", + args[0] + ); + eprintln!( + " {} http://127.0.0.1:9067 http://127.0.0.1:8232 recommended results.csv", + args[0] + ); + eprintln!( + " {} http://127.0.0.1:9067 http://127.0.0.1:8232 density 1000 density.csv", + args[0] + ); + eprintln!( + " {} http://127.0.0.1:9067 http://127.0.0.1:8232 complete 2800000 2800100", + args[0] + ); + std::process::exit(1); + } + + let lightwalletd_url = &args[1]; + let zebra_rpc_url = &args[2]; + let mode = &args[3]; + + // Get current tip from Zebra + let estimator = + TransparentEstimator::new(zebra_rpc_url.to_string(), lightwalletd_url.to_string()); + + let current_tip = estimator.get_current_tip().await?; + println!("Current blockchain tip: {}", current_tip); + println!(); + + let (blocks, output_file) = match mode.as_str() { + "range" => { + if args.len() < 6 { + eprintln!("Error: range mode requires "); + std::process::exit(1); + } + let start: u64 = args[4].parse()?; + let end: u64 = args[5].parse()?; + let output = args.get(6).map(|s| s.as_str()).unwrap_or("results.csv"); + ((start..=end).collect(), output.to_string()) + } + "complete" => { + if args.len() < 6 { + eprintln!("Error: complete mode requires "); + std::process::exit(1); + } + let start: u64 = args[4].parse()?; + let end: u64 = args[5].parse()?; + let output = args.get(6).map(|s| s.as_str()).unwrap_or("complete.csv"); + println!("Complete analysis: every block from {} to {}", start, end); + println!("Total blocks: {}", end - start + 1); + let sampler = Sampler::new( + SamplingStrategy::Complete { start, end }, + current_tip, + Some(42), + ); + (sampler.generate_samples(), output.to_string()) + } + "density" => { + if args.len() < 5 { + eprintln!("Error: density mode requires "); + std::process::exit(1); + } + let every_n: u64 = args[4].parse()?; + let output = args + .get(5) + .map(|s| s.as_str()) + .unwrap_or("density_sample.csv"); + let sampler = Sampler::new( + SamplingStrategy::FixedDensity { every_n }, + current_tip, + Some(42), + ); + println!("{}", sampler.describe()); + (sampler.generate_samples(), output.to_string()) + } + "quick" => { + let sampler = create_quick_sampler(current_tip); + println!("{}", sampler.describe()); + let output = args + .get(4) + .map(|s| s.as_str()) + .unwrap_or("quick_sample.csv"); + (sampler.generate_samples(), output.to_string()) + } + "recommended" => { + let sampler = create_recommended_sampler(current_tip); + println!("{}", sampler.describe()); + let output = args + .get(4) + .map(|s| s.as_str()) + .unwrap_or("recommended_sample.csv"); + (sampler.generate_samples(), output.to_string()) + } + "thorough" => { + let sampler = create_thorough_sampler(current_tip); + println!("{}", sampler.describe()); + let output = args + .get(4) + .map(|s| s.as_str()) + .unwrap_or("thorough_sample.csv"); + (sampler.generate_samples(), output.to_string()) + } + "equal" => { + let sampler = create_equal_sampler(current_tip); + println!("{}", sampler.describe()); + let output = args + .get(4) + .map(|s| s.as_str()) + .unwrap_or("equal_sample.csv"); + (sampler.generate_samples(), output.to_string()) + } + "proportional" => { + let sampler = create_proportional_sampler(current_tip); + println!("{}", sampler.describe()); + let output = args + .get(4) + .map(|s| s.as_str()) + .unwrap_or("proportional_sample.csv"); + (sampler.generate_samples(), output.to_string()) + } + "weighted" => { + let sampler = create_weighted_sampler(current_tip); + println!("{}", sampler.describe()); + let output = args + .get(4) + .map(|s| s.as_str()) + .unwrap_or("weighted_sample.csv"); + (sampler.generate_samples(), output.to_string()) + } + _ => { + eprintln!("Error: Unknown mode '{}'", mode); + std::process::exit(1); + } + }; + + println!("Analyzing {} blocks...", blocks.len()); + println!( + "Fetching real compact blocks from lightwalletd: {}", + lightwalletd_url + ); + println!("Fetching full blocks from Zebra: {}", zebra_rpc_url); + println!(); + + let results = estimator.analyze_blocks(&blocks).await?; + + estimator.write_csv(&results, &output_file)?; + println!("\nDetailed results written to: {}", output_file); + + estimator.print_summary(&results); + + Ok(()) +} diff --git a/analyzer/src/sampling.rs b/analyzer/src/sampling.rs new file mode 100644 index 0000000..1e0680f --- /dev/null +++ b/analyzer/src/sampling.rs @@ -0,0 +1,409 @@ +// Add to Cargo.toml: +// rand = "0.8" + +use rand::seq::SliceRandom; +use rand::SeedableRng; +use std::collections::BTreeSet; + +#[derive(Debug, Clone)] +pub struct Era { + pub name: String, + pub start: u64, + pub end: u64, +} + +impl Era { + pub fn zcash_eras(current_tip: u64) -> Vec { + vec![ + Era { + name: "sapling".to_string(), + start: 419_200, + end: 653_599, + }, + Era { + name: "blossom".to_string(), + start: 653_600, + end: 902_999, + }, + Era { + name: "heartwood".to_string(), + start: 903_000, + end: 1_046_399, + }, + Era { + name: "canopy".to_string(), + start: 1_046_400, + end: 1_687_103, + }, + Era { + name: "nu5".to_string(), + start: 1_687_104, + end: 2_726_399, + }, + Era { + name: "nu6".to_string(), + start: 2_726_400, + end: current_tip, + }, + ] + } + + pub fn size(&self) -> u64 { + self.end - self.start + 1 + } + + pub fn get_era_for_height(height: u64, eras: &[Era]) -> String { + for era in eras { + if height >= era.start && height <= era.end { + return era.name.clone(); + } + } + "unknown".to_string() + } +} + +#[derive(Debug)] +pub enum SamplingStrategy { + // Sample N blocks from each era equally + EqualPerEra { + samples_per_era: usize, + }, + + // Sample proportionally to era size + Proportional { + total_samples: usize, + }, + + // Equal per era + additional recent samples + HybridRecent { + base_per_era: usize, + recent_additional: usize, + recent_window: u64, // e.g., last 100k blocks + }, + + // Fixed density: 1 in every N blocks + FixedDensity { + every_n: u64, + }, + + // Custom weights per era + Weighted { + weights: Vec<(String, f64)>, // (era_name, weight_0_to_1) + total_samples: usize, + }, + + // All blocks in range (for small ranges) + Complete { + start: u64, + end: u64, + }, +} + +pub struct Sampler { + strategy: SamplingStrategy, + eras: Vec, + seed: Option, +} + +impl Sampler { + pub fn new(strategy: SamplingStrategy, current_tip: u64, seed: Option) -> Self { + let eras = Era::zcash_eras(current_tip); + Self { + strategy, + eras, + seed, + } + } + + pub fn generate_samples(&self) -> Vec { + match &self.strategy { + SamplingStrategy::EqualPerEra { samples_per_era } => { + self.equal_per_era(*samples_per_era) + } + SamplingStrategy::Proportional { total_samples } => self.proportional(*total_samples), + SamplingStrategy::HybridRecent { + base_per_era, + recent_additional, + recent_window, + } => self.hybrid_recent(*base_per_era, *recent_additional, *recent_window), + SamplingStrategy::FixedDensity { every_n } => self.fixed_density(*every_n), + SamplingStrategy::Weighted { + weights, + total_samples, + } => self.weighted(weights, *total_samples), + SamplingStrategy::Complete { start, end } => (*start..=*end).collect(), + } + } + + fn equal_per_era(&self, samples_per_era: usize) -> Vec { + let mut all_samples = BTreeSet::new(); + + for era in &self.eras { + let samples = self.random_sample_from_range( + era.start, + era.end, + samples_per_era.min(era.size() as usize), + ); + all_samples.extend(samples); + } + + all_samples.into_iter().collect() + } + + fn proportional(&self, total_samples: usize) -> Vec { + let mut all_samples = BTreeSet::new(); + let total_blocks: u64 = self.eras.iter().map(|e| e.size()).sum(); + + for era in &self.eras { + let era_proportion = era.size() as f64 / total_blocks as f64; + let era_samples = (total_samples as f64 * era_proportion).round() as usize; + + let samples = self.random_sample_from_range( + era.start, + era.end, + era_samples.min(era.size() as usize), + ); + all_samples.extend(samples); + } + + all_samples.into_iter().collect() + } + + fn hybrid_recent( + &self, + base_per_era: usize, + recent_additional: usize, + recent_window: u64, + ) -> Vec { + let mut all_samples = BTreeSet::new(); + + // Base samples from each era + for era in &self.eras { + let samples = self.random_sample_from_range( + era.start, + era.end, + base_per_era.min(era.size() as usize), + ); + all_samples.extend(samples); + } + + // Additional samples from recent blocks + if let Some(last_era) = self.eras.last() { + let recent_start = last_era.end.saturating_sub(recent_window); + let samples = + self.random_sample_from_range(recent_start, last_era.end, recent_additional); + all_samples.extend(samples); + } + + all_samples.into_iter().collect() + } + + fn fixed_density(&self, every_n: u64) -> Vec { + let mut samples = Vec::new(); + + for era in &self.eras { + let mut height = era.start; + while height <= era.end { + samples.push(height); + height += every_n; + } + } + + samples + } + + fn weighted(&self, weights: &[(String, f64)], total_samples: usize) -> Vec { + let mut all_samples = BTreeSet::new(); + let weight_sum: f64 = weights.iter().map(|(_, w)| w).sum(); + + for (era_name, weight) in weights { + if let Some(era) = self.eras.iter().find(|e| &e.name == era_name) { + let era_samples = ((total_samples as f64) * (weight / weight_sum)).round() as usize; + let samples = self.random_sample_from_range( + era.start, + era.end, + era_samples.min(era.size() as usize), + ); + all_samples.extend(samples); + } + } + + all_samples.into_iter().collect() + } + + fn random_sample_from_range(&self, start: u64, end: u64, sample_size: usize) -> Vec { + let range_size = (end - start + 1) as usize; + + if sample_size >= range_size { + // If we want more samples than available, just return all + return (start..=end).collect(); + } + + let mut rng = if let Some(seed) = self.seed { + rand::rngs::StdRng::seed_from_u64(seed) + } else { + rand::rngs::StdRng::from_entropy() + }; + + let all_heights: Vec = (start..=end).collect(); + let mut samples: Vec = all_heights + .choose_multiple(&mut rng, sample_size) + .cloned() + .collect(); + + samples.sort(); + samples + } + + pub fn describe(&self) -> String { + let samples = self.generate_samples(); + let total = samples.len(); + + let mut description = format!("Sampling Strategy: {:?}\n", self.strategy); + description.push_str(&format!("Total samples: {}\n\n", total)); + description.push_str("Distribution by era:\n"); + + for era in &self.eras { + let era_samples = samples + .iter() + .filter(|&&h| h >= era.start && h <= era.end) + .count(); + let percentage = (era_samples as f64 / total as f64) * 100.0; + let density = era_samples as f64 / era.size() as f64; + + description.push_str(&format!( + " {}: {} samples ({:.1}% of total, 1 in {:.0} blocks)\n", + era.name, + era_samples, + percentage, + 1.0 / density + )); + } + + description + } +} + +// Example usage in main +pub fn create_recommended_sampler(current_tip: u64) -> Sampler { + // Hybrid approach: 750 samples per era + 2000 from recent 100k blocks + Sampler::new( + SamplingStrategy::HybridRecent { + base_per_era: 750, + recent_additional: 2000, + recent_window: 100_000, + }, + current_tip, + Some(42), // Fixed seed for reproducibility + ) +} + +pub fn create_quick_sampler(current_tip: u64) -> Sampler { + // Quick analysis: fewer samples + Sampler::new( + SamplingStrategy::HybridRecent { + base_per_era: 250, + recent_additional: 500, + recent_window: 50_000, + }, + current_tip, + Some(42), + ) +} + +pub fn create_thorough_sampler(current_tip: u64) -> Sampler { + // Thorough analysis: more samples + Sampler::new( + SamplingStrategy::HybridRecent { + base_per_era: 1500, + recent_additional: 5000, + recent_window: 200_000, + }, + current_tip, + Some(42), + ) +} + +pub fn create_equal_sampler(current_tip: u64) -> Sampler { + // Equal samples per era + Sampler::new( + SamplingStrategy::EqualPerEra { + samples_per_era: 1000, + }, + current_tip, + Some(42), + ) +} + +pub fn create_proportional_sampler(current_tip: u64) -> Sampler { + // Proportional to era size + Sampler::new( + SamplingStrategy::Proportional { + total_samples: 5000, + }, + current_tip, + Some(42), + ) +} + +pub fn create_weighted_sampler(current_tip: u64) -> Sampler { + // Custom weights - focus on recent + Sampler::new( + SamplingStrategy::Weighted { + weights: vec![ + ("sapling".to_string(), 0.10), + ("blossom".to_string(), 0.10), + ("heartwood".to_string(), 0.10), + ("canopy".to_string(), 0.20), + ("nu5".to_string(), 0.30), + ("nu6".to_string(), 0.20), + ], + total_samples: 5000, + }, + current_tip, + Some(42), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_equal_per_era() { + // Use a tip that's well into NU6 era + let sampler = create_equal_sampler(2_900_000); + let samples = sampler.generate_samples(); + + // Should have roughly 6000 samples (1000 per era × 6 eras) + assert!(samples.len() >= 5500 && samples.len() <= 6500); + + // Samples should be sorted + assert!(samples.windows(2).all(|w| w[0] < w[1])); + } + + #[test] + fn test_hybrid_recent() { + let sampler = create_recommended_sampler(2_900_000); + let samples = sampler.generate_samples(); + + println!("{}", sampler.describe()); + + // Base: 750 × 6 eras = 4500 + // Recent: 2000 additional + // Total could be up to 6500 (with some overlap deduplication via BTreeSet) + assert!(samples.len() >= 4000 && samples.len() <= 7000); + } + + #[test] + fn test_reproducibility() { + let sampler1 = create_equal_sampler(2_900_000); + let sampler2 = create_equal_sampler(2_900_000); + + let samples1 = sampler1.generate_samples(); + let samples2 = sampler2.generate_samples(); + + // Same seed should produce identical samples + assert_eq!(samples1, samples2); + } +} diff --git a/build.rs b/build.rs deleted file mode 100644 index 8b51f54..0000000 --- a/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -fn main() -> Result<(), Box> { - tonic_build::configure() - .build_server(false) // We only need the client - .compile( - &["proto/service.proto", "proto/compact_formats.proto"], - &["proto/"], - )?; - Ok(()) -} diff --git a/docs/analysis/bandwidth_impact.png b/docs/analysis/bandwidth_impact.png new file mode 100644 index 0000000..5cd747b Binary files /dev/null and b/docs/analysis/bandwidth_impact.png differ diff --git a/docs/analysis/by_era.png b/docs/analysis/by_era.png new file mode 100644 index 0000000..c453cd4 Binary files /dev/null and b/docs/analysis/by_era.png differ diff --git a/docs/analysis/correlations.png b/docs/analysis/correlations.png new file mode 100644 index 0000000..1e9c4d1 Binary files /dev/null and b/docs/analysis/correlations.png differ diff --git a/docs/analysis/cumulative.png b/docs/analysis/cumulative.png new file mode 100644 index 0000000..101bd66 Binary files /dev/null and b/docs/analysis/cumulative.png differ diff --git a/docs/analysis/distribution.png b/docs/analysis/distribution.png new file mode 100644 index 0000000..edd7221 Binary files /dev/null and b/docs/analysis/distribution.png differ diff --git a/docs/analysis/heatmap.png b/docs/analysis/heatmap.png new file mode 100644 index 0000000..ed4a33e Binary files /dev/null and b/docs/analysis/heatmap.png differ diff --git a/docs/analysis/statistical_report.md b/docs/analysis/statistical_report.md new file mode 100644 index 0000000..a220f7c --- /dev/null +++ b/docs/analysis/statistical_report.md @@ -0,0 +1,88 @@ +# Statistical Analysis Report + +**Zcash Compact Block Transparent Data Overhead** + +--- + +## Summary Statistics + +- **Total blocks analyzed:** 13,984 +- **Block height range:** 419,402 - 3,098,051 + +## Overhead Percentage + +| Metric | Value | +|--------|-------| +| Mean | 980.50% | +| Median | 173.81% | +| Std Dev| 6706.43% | +| Min | 0.08% | +| Max | 473783.33% | + +## Percentiles + +| Percentile | Overhead | +|------------|----------| +| P25 | 159.34% | +| P50 | 173.81% | +| P75 | 452.75% | +| P90 | 1657.47% | +| P95 | 3719.88% | +| P99 | 12968.45% | + +### Cumulative + +![Cumulative](docs/analysis/cumulative.png) + +### Distribution +![Distribution](docs/analysis/distribution.png) + +## Confidence Intervals (95%) + +- **Mean overhead:** 980.50% ± 111.16% +- **Range:** [869.34%, 1091.67%] + +## Statistics by Era + +![Statistics by era](docs/analysis/by_era.png) +![heatmap by era](docs/analysis/heatmap.png) + +| Era | Count | Mean | Std Dev | Median | Min | Max | +|-----|-------|------|---------|--------|-----|-----| +| Sapling | 1,500 | 3097.70% | 17510.41% | 540.90% | 18.55% | 473783.33% | +| Blossom | 1,500 | 1215.30% | 7770.75% | 303.57% | 1.58% | 281783.33% | +| Heartwood | 1,500 | 1435.15% | 2985.99% | 391.07% | 20.00% | 33155.95% | +| Canopy | 1,500 | 1777.54% | 4114.49% | 422.02% | 21.13% | 61611.90% | +| Nu5 | 1,500 | 633.78% | 4192.93% | 232.22% | 0.08% | 139238.89% | +| Nu6 | 6,484 | 227.05% | 518.88% | 159.34% | 0.65% | 19238.61% | + +## Practical Bandwidth Impact + +### Average Block Sizes + +- **Current:** 2.13 KB +- **With transparent:** 3.75 KB +- **Delta:** 1.62 KB + +### Daily Sync (~1152 blocks) + +- **Current:** 2.46 MB +- **With transparent:** 4.32 MB +- **Additional:** 1.86 MB (75.9%) + +![bandwidth impact](docs/analysis/bandwidth_impact.png) +## Correlations + +![correlations](docs/analysis/correlations.png) + +| Variables | Correlation (r) | +|-----------|----------------| +| Transparent inputs → delta bytes | 0.941 | +| Transparent outputs → delta bytes | 0.434 | +| Transaction count → overhead % | 0.136 | + +## Decision Framework + +- **Median overhead:** 173.8% +- **95th percentile:** 3719.9% + diff --git a/docs/analysis/time_series.png b/docs/analysis/time_series.png new file mode 100644 index 0000000..b80045e Binary files /dev/null and b/docs/analysis/time_series.png differ diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..c22c6f0 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,58 @@ +# Architecture + +## System Overview + +``` +┌─────────────┐ ┌──────────────┐ +│ Lightwalletd│◄────────┤ Analyzer │ +│ (gRPC) │ │ (Rust) │ +└─────────────┘ └──────┬───────┘ + │ +┌─────────────┐ │ +│ Zebrad │◄───────────────┘ +│ (JSON-RPC) │ +└─────────────┘ + + │ + ├─────► CSV Output + │ + ▼ +┌──────────────┐ +│ Visualizer │ +│ (Python) │ +└──────┬───────┘ + │ + ├─────► Charts (PNG) + └─────► Statistical Report (TXT) +``` + +## Data Flow + +1. **Sampling**: Generate block heights to analyze +2. **Fetch Real Data**: Get CompactBlock from lightwalletd +3. **Fetch Full Data**: Get full block from Zebrad +4. **Estimate**: Calculate transparent overhead +5. **Output**: Write to CSV +6. **Visualize**: Generate charts and statistics + +## Key Components + +### Sampling Module +- Multiple strategies (equal, proportional, hybrid, etc.) +- Era-aware sampling +- Reproducible with fixed seeds + +### Estimator Module +- Protobuf size calculations +- Transparent I/O overhead estimation +- Accurate field-level sizing + +### RPC Clients +- gRPC client for lightwalletd +- JSON-RPC client for Zebrad +- Error handling and retries + +### Visualizer +- Statistical analysis +- Chart generation +- Decision recommendations diff --git a/docs/sampling-strategies.md b/docs/sampling-strategies.md new file mode 100644 index 0000000..e67f78b --- /dev/null +++ b/docs/sampling-strategies.md @@ -0,0 +1,53 @@ +# Sampling Strategies + +## Overview + +Analyzing 2.7M+ blocks would take days. Statistical sampling provides accurate results in minutes. + +## Available Strategies + +### 1. Quick (~1,500 blocks, 15 min) +- 250 per era +- 500 additional recent samples +- Good for initial exploration + +### 2. Recommended (~5,000 blocks, 30 min) +- 750 per era +- 2,000 additional recent samples +- Best balance of accuracy and speed +- **Use this for most analyses** + +### 3. Thorough (~11,000 blocks, 2 hr) +- 1,500 per era +- 5,000 additional recent samples +- Highest confidence intervals +- Use for final analysis + +### 4. Equal (~4,000 blocks) +- 1,000 samples per era +- Good for comparing eras +- May over-represent old eras + +### 5. Proportional (~5,000 blocks) +- Samples match blockchain distribution +- Statistically representative +- Gives equal weight to old and new + +### 6. Weighted (~5,000 blocks) +- Custom weights per era +- Default: heavier on recent +- Flexible for specific needs + +## Choosing a Strategy + +**For protocol decisions**: Use Recommended or Thorough +**For era comparison**: Use Equal +**For quick validation**: Use Quick +**For custom needs**: Use Weighted + +## Sample Size Calculations + +For 95% confidence and ±2% margin of error: +n = (1.96² × 0.5 × 0.5) / 0.02² ≈ 2,401 + +We use 5,000+ samples for better confidence. diff --git a/examples/era_comparison.sh b/examples/era_comparison.sh new file mode 100755 index 0000000..eef1280 --- /dev/null +++ b/examples/era_comparison.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Compare overhead across different eras + +cd "$(dirname "$0")/.." +ANALYZER="./analyzer/target/release/compact-block-analyzer" +OUTPUT="results/era_comparison_$(date +%Y%m%d)" +mkdir -p "$OUTPUT" + +echo "🔬 Running era comparison analysis..." + +# Sample each era +echo "📊 Analyzing Sapling era..." +$ANALYZER http://127.0.0.1:9067 http://127.0.0.1:8232 range 500000 500500 "$OUTPUT/sapling.csv" + +echo "📊 Analyzing Blossom era..." +$ANALYZER http://127.0.0.1:9067 http://127.0.0.1:8232 range 750000 750500 "$OUTPUT/blossom.csv" + +echo "📊 Analyzing Canopy era..." +$ANALYZER http://127.0.0.1:9067 http://127.0.0.1:8232 range 1300000 1300500 "$OUTPUT/canopy.csv" + +echo "📊 Analyzing NU5 era..." +$ANALYZER http://127.0.0.1:9067 http://127.0.0.1:8232 range 2000000 2000500 "$OUTPUT/nu5.csv" + +echo "📊 Analyzing NU6 era..." +$ANALYZER http://127.0.0.1:9067 http://127.0.0.1:8232 range 2800000 2800500 "$OUTPUT/nu6.csv" + +# Visualize each +echo "📈 Generating visualizations..." +cd visualization +source venv/bin/activate +for csv in "../$OUTPUT"/*.csv; do + basename="${csv%.csv}" + python visualize.py "$csv" -o "${basename}_charts" +done +cd .. + +echo "✅ Era comparison complete: $OUTPUT" diff --git a/examples/quick_analysis.sh b/examples/quick_analysis.sh new file mode 100755 index 0000000..129c6ba --- /dev/null +++ b/examples/quick_analysis.sh @@ -0,0 +1,7 @@ +#!/bin/bash +cd "$(dirname "$0")/.." +./analyzer/target/release/compact-block-analyzer \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + quick \ + results/quick_$(date +%Y%m%d).csv diff --git a/examples/recommended_analysis.sh b/examples/recommended_analysis.sh new file mode 100755 index 0000000..f45e0a4 --- /dev/null +++ b/examples/recommended_analysis.sh @@ -0,0 +1,7 @@ +#!/bin/bash +cd "$(dirname "$0")/.." +./analyzer/target/release/compact-block-analyzer \ + http://127.0.0.1:9067 \ + http://127.0.0.1:8232 \ + recommended \ + results/recommended_$(date +%Y%m%d).csv diff --git a/proto/compact_formats.proto b/proto/compact_formats.proto deleted file mode 100644 index 09df06d..0000000 --- a/proto/compact_formats.proto +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright (c) 2019-2021 The Zcash developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or https://www.opensource.org/licenses/mit-license.php . - -syntax = "proto3"; -package cash.z.wallet.sdk.rpc; -option go_package = "lightwalletd/walletrpc"; -option swift_prefix = ""; - -// Remember that proto3 fields are all optional. A field that is not present will be set to its zero value. -// bytes fields of hashes are in canonical little-endian format. - -// ChainMetadata represents information about the state of the chain as of a given block. -message ChainMetadata { - uint32 saplingCommitmentTreeSize = 1; // the size of the Sapling note commitment tree as of the end of this block - uint32 orchardCommitmentTreeSize = 2; // the size of the Orchard note commitment tree as of the end of this block -} - -// CompactBlock is a packaging of ONLY the data from a block that's needed to: -// 1. Detect a payment to your shielded Sapling address -// 2. Detect a spend of your shielded Sapling notes -// 3. Update your witnesses to generate new Sapling spend proofs. -message CompactBlock { - uint32 protoVersion = 1; // the version of this wire format, for storage - uint64 height = 2; // the height of this block - bytes hash = 3; // the ID (hash) of this block, same as in block explorers - bytes prevHash = 4; // the ID (hash) of this block's predecessor - uint32 time = 5; // Unix epoch time when the block was mined - bytes header = 6; // (hash, prevHash, and time) OR (full header) - repeated CompactTx vtx = 7; // zero or more compact transactions from this block - ChainMetadata chainMetadata = 8; // information about the state of the chain as of this block -} - -// CompactTx contains the minimum information for a wallet to know if this transaction -// is relevant to it (either pays to it or spends from it) via shielded elements -// only. This message will not encode a transparent-to-transparent transaction. -message CompactTx { - // Index and hash will allow the receiver to call out to chain - // explorers or other data structures to retrieve more information - // about this transaction. - uint64 index = 1; // the index within the full block - bytes hash = 2; // the ID (hash) of this transaction, same as in block explorers - - // The transaction fee: present if server can provide. In the case of a - // stateless server and a transaction with transparent inputs, this will be - // unset because the calculation requires reference to prior transactions. - // If there are no transparent inputs, the fee will be calculable as: - // valueBalanceSapling + valueBalanceOrchard + sum(vPubNew) - sum(vPubOld) - sum(tOut) - uint32 fee = 3; - - repeated CompactSaplingSpend spends = 4; - repeated CompactSaplingOutput outputs = 5; - repeated CompactOrchardAction actions = 6; -} - -// CompactSaplingSpend is a Sapling Spend Description as described in 7.3 of the Zcash -// protocol specification. -message CompactSaplingSpend { - bytes nf = 1; // nullifier (see the Zcash protocol specification) -} - -// output encodes the `cmu` field, `ephemeralKey` field, and a 52-byte prefix of the -// `encCiphertext` field of a Sapling Output Description. These fields are described in -// section 7.4 of the Zcash protocol spec: -// https://zips.z.cash/protocol/protocol.pdf#outputencodingandconsensus -// Total size is 116 bytes. -message CompactSaplingOutput { - bytes cmu = 1; // note commitment u-coordinate - bytes ephemeralKey = 2; // ephemeral public key - bytes ciphertext = 3; // first 52 bytes of ciphertext -} - -// https://github.com/zcash/zips/blob/main/zip-0225.rst#orchard-action-description-orchardaction -// (but not all fields are needed) -message CompactOrchardAction { - bytes nullifier = 1; // [32] The nullifier of the input note - bytes cmx = 2; // [32] The x-coordinate of the note commitment for the output note - bytes ephemeralKey = 3; // [32] An encoding of an ephemeral Pallas public key - bytes ciphertext = 4; // [52] The first 52 bytes of the encCiphertext field -} diff --git a/proto/darkside.proto b/proto/darkside.proto deleted file mode 100644 index 02a8855..0000000 --- a/proto/darkside.proto +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) 2019-2020 The Zcash developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or https://www.opensource.org/licenses/mit-license.php . - -syntax = "proto3"; -package cash.z.wallet.sdk.rpc; -option go_package = "lightwalletd/walletrpc"; -option swift_prefix = ""; -import "service.proto"; - -message DarksideMetaState { - int32 saplingActivation = 1; - string branchID = 2; - string chainName = 3; - uint32 startSaplingCommitmentTreeSize = 4; - uint32 startOrchardCommitmentTreeSize = 5; -} - -// A block is a hex-encoded string. -message DarksideBlock { - string block = 1; -} - -// DarksideBlocksURL is typically something like: -// https://raw.githubusercontent.com/zcash-hackworks/darksidewalletd-test-data/master/basic-reorg/before-reorg.txt -message DarksideBlocksURL { - string url = 1; -} - -// DarksideTransactionsURL refers to an HTTP source that contains a list -// of hex-encoded transactions, one per line, that are to be associated -// with the given height (fake-mined into the block at that height) -message DarksideTransactionsURL { - int32 height = 1; - string url = 2; -} - -message DarksideHeight { - int32 height = 1; -} - -message DarksideEmptyBlocks { - int32 height = 1; - int32 nonce = 2; - int32 count = 3; -} - -message DarksideSubtreeRoots { - ShieldedProtocol shieldedProtocol = 1; - uint32 startIndex = 2; - repeated SubtreeRoot subtreeRoots = 3; -} - -// Darksidewalletd maintains two staging areas, blocks and transactions. The -// Stage*() gRPCs add items to the staging area; ApplyStaged() "applies" everything -// in the staging area to the working (operational) state that the mock zcashd -// serves; transactions are placed into their corresponding blocks (by height). -service DarksideStreamer { - // Reset reverts all darksidewalletd state (active block range, latest height, - // staged blocks and transactions) and lightwalletd state (cache) to empty, - // the same as the initial state. This occurs synchronously and instantaneously; - // no reorg happens in lightwalletd. This is good to do before each independent - // test so that no state leaks from one test to another. - // Also sets (some of) the values returned by GetLightdInfo(). The Sapling - // activation height specified here must be where the block range starts. - rpc Reset(DarksideMetaState) returns (Empty) {} - - // StageBlocksStream accepts a list of blocks and saves them into the blocks - // staging area until ApplyStaged() is called; there is no immediate effect on - // the mock zcashd. Blocks are hex-encoded. Order is important, see ApplyStaged. - rpc StageBlocksStream(stream DarksideBlock) returns (Empty) {} - - // StageBlocks is the same as StageBlocksStream() except the blocks are fetched - // from the given URL. Blocks are one per line, hex-encoded (not JSON). - rpc StageBlocks(DarksideBlocksURL) returns (Empty) {} - - // StageBlocksCreate is like the previous two, except it creates 'count' - // empty blocks at consecutive heights starting at height 'height'. The - // 'nonce' is part of the header, so it contributes to the block hash; this - // lets you create identical blocks (same transactions and height), but with - // different hashes. - rpc StageBlocksCreate(DarksideEmptyBlocks) returns (Empty) {} - - // StageTransactionsStream stores the given transaction-height pairs in the - // staging area until ApplyStaged() is called. Note that these transactions - // are not returned by the production GetTransaction() gRPC until they - // appear in a "mined" block (contained in the active blockchain presented - // by the mock zcashd). - rpc StageTransactionsStream(stream RawTransaction) returns (Empty) {} - - // StageTransactions is the same except the transactions are fetched from - // the given url. They are all staged into the block at the given height. - // Staging transactions to different heights requires multiple calls. - rpc StageTransactions(DarksideTransactionsURL) returns (Empty) {} - - // ApplyStaged iterates the list of blocks that were staged by the - // StageBlocks*() gRPCs, in the order they were staged, and "merges" each - // into the active, working blocks list that the mock zcashd is presenting - // to lightwalletd. Even as each block is applied, the active list can't - // have gaps; if the active block range is 1000-1006, and the staged block - // range is 1003-1004, the resulting range is 1000-1004, with 1000-1002 - // unchanged, blocks 1003-1004 from the new range, and 1005-1006 dropped. - // - // After merging all blocks, ApplyStaged() appends staged transactions (in - // the order received) into each one's corresponding (by height) block - // The staging area is then cleared. - // - // The argument specifies the latest block height that mock zcashd reports - // (i.e. what's returned by GetLatestBlock). Note that ApplyStaged() can - // also be used to simply advance the latest block height presented by mock - // zcashd. That is, there doesn't need to be anything in the staging area. - rpc ApplyStaged(DarksideHeight) returns (Empty) {} - - // Calls to the production gRPC SendTransaction() store the transaction in - // a separate area (not the staging area); this method returns all transactions - // in this separate area, which is then cleared. The height returned - // with each transaction is -1 (invalid) since these transactions haven't - // been mined yet. The intention is that the transactions returned here can - // then, for example, be given to StageTransactions() to get them "mined" - // into a specified block on the next ApplyStaged(). - rpc GetIncomingTransactions(Empty) returns (stream RawTransaction) {} - - // Clear the incoming transaction pool. - rpc ClearIncomingTransactions(Empty) returns (Empty) {} - - // Add a GetAddressUtxosReply entry to be returned by GetAddressUtxos(). - // There is no staging or applying for these, very simple. - rpc AddAddressUtxo(GetAddressUtxosReply) returns (Empty) {} - - // Clear the list of GetAddressUtxos entries (can't fail) - rpc ClearAddressUtxo(Empty) returns (Empty) {} - - // Adds a GetTreeState to the tree state cache - rpc AddTreeState(TreeState) returns (Empty) {} - - // Removes a GetTreeState for the given height from cache if present (can't fail) - rpc RemoveTreeState(BlockID) returns (Empty) {} - - // Clear the list of GetTreeStates entries (can't fail) - rpc ClearAllTreeStates(Empty) returns (Empty) {} - - // Sets the subtree roots cache (for GetSubtreeRoots), - // replacing any existing entries - rpc SetSubtreeRoots(DarksideSubtreeRoots) returns (Empty) {} - - // Stop causes the server to shut down cleanly. - rpc Stop(Empty) returns (Empty) {} -} diff --git a/proto/service.proto b/proto/service.proto deleted file mode 100644 index 669a62d..0000000 --- a/proto/service.proto +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright (c) 2019-2020 The Zcash developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or https://www.opensource.org/licenses/mit-license.php . - -syntax = "proto3"; -package cash.z.wallet.sdk.rpc; -option go_package = "lightwalletd/walletrpc"; -option swift_prefix = ""; -import "compact_formats.proto"; - -// A BlockID message contains identifiers to select a block: a height or a -// hash. Specification by hash is not implemented, but may be in the future. -message BlockID { - uint64 height = 1; - bytes hash = 2; -} - -// BlockRange specifies a series of blocks from start to end inclusive. -// Both BlockIDs must be heights; specification by hash is not yet supported. -message BlockRange { - BlockID start = 1; - BlockID end = 2; -} - -// A TxFilter contains the information needed to identify a particular -// transaction: either a block and an index, or a direct transaction hash. -// Currently, only specification by hash is supported. -message TxFilter { - BlockID block = 1; // block identifier, height or hash - uint64 index = 2; // index within the block - bytes hash = 3; // transaction ID (hash, txid) -} - -// RawTransaction contains the complete transaction data. It also includes the -// height for the block in which the transaction was included in the main -// chain, if any (as detailed below). -message RawTransaction { - // The serialized representation of the Zcash transaction. - bytes data = 1; - // The height at which the transaction is mined, or a sentinel value. - // - // Due to an error in the original protobuf definition, it is necessary to - // reinterpret the result of the `getrawtransaction` RPC call. Zcashd will - // return the int64 value `-1` for the height of transactions that appear - // in the block index, but which are not mined in the main chain. Here, the - // height field of `RawTransaction` was erroneously created as a `uint64`, - // and as such we must map the response from the zcashd RPC API to be - // representable within this space. Additionally, the `height` field will - // be absent for transactions in the mempool, resulting in the default - // value of `0` being set. Therefore, the meanings of the `height` field of - // the `RawTransaction` type are as follows: - // - // * height 0: the transaction is in the mempool - // * height 0xffffffffffffffff: the transaction has been mined on a fork that - // is not currently the main chain - // * any other height: the transaction has been mined in the main chain at the - // given height - uint64 height = 2; -} - -// A SendResponse encodes an error code and a string. It is currently used -// only by SendTransaction(). If error code is zero, the operation was -// successful; if non-zero, it and the message specify the failure. -message SendResponse { - int32 errorCode = 1; - string errorMessage = 2; -} - -// Chainspec is a placeholder to allow specification of a particular chain fork. -message ChainSpec {} - -// Empty is for gRPCs that take no arguments, currently only GetLightdInfo. -message Empty {} - -// LightdInfo returns various information about this lightwalletd instance -// and the state of the blockchain. -message LightdInfo { - string version = 1; - string vendor = 2; - bool taddrSupport = 3; // true - string chainName = 4; // either "main" or "test" - uint64 saplingActivationHeight = 5; // depends on mainnet or testnet - string consensusBranchId = 6; // protocol identifier, see consensus/upgrades.cpp - uint64 blockHeight = 7; // latest block on the best chain - string gitCommit = 8; - string branch = 9; - string buildDate = 10; - string buildUser = 11; - uint64 estimatedHeight = 12; // less than tip height if zcashd is syncing - string zcashdBuild = 13; // example: "v4.1.1-877212414" - string zcashdSubversion = 14; // example: "/MagicBean:4.1.1/" - string donationAddress = 15; // Zcash donation UA address -} - -// TransparentAddressBlockFilter restricts the results to the given address -// or block range. -message TransparentAddressBlockFilter { - string address = 1; // t-address - BlockRange range = 2; // start, end heights -} - -// Duration is currently used only for testing, so that the Ping rpc -// can simulate a delay, to create many simultaneous connections. Units -// are microseconds. -message Duration { - int64 intervalUs = 1; -} - -// PingResponse is used to indicate concurrency, how many Ping rpcs -// are executing upon entry and upon exit (after the delay). -// This rpc is used for testing only. -message PingResponse { - int64 entry = 1; - int64 exit = 2; -} - -message Address { - string address = 1; -} -message AddressList { - repeated string addresses = 1; -} -message Balance { - int64 valueZat = 1; -} - -// The a shortened transaction ID is the prefix in big-endian (hex) format -// (then converted to binary). -message Exclude { - repeated bytes txid = 1; -} - -// The TreeState is derived from the Zcash z_gettreestate rpc. -message TreeState { - string network = 1; // "main" or "test" - uint64 height = 2; // block height - string hash = 3; // block id - uint32 time = 4; // Unix epoch time when the block was mined - string saplingTree = 5; // sapling commitment tree state - string orchardTree = 6; // orchard commitment tree state -} - -enum ShieldedProtocol { - sapling = 0; - orchard = 1; -} - -message GetSubtreeRootsArg { - uint32 startIndex = 1; // Index identifying where to start returning subtree roots - ShieldedProtocol shieldedProtocol = 2; // Shielded protocol to return subtree roots for - uint32 maxEntries = 3; // Maximum number of entries to return, or 0 for all entries. -} -message SubtreeRoot { - bytes rootHash = 2; // The 32-byte Merkle root of the subtree. - bytes completingBlockHash = 3; // The hash of the block that completed this subtree. - uint64 completingBlockHeight = 4; // The height of the block that completed this subtree in the main chain. -} - -// Results are sorted by height, which makes it easy to issue another -// request that picks up from where the previous left off. -message GetAddressUtxosArg { - repeated string addresses = 1; - uint64 startHeight = 2; - uint32 maxEntries = 3; // zero means unlimited -} -message GetAddressUtxosReply { - string address = 6; - bytes txid = 1; - int32 index = 2; - bytes script = 3; - int64 valueZat = 4; - uint64 height = 5; -} -message GetAddressUtxosReplyList { - repeated GetAddressUtxosReply addressUtxos = 1; -} - -service CompactTxStreamer { - // Return the height of the tip of the best chain - rpc GetLatestBlock(ChainSpec) returns (BlockID) {} - // Return the compact block corresponding to the given block identifier - rpc GetBlock(BlockID) returns (CompactBlock) {} - // Same as GetBlock except actions contain only nullifiers - rpc GetBlockNullifiers(BlockID) returns (CompactBlock) {} - // Return a list of consecutive compact blocks - rpc GetBlockRange(BlockRange) returns (stream CompactBlock) {} - // Same as GetBlockRange except actions contain only nullifiers - rpc GetBlockRangeNullifiers(BlockRange) returns (stream CompactBlock) {} - - // Return the requested full (not compact) transaction (as from zcashd) - rpc GetTransaction(TxFilter) returns (RawTransaction) {} - // Submit the given transaction to the Zcash network - rpc SendTransaction(RawTransaction) returns (SendResponse) {} - - // Return the transactions corresponding to the given t-address within the given block range - // NB - this method is misnamed, it returns transactions, not transaction IDs. - // NOTE: this method is deprecated, please use GetTaddressTransactions instead. - rpc GetTaddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {} - - // Return the transactions corresponding to the given t-address within the given block range - rpc GetTaddressTransactions(TransparentAddressBlockFilter) returns (stream RawTransaction) {} - - rpc GetTaddressBalance(AddressList) returns (Balance) {} - rpc GetTaddressBalanceStream(stream Address) returns (Balance) {} - - // Return the compact transactions currently in the mempool; the results - // can be a few seconds out of date. If the Exclude list is empty, return - // all transactions; otherwise return all *except* those in the Exclude list - // (if any); this allows the client to avoid receiving transactions that it - // already has (from an earlier call to this rpc). The transaction IDs in the - // Exclude list can be shortened to any number of bytes to make the request - // more bandwidth-efficient; if two or more transactions in the mempool - // match a shortened txid, they are all sent (none is excluded). Transactions - // in the exclude list that don't exist in the mempool are ignored. - // - // The a shortened transaction ID is the prefix in big-endian (hex) format - // (then converted to binary). See smoke-test.bash for examples. - rpc GetMempoolTx(Exclude) returns (stream CompactTx) {} - - // Return a stream of current Mempool transactions. This will keep the output stream open while - // there are mempool transactions. It will close the returned stream when a new block is mined. - rpc GetMempoolStream(Empty) returns (stream RawTransaction) {} - - // GetTreeState returns the note commitment tree state corresponding to the given block. - // See section 3.7 of the Zcash protocol specification. It returns several other useful - // values also (even though they can be obtained using GetBlock). - // The block can be specified by either height or hash. - rpc GetTreeState(BlockID) returns (TreeState) {} - rpc GetLatestTreeState(Empty) returns (TreeState) {} - - // Returns a stream of information about roots of subtrees of the note commitment tree - // for the specified shielded protocol (Sapling or Orchard). - rpc GetSubtreeRoots(GetSubtreeRootsArg) returns (stream SubtreeRoot) {} - - rpc GetAddressUtxos(GetAddressUtxosArg) returns (GetAddressUtxosReplyList) {} - rpc GetAddressUtxosStream(GetAddressUtxosArg) returns (stream GetAddressUtxosReply) {} - - // Return information about this lightwalletd instance and the blockchain - rpc GetLightdInfo(Empty) returns (LightdInfo) {} - // Testing-only, requires lightwalletd --ping-very-insecure (do not enable in production) - rpc Ping(Duration) returns (PingResponse) {} -} diff --git a/results/.gitkeep b/results/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/results/README.md b/results/README.md new file mode 100644 index 0000000..a02567a --- /dev/null +++ b/results/README.md @@ -0,0 +1,24 @@ +# Results Directory + +This directory contains analysis outputs. Each run creates a timestamped subdirectory: + +``` +results/ +├── 20250108_143022/ +│ ├── recommended.csv # Raw data +│ ├── charts/ # Generated visualizations +│ │ ├── distribution.png +│ │ ├── time_series.png +│ │ ├── by_era.png +│ │ ├── correlations.png +│ │ ├── cumulative.png +│ │ ├── bandwidth_impact.png +│ │ ├── heatmap.png +│ │ └── statistical_report.txt +│ └── ... +└── 20250108_150314/ + └── ... +``` + +Results are gitignored to avoid bloating the repository. +To save results permanently, copy them elsewhere or commit specific files. diff --git a/scripts/fetch_protos.sh b/scripts/fetch_protos.sh new file mode 100755 index 0000000..655d05b --- /dev/null +++ b/scripts/fetch_protos.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +PROTO_DIR="analyzer/proto" +mkdir -p "$PROTO_DIR" + +echo "📥 Fetching protocol buffers from lightwallet-protocol (main branch)..." + +# Clone main branch +TEMP_DIR=$(mktemp -d) +git clone https://github.com/zcash/lightwallet-protocol.git "$TEMP_DIR" + +echo "On branch: $(git -C "$TEMP_DIR" branch --show-current)" + +# Copy proto files from walletrpc directory (using absolute paths) +cp "$TEMP_DIR/walletrpc/compact_formats.proto" "$PROTO_DIR/" +cp "$TEMP_DIR/walletrpc/service.proto" "$PROTO_DIR/" + +# Cleanup +rm -rf "$TEMP_DIR" + +echo "✅ Proto files updated in $PROTO_DIR (main branch)" +ls -la "$PROTO_DIR" diff --git a/scripts/run_all_analyses.sh b/scripts/run_all_analyses.sh new file mode 100755 index 0000000..8382b48 --- /dev/null +++ b/scripts/run_all_analyses.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +ANALYZER="./analyzer/target/release/compact-block-analyzer" +LIGHTWALLETD=${1:-"http://127.0.0.1:9067"} +ZEBRAD=${2:-"http://127.0.0.1:8232"} +OUTPUT_DIR="results/$(date +%Y%m%d_%H%M%S)" + +mkdir -p "$OUTPUT_DIR" + +echo "🔬 Running comprehensive analysis suite..." +echo "📁 Output directory: $OUTPUT_DIR" +echo "" + +# Quick analysis +echo "⚡ Running quick analysis..." +$ANALYZER "$LIGHTWALLETD" "$ZEBRAD" quick "$OUTPUT_DIR/quick.csv" + +# Recommended analysis +echo "📊 Running recommended analysis..." +$ANALYZER "$LIGHTWALLETD" "$ZEBRAD" recommended "$OUTPUT_DIR/recommended.csv" + +# Generate visualizations +echo "📈 Generating visualizations..." +cd visualization +source venv/bin/activate +python visualize.py "../$OUTPUT_DIR/recommended.csv" -o "../$OUTPUT_DIR/charts" +cd .. + +echo "" +echo "✅ Analysis complete!" +echo "📊 Results in: $OUTPUT_DIR" +echo "📈 Charts in: $OUTPUT_DIR/charts" +echo "📄 Report: $OUTPUT_DIR/charts/statistical_report.txt" diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..98389fd --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -e + +echo "Setting up Compact Block Analyzer..." + +# Check prerequisites +command -v rustc >/dev/null 2>&1 || { echo "Rust not installed"; exit 1; } +command -v python3 >/dev/null 2>&1 || { echo "Python3 not installed"; exit 1; } + +# Fetch proto files +./scripts/fetch_protos.sh + +# Build Rust analyzer +cd analyzer +cargo build --release +cd .. + +# Setup Python environment +cd visualization +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +cd .. + +echo "✅ Setup complete!" +echo "Run: cd analyzer && cargo run --release -- --help" diff --git a/src/main.rs b/src/main.rs deleted file mode 100644 index b0f6e1b..0000000 --- a/src/main.rs +++ /dev/null @@ -1,368 +0,0 @@ -use anyhow::Result; -use serde::{Deserialize, Serialize}; - -// Include generated gRPC client code -pub mod cash { - pub mod z { - pub mod wallet { - pub mod sdk { - pub mod rpc { - tonic::include_proto!("cash.z.wallet.sdk.rpc"); - } - } - } - } -} - -use cash::z::wallet::sdk::rpc::{ - compact_tx_streamer_client::CompactTxStreamerClient, - BlockId, - BlockRange, -}; - -// Zebra RPC structures -#[derive(Debug, Deserialize)] -struct ZebraBlock { - tx: Vec, -} - -#[derive(Debug, Deserialize)] -struct ZebraTransaction { - vin: Vec, - vout: Vec, -} - -#[derive(Debug, Deserialize)] -struct ZebraVin { - txid: Option, - vout: Option, -} - - -#[derive(Debug, Deserialize)] -struct ZebraVout { - value: f64, - #[serde(rename = "scriptPubKey")] - script_pubkey: ScriptPubKey, -} - -#[derive(Debug, Deserialize)] -struct ScriptPubKey { - hex: String, -} - -#[derive(Debug, Serialize)] -struct BlockAnalysis { - height: u64, - current_compact_size: usize, - estimated_with_transparent: usize, - delta_bytes: i64, - delta_percent: f64, - tx_count: usize, - transparent_inputs: usize, - transparent_outputs: usize, -} - -struct TransparentEstimator { - zebra_rpc_url: String, - lightwalletd_url: String, - http_client: reqwest::Client, -} - -impl TransparentEstimator { - fn new(zebra_rpc_url: String, lightwalletd_url: String) -> Self { - Self { - zebra_rpc_url, - lightwalletd_url, - http_client: reqwest::Client::new(), - } - } - - async fn get_compact_block_from_lightwalletd(&self, height: u64) -> Result> { - // Connect to lightwalletd gRPC service - let mut client = CompactTxStreamerClient::connect(self.lightwalletd_url.clone()).await?; - - // Request a single block by creating a range with start=end=height - let request = tonic::Request::new(BlockRange { - start: Some(BlockId { - height: height as u64, - hash: vec![] - }), - end: Some(BlockId { - height: height as u64, - hash: vec![] - }), - }); - - // Get the block stream (will have just one block) - let mut stream = client.get_block_range(request).await?.into_inner(); - - // Get the single block from the stream - if let Some(compact_block) = stream.message().await? { - // Encode the CompactBlock to bytes to measure its actual size - use prost::Message; - let mut buf = Vec::new(); - compact_block.encode(&mut buf)?; - return Ok(buf); - } - - anyhow::bail!("Block {} not found in lightwalletd", height) - } - - async fn get_full_block_from_zebra(&self, height: u64) -> Result { - let request = serde_json::json!({ - "jsonrpc": "2.0", - "method": "getblock", - "params": [height.to_string(), 2], - "id": 1 - }); - - let response: serde_json::Value = self - .http_client - .post(&self.zebra_rpc_url) - .json(&request) - .send() - .await? - .json() - .await?; - - let block: ZebraBlock = serde_json::from_value(response["result"].clone())?; - Ok(block) - } - - fn estimate_transparent_overhead(&self, block: &ZebraBlock) -> (usize, usize, usize) { - // Estimate protobuf size for transparent data based on actual PR definitions: - // - // message OutPoint { - // bytes txid = 1; // 32 bytes - // uint32 index = 2; // varint - // } - // - // message CompactTxIn { - // OutPoint prevout = 1; // nested message - // } - // - // message TxOut { - // uint32 value = 1; // varint (note: uint32, not uint64!) - // bytes scriptPubKey = 2; // variable length - // } - // - // CompactTx gets: - // repeated CompactTxIn vin = 7; - // repeated TxOut vout = 8; - - let mut total_overhead = 0; - let mut input_count = 0; - let mut output_count = 0; - - for tx in &block.tx { - // Estimate CompactTxIn (repeated field in CompactTx) - for vin in &tx.vin { - if let (Some(txid), Some(vout_idx)) = (&vin.txid, &vin.vout) { - // Not a coinbase - - // OutPoint message size: - let mut outpoint_size = 0; - - // Field 1: bytes txid = 32 bytes - // Tag (1 byte) + length varint (1 byte) + 32 bytes - outpoint_size += 1 + 1 + 32; - - // Field 2: uint32 index (varint) - // Tag (1 byte) + varint value (1-5 bytes, typically 1-2) - outpoint_size += 1 + Self::varint_size(*vout_idx as usize); - - // CompactTxIn wraps OutPoint as field 1 - let mut compact_txin_size = 0; - // Tag for field 1 (1 byte) + length of OutPoint + OutPoint data - compact_txin_size += 1 + Self::varint_size(outpoint_size) + outpoint_size; - - // This CompactTxIn is in a repeated field (vin = 7) in CompactTx - // Tag for repeated field (1 byte) + length + message - let vin_entry_size = 1 + Self::varint_size(compact_txin_size) + compact_txin_size; - - total_overhead += vin_entry_size; - input_count += 1; - } - } - - // Estimate TxOut (repeated field in CompactTx) - for vout in &tx.vout { - let mut txout_size = 0; - - // Field 1: uint32 value (varint) - let value_zatoshis = (vout.value * 100_000_000.0) as u64; - txout_size += 1 + Self::varint_size(value_zatoshis as usize); - - // Field 2: bytes scriptPubKey - let script_len = vout.script_pubkey.hex.len() / 2; // hex to bytes - txout_size += 1 + Self::varint_size(script_len) + script_len; - - // This TxOut is in a repeated field (vout = 8) in CompactTx - // Tag for repeated field (1 byte) + length + message - let vout_entry_size = 1 + Self::varint_size(txout_size) + txout_size; - - total_overhead += vout_entry_size; - output_count += 1; - } - } - - (total_overhead, input_count, output_count) - } - - fn varint_size(value: usize) -> usize { - // Protobuf varint encoding size - match value { - 0..=127 => 1, - 128..=16383 => 2, - 16384..=2097151 => 3, - 2097152..=268435455 => 4, - _ => 5, - } - } - - async fn analyze_block(&self, height: u64) -> Result { - // Get actual compact block from lightwalletd - let compact_block_bytes = self.get_compact_block_from_lightwalletd(height).await?; - let current_size = compact_block_bytes.len(); - - // Get full block from Zebra to calculate transparent overhead - let full_block = self.get_full_block_from_zebra(height).await?; - let (transparent_overhead, input_count, output_count) = - self.estimate_transparent_overhead(&full_block); - - let estimated_size = current_size + transparent_overhead; - let delta = estimated_size as i64 - current_size as i64; - let delta_percent = if current_size > 0 { - (delta as f64 / current_size as f64) * 100.0 - } else { - 0.0 - }; - - Ok(BlockAnalysis { - height, - current_compact_size: current_size, - estimated_with_transparent: estimated_size, - delta_bytes: delta, - delta_percent, - tx_count: full_block.tx.len(), - transparent_inputs: input_count, - transparent_outputs: output_count, - }) - } - - async fn analyze_range(&self, start: u64, end: u64) -> Result> { - let mut results = Vec::new(); - - for height in start..=end { - match self.analyze_block(height).await { - Ok(analysis) => { - println!( - "Block {}: current={} bytes, estimated={} bytes, delta=+{} bytes ({:.2}%), tx={}, tin={}, tout={}", - height, - analysis.current_compact_size, - analysis.estimated_with_transparent, - analysis.delta_bytes, - analysis.delta_percent, - analysis.tx_count, - analysis.transparent_inputs, - analysis.transparent_outputs - ); - results.push(analysis); - } - Err(e) => { - eprintln!("Error analyzing block {}: {}", height, e); - } - } - - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - } - - Ok(results) - } - - fn write_csv(&self, results: &[BlockAnalysis], filename: &str) -> Result<()> { - let mut wtr = csv::Writer::from_path(filename)?; - for result in results { - wtr.serialize(result)?; - } - wtr.flush()?; - Ok(()) - } - - fn print_summary(&self, results: &[BlockAnalysis]) { - if results.is_empty() { - return; - } - - let total_current: usize = results.iter().map(|r| r.current_compact_size).sum(); - let total_estimated: usize = results.iter().map(|r| r.estimated_with_transparent).sum(); - let total_delta = total_estimated as i64 - total_current as i64; - - let mut deltas: Vec = results.iter().map(|r| r.delta_percent).collect(); - deltas.sort_by(|a, b| a.partial_cmp(b).unwrap()); - - let median = deltas[deltas.len() / 2]; - let p95_idx = ((deltas.len() as f64) * 0.95) as usize; - let p95 = deltas[p95_idx.min(deltas.len() - 1)]; - - println!("\n=== ANALYSIS SUMMARY ==="); - println!("Blocks analyzed: {}", results.len()); - println!("\nCurrent compact blocks:"); - println!(" Total: {} bytes ({:.2} MB)", total_current, total_current as f64 / 1_000_000.0); - println!("\nWith transparent data:"); - println!(" Estimated total: {} bytes ({:.2} MB)", total_estimated, total_estimated as f64 / 1_000_000.0); - println!(" Delta: +{} bytes ({:.2} MB)", total_delta, total_delta as f64 / 1_000_000.0); - println!(" Overall increase: {:.2}%", (total_delta as f64 / total_current as f64) * 100.0); - println!("\nPer-block statistics:"); - println!(" Median increase: {:.2}%", median); - println!(" 95th percentile: {:.2}%", p95); - println!(" Min: {:.2}%", deltas[0]); - println!(" Max: {:.2}%", deltas[deltas.len() - 1]); - - // Practical impact examples - println!("\nPractical impact:"); - let blocks_per_day = 24 * 60 * 60 / 75; // 1 block every 75 seconds on Zcash - let daily_current = (total_current as f64 / results.len() as f64) * blocks_per_day as f64; - let daily_estimated = (total_estimated as f64 / results.len() as f64) * blocks_per_day as f64; - println!(" Current daily sync (~{} blocks): {:.2} MB", blocks_per_day, daily_current / 1_000_000.0); - println!(" With transparent: {:.2} MB", daily_estimated / 1_000_000.0); - println!(" Additional bandwidth per day: {:.2} MB", (daily_estimated - daily_current) / 1_000_000.0); - } -} - -#[tokio::main] -async fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - - if args.len() < 5 { - eprintln!("Usage: {} [output.csv]", args[0]); - eprintln!("Example: {} http://localhost:9067 http://localhost:8232 2400000 2401000 results.csv", args[0]); - std::process::exit(1); - } - - let lightwalletd_url = &args[1]; - let zebra_rpc_url = &args[2]; - let start_height: u64 = args[3].parse()?; - let end_height: u64 = args[4].parse()?; - let output_file = args.get(5).map(|s| s.as_str()).unwrap_or("analysis.csv"); - - let estimator = TransparentEstimator::new( - zebra_rpc_url.to_string(), - lightwalletd_url.to_string(), - ); - - println!("Analyzing blocks {} to {}...", start_height, end_height); - println!("Fetching real compact blocks from lightwalletd: {}", lightwalletd_url); - println!("Fetching full blocks from Zebrad: {}", zebra_rpc_url); - println!(); - - let results = estimator.analyze_range(start_height, end_height).await?; - - estimator.write_csv(&results, output_file)?; - println!("\nDetailed results written to: {}", output_file); - - estimator.print_summary(&results); - - Ok(()) -} diff --git a/visualization/.python-version b/visualization/.python-version new file mode 100644 index 0000000..1d17dae --- /dev/null +++ b/visualization/.python-version @@ -0,0 +1 @@ +.venv diff --git a/visualization/README.md b/visualization/README.md new file mode 100644 index 0000000..d981123 --- /dev/null +++ b/visualization/README.md @@ -0,0 +1,23 @@ +# Visualization Tools + +## Setup + +```bash +cd visualization +python3 -m venv venv +source venv/bin/activate # On Windows: venv\Scripts\activate +pip install -r requirements.txt +``` + +## Usage + +```bash +python visualize.py ../results/analysis.csv --output-dir ./charts +``` + +## Output + +Generates: +- Statistical charts (PNG files) +- Statistical report (TXT file) +- Decision recommendations diff --git a/visualization/charts/.gitignore b/visualization/charts/.gitignore new file mode 100644 index 0000000..71a8ecc --- /dev/null +++ b/visualization/charts/.gitignore @@ -0,0 +1,9 @@ +bandwidth_impact.png +by_era.png +correlations.png +cumulative.png +distribution.png +heatmap.png +statistical_report.txt +time_series.png + diff --git a/visualization/charts/.gitkeep b/visualization/charts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/visualization/requirements.txt b/visualization/requirements.txt new file mode 100644 index 0000000..0968df0 --- /dev/null +++ b/visualization/requirements.txt @@ -0,0 +1,5 @@ +pandas>=2.0.0 +matplotlib>=3.7.0 +seaborn>=0.12.0 +numpy>=1.24.0 +scipy>=1.10.0 diff --git a/visualization/visualize.py b/visualization/visualize.py new file mode 100644 index 0000000..3ba4930 --- /dev/null +++ b/visualization/visualize.py @@ -0,0 +1,908 @@ +#!/usr/bin/env python3 +""" +Visualization and statistical analysis for Zcash compact block overhead data. + +Usage: + python visualize.py results.csv [--output-dir ./charts] +""" + +import pandas as pd +import matplotlib.pyplot as plt +import seaborn as sns +import numpy as np +from scipy import stats +import argparse +from pathlib import Path + +# Set style +sns.set_style("whitegrid") +plt.rcParams["figure.figsize"] = (12, 6) +plt.rcParams["font.size"] = 10 + + +class CompactBlockAnalyzer: + def __init__(self, csv_path): + self.df = pd.read_csv(csv_path) + self.df["delta_mb"] = self.df["delta_bytes"] / 1_000_000 + self.df["current_mb"] = self.df["current_compact_size"] / 1_000_000 + self.df["estimated_mb"] = self.df["estimated_with_transparent"] / 1_000_000 + + def generate_all_charts(self, output_dir="./charts"): + """Generate all visualization charts""" + output_path = Path(output_dir) + output_path.mkdir(exist_ok=True) + + print("Generating visualizations...") + + self.plot_distribution(output_path / "distribution.png") + self.plot_time_series(output_path / "time_series.png") + self.plot_by_era(output_path / "by_era.png") + self.plot_correlations(output_path / "correlations.png") + self.plot_cumulative(output_path / "cumulative.png") + self.plot_bandwidth_impact(output_path / "bandwidth_impact.png") + self.plot_heatmap(output_path / "heatmap.png") + + # Generate statistical report + self.generate_report(output_path / "statistical_report.txt") + + print(f"\nAll visualizations saved to: {output_path}") + + def plot_distribution(self, filename): + """Histogram of overhead distribution - both percentage and absolute""" + fig, axes = plt.subplots(2, 2, figsize=(14, 10)) + + # Top row: Percentage overhead + # Histogram with KDE + axes[0, 0].hist( + self.df["delta_percent"], + bins=50, + alpha=0.7, + edgecolor="black", + density=True, + label="Distribution", + ) + + # Add KDE + kde_x = np.linspace( + self.df["delta_percent"].min(), self.df["delta_percent"].max(), 100 + ) + kde = stats.gaussian_kde(self.df["delta_percent"]) + axes[0, 0].plot(kde_x, kde(kde_x), "r-", linewidth=2, label="KDE") + + # Add median and mean + median = self.df["delta_percent"].median() + mean = self.df["delta_percent"].mean() + axes[0, 0].axvline( + median, + color="green", + linestyle="--", + linewidth=2, + label=f"Median: {median:.1f}%", + ) + axes[0, 0].axvline( + mean, + color="orange", + linestyle="--", + linewidth=2, + label=f"Mean: {mean:.1f}%", + ) + + axes[0, 0].set_xlabel("Overhead Percentage (%)") + axes[0, 0].set_ylabel("Density") + axes[0, 0].set_title("Distribution of Overhead (Percentage)") + axes[0, 0].legend() + axes[0, 0].grid(True, alpha=0.3) + + # Box plot - percentage + axes[0, 1].boxplot(self.df["delta_percent"], vert=True) + axes[0, 1].set_ylabel("Overhead Percentage (%)") + axes[0, 1].set_title("Overhead Distribution (Box Plot - Percentage)") + axes[0, 1].grid(True, alpha=0.3) + + # Bottom row: Absolute overhead in KB + self.df["delta_kb"] = self.df["delta_bytes"] / 1000 + + # Histogram with KDE - absolute + axes[1, 0].hist( + self.df["delta_kb"], + bins=50, + alpha=0.7, + edgecolor="black", + density=True, + label="Distribution", + color="coral", + ) + + # Add KDE + kde_x_kb = np.linspace( + self.df["delta_kb"].min(), self.df["delta_kb"].max(), 100 + ) + kde_kb = stats.gaussian_kde(self.df["delta_kb"]) + axes[1, 0].plot(kde_x_kb, kde_kb(kde_x_kb), "r-", linewidth=2, label="KDE") + + # Add median and mean + median_kb = self.df["delta_kb"].median() + mean_kb = self.df["delta_kb"].mean() + axes[1, 0].axvline( + median_kb, + color="green", + linestyle="--", + linewidth=2, + label=f"Median: {median_kb:.1f} KB", + ) + axes[1, 0].axvline( + mean_kb, + color="orange", + linestyle="--", + linewidth=2, + label=f"Mean: {mean_kb:.1f} KB", + ) + + axes[1, 0].set_xlabel("Overhead (KB per block)") + axes[1, 0].set_ylabel("Density") + axes[1, 0].set_title("Distribution of Overhead (Absolute Size)") + axes[1, 0].legend() + axes[1, 0].grid(True, alpha=0.3) + + # Box plot - absolute + axes[1, 1].boxplot(self.df["delta_kb"], vert=True) + axes[1, 1].set_ylabel("Overhead (KB per block)") + axes[1, 1].set_title("Overhead Distribution (Box Plot - Absolute)") + axes[1, 1].grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig(filename, dpi=300, bbox_inches="tight") + plt.close() + print(f" ✓ Distribution chart saved: {filename}") + + def plot_time_series(self, filename): + """Overhead over blockchain height""" + fig, axes = plt.subplots(2, 1, figsize=(14, 10)) + + # Scatter plot with trend + axes[0].scatter( + self.df["height"], + self.df["delta_percent"], + alpha=0.4, + s=10, + c=self.df["delta_percent"], + cmap="YlOrRd", + ) + + # Add rolling mean + window = max(len(self.df) // 50, 10) + rolling_mean = ( + self.df.set_index("height")["delta_percent"].rolling(window).mean() + ) + axes[0].plot( + rolling_mean.index, + rolling_mean.values, + "b-", + linewidth=2, + label=f"Rolling Mean (window={window})", + ) + + # Add era boundaries + eras = [ + (419_200, "Sapling", "red"), + (653_600, "Blossom", "blue"), + (903_800, "Heartwood", "purple"), + (1_046_400, "Canopy", "orange"), + (1_687_104, "NU5", "green"), + (2_726_400, "NU6", "brown"), + ] + + for height, name, color in eras: + if height >= self.df["height"].min() and height <= self.df["height"].max(): + axes[0].axvline( + height, color=color, alpha=0.3, linestyle="--", linewidth=1.5 + ) + axes[0].text( + height, + axes[0].get_ylim()[1] * 0.95, + name, + rotation=90, + verticalalignment="top", + color=color, + ) + + axes[0].set_xlabel("Block Height") + axes[0].set_ylabel("Overhead (%)") + axes[0].set_title("Compact Block Overhead Over Time") + axes[0].legend() + axes[0].grid(True, alpha=0.3) + + # Absolute size over time + axes[1].scatter( + self.df["height"], + self.df["delta_mb"], + alpha=0.4, + s=10, + c=self.df["delta_mb"], + cmap="YlOrRd", + ) + + rolling_delta = self.df.set_index("height")["delta_mb"].rolling(window).mean() + axes[1].plot( + rolling_delta.index, + rolling_delta.values, + "b-", + linewidth=2, + label=f"Rolling Mean", + ) + + for height, name, color in eras: + if height >= self.df["height"].min() and height <= self.df["height"].max(): + axes[1].axvline( + height, color=color, alpha=0.3, linestyle="--", linewidth=1.5 + ) + + axes[1].set_xlabel("Block Height") + axes[1].set_ylabel("Delta Size (MB)") + axes[1].set_title("Absolute Size Increase Over Time") + axes[1].legend() + axes[1].grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig(filename, dpi=300, bbox_inches="tight") + plt.close() + print(f" ✓ Time series chart saved: {filename}") + + def plot_by_era(self, filename): + """Compare distributions across eras - both percentage and absolute""" + fig, axes = plt.subplots(2, 3, figsize=(18, 10)) + + era_order = ["sapling", "blossom", "heartwood", "canopy", "nu5", "nu6"] + available_eras = [e for e in era_order if e in self.df["era"].values] + + colors = [ + "lightgreen", + "lightyellow", + "lightpink", + "lightcoral", + "lavender", + "peachpuff", + ] + + # Add KB column + self.df["delta_kb"] = self.df["delta_bytes"] / 1000 + + # Top row: Percentage overhead + # Box plot by era - percentage + box_data_pct = [ + self.df[self.df["era"] == era]["delta_percent"].values + for era in available_eras + ] + + bp1 = axes[0, 0].boxplot(box_data_pct, labels=available_eras, patch_artist=True) + for patch, color in zip(bp1["boxes"], colors[: len(available_eras)]): + patch.set_facecolor(color) + + axes[0, 0].set_ylabel("Overhead (%)") + axes[0, 0].set_title("Overhead Distribution by Era (Percentage)") + axes[0, 0].grid(True, alpha=0.3) + axes[0, 0].tick_params(axis="x", rotation=45) + + # Violin plot - percentage + if len(available_eras) > 0: + era_df = self.df[self.df["era"].isin(available_eras)] + sns.violinplot( + data=era_df, + x="era", + y="delta_percent", + order=available_eras, + ax=axes[0, 1], + ) + axes[0, 1].set_ylabel("Overhead (%)") + axes[0, 1].set_title("Overhead Distribution by Era (Violin - Percentage)") + axes[0, 1].grid(True, alpha=0.3) + axes[0, 1].tick_params(axis="x", rotation=45) + + # Bar chart of means - percentage + era_stats_pct = self.df.groupby("era")["delta_percent"].agg(["mean", "std"]) + era_stats_pct = era_stats_pct.reindex(available_eras) + + x = np.arange(len(available_eras)) + axes[0, 2].bar( + x, + era_stats_pct["mean"], + yerr=era_stats_pct["std"], + capsize=5, + alpha=0.7, + color=colors[: len(available_eras)], + ) + axes[0, 2].set_xticks(x) + axes[0, 2].set_xticklabels(available_eras, rotation=45) + axes[0, 2].set_ylabel("Mean Overhead (%)") + axes[0, 2].set_title("Average Overhead by Era (Percentage)") + axes[0, 2].grid(True, alpha=0.3) + + # Bottom row: Absolute KB overhead + # Box plot by era - KB + box_data_kb = [ + self.df[self.df["era"] == era]["delta_kb"].values for era in available_eras + ] + + bp2 = axes[1, 0].boxplot(box_data_kb, labels=available_eras, patch_artist=True) + for patch, color in zip(bp2["boxes"], colors[: len(available_eras)]): + patch.set_facecolor(color) + + axes[1, 0].set_ylabel("Overhead (KB per block)") + axes[1, 0].set_title("Overhead Distribution by Era (Absolute)") + axes[1, 0].grid(True, alpha=0.3) + axes[1, 0].tick_params(axis="x", rotation=45) + + # Violin plot - KB + if len(available_eras) > 0: + sns.violinplot( + data=era_df, + x="era", + y="delta_kb", + order=available_eras, + ax=axes[1, 1], + color="coral", + ) + axes[1, 1].set_ylabel("Overhead (KB per block)") + axes[1, 1].set_title("Overhead Distribution by Era (Violin - Absolute)") + axes[1, 1].grid(True, alpha=0.3) + axes[1, 1].tick_params(axis="x", rotation=45) + + # Bar chart of means - KB + era_stats_kb = self.df.groupby("era")["delta_kb"].agg(["mean", "std"]) + era_stats_kb = era_stats_kb.reindex(available_eras) + + axes[1, 2].bar( + x, + era_stats_kb["mean"], + yerr=era_stats_kb["std"], + capsize=5, + alpha=0.7, + color=colors[: len(available_eras)], + ) + axes[1, 2].set_xticks(x) + axes[1, 2].set_xticklabels(available_eras, rotation=45) + axes[1, 2].set_ylabel("Mean Overhead (KB per block)") + axes[1, 2].set_title("Average Overhead by Era (Absolute)") + axes[1, 2].grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig(filename, dpi=300, bbox_inches="tight") + plt.close() + print(f" ✓ Era comparison chart saved: {filename}") + + def plot_correlations(self, filename): + """Correlation between overhead and transaction characteristics""" + fig, axes = plt.subplots(2, 2, figsize=(14, 10)) + + # Overhead vs transparent inputs + axes[0, 0].scatter( + self.df["transparent_inputs"], self.df["delta_bytes"], alpha=0.5, s=20 + ) + z = np.polyfit(self.df["transparent_inputs"], self.df["delta_bytes"], 1) + p = np.poly1d(z) + axes[0, 0].plot( + self.df["transparent_inputs"], + p(self.df["transparent_inputs"]), + "r--", + linewidth=2, + label=f"Trend", + ) + + corr = self.df["transparent_inputs"].corr(self.df["delta_bytes"]) + axes[0, 0].set_xlabel("Transparent Inputs") + axes[0, 0].set_ylabel("Delta (bytes)") + axes[0, 0].set_title(f"Overhead vs Transparent Inputs (r={corr:.3f})") + axes[0, 0].legend() + axes[0, 0].grid(True, alpha=0.3) + + # Overhead vs transparent outputs + axes[0, 1].scatter( + self.df["transparent_outputs"], + self.df["delta_bytes"], + alpha=0.5, + s=20, + color="orange", + ) + z = np.polyfit(self.df["transparent_outputs"], self.df["delta_bytes"], 1) + p = np.poly1d(z) + axes[0, 1].plot( + self.df["transparent_outputs"], + p(self.df["transparent_outputs"]), + "r--", + linewidth=2, + label="Trend", + ) + + corr = self.df["transparent_outputs"].corr(self.df["delta_bytes"]) + axes[0, 1].set_xlabel("Transparent Outputs") + axes[0, 1].set_ylabel("Delta (bytes)") + axes[0, 1].set_title(f"Overhead vs Transparent Outputs (r={corr:.3f})") + axes[0, 1].legend() + axes[0, 1].grid(True, alpha=0.3) + + # Overhead vs total transactions + axes[1, 0].scatter( + self.df["tx_count"], + self.df["delta_percent"], + alpha=0.5, + s=20, + color="green", + ) + + corr = self.df["tx_count"].corr(self.df["delta_percent"]) + axes[1, 0].set_xlabel("Transaction Count") + axes[1, 0].set_ylabel("Overhead (%)") + axes[1, 0].set_title(f"Overhead % vs Transaction Count (r={corr:.3f})") + axes[1, 0].grid(True, alpha=0.3) + + # Current size vs overhead + axes[1, 1].scatter( + self.df["current_compact_size"], + self.df["delta_percent"], + alpha=0.5, + s=20, + color="purple", + ) + + corr = self.df["current_compact_size"].corr(self.df["delta_percent"]) + axes[1, 1].set_xlabel("Current Compact Block Size (bytes)") + axes[1, 1].set_ylabel("Overhead (%)") + axes[1, 1].set_title(f"Overhead % vs Block Size (r={corr:.3f})") + axes[1, 1].grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig(filename, dpi=300, bbox_inches="tight") + plt.close() + print(f" ✓ Correlation chart saved: {filename}") + + def plot_cumulative(self, filename): + """Cumulative distribution function""" + fig, axes = plt.subplots(1, 2, figsize=(14, 5)) + + # CDF of overhead percentage + sorted_overhead = np.sort(self.df["delta_percent"]) + cumulative = np.arange(1, len(sorted_overhead) + 1) / len(sorted_overhead) + + axes[0].plot(sorted_overhead, cumulative * 100, linewidth=2) + + # Mark percentiles + percentiles = [50, 75, 90, 95, 99] + for p in percentiles: + value = np.percentile(sorted_overhead, p) + axes[0].axvline( + value, linestyle="--", alpha=0.5, label=f"P{p}: {value:.1f}%" + ) + axes[0].axhline(p, linestyle=":", alpha=0.3) + + axes[0].set_xlabel("Overhead (%)") + axes[0].set_ylabel("Cumulative Percentage of Blocks") + axes[0].set_title("Cumulative Distribution of Overhead") + axes[0].legend() + axes[0].grid(True, alpha=0.3) + + # CDF of absolute delta + sorted_delta = np.sort(self.df["delta_bytes"]) + axes[1].plot(sorted_delta / 1000, cumulative * 100, linewidth=2, color="orange") + + for p in percentiles: + value = np.percentile(sorted_delta, p) / 1000 + axes[1].axvline( + value, linestyle="--", alpha=0.5, label=f"P{p}: {value:.1f}KB" + ) + + axes[1].set_xlabel("Delta Size (KB)") + axes[1].set_ylabel("Cumulative Percentage of Blocks") + axes[1].set_title("Cumulative Distribution of Absolute Overhead") + axes[1].legend() + axes[1].grid(True, alpha=0.3) + + plt.tight_layout() + plt.savefig(filename, dpi=300, bbox_inches="tight") + plt.close() + print(f" ✓ Cumulative distribution chart saved: {filename}") + + def plot_bandwidth_impact(self, filename): + """Practical bandwidth impact scenarios""" + fig, axes = plt.subplots(2, 2, figsize=(14, 10)) + + # Calculate metrics + avg_current = self.df["current_compact_size"].mean() + avg_with_transparent = self.df["estimated_with_transparent"].mean() + avg_delta = avg_with_transparent - avg_current + + # Calculate blocks per day based on current network (post-Blossom) + # Post-Blossom (after block 653,600): 75s blocks = 1,152 blocks/day + # This is what matters for current light clients + blocks_per_day = 1152 + + daily_current_mb = (avg_current * blocks_per_day) / 1_000_000 + daily_with_mb = (avg_with_transparent * blocks_per_day) / 1_000_000 + daily_delta_mb = daily_with_mb - daily_current_mb + + # Daily sync with absolute delta + axes[0, 0].bar( + ["Current", "With\nTransparent"], + [daily_current_mb, daily_with_mb], + color=["steelblue", "coral"], + ) + axes[0, 0].set_ylabel("MB") + axes[0, 0].set_title(f"Daily Sync Bandwidth\n(~{blocks_per_day} blocks/day)") + axes[0, 0].grid(True, alpha=0.3, axis="y") + + for i, v in enumerate([daily_current_mb, daily_with_mb]): + axes[0, 0].text( + i, v + 1, f"{v:.1f} MB", ha="center", va="bottom", fontweight="bold" + ) + + # Add delta annotation + axes[0, 0].text( + 0.5, + max(daily_current_mb, daily_with_mb) * 0.5, + f"Δ = +{daily_delta_mb:.1f} MB\n({(daily_delta_mb/daily_current_mb)*100:.1f}%)", + ha="center", + fontsize=12, + fontweight="bold", + bbox=dict(boxstyle="round", facecolor="yellow", alpha=0.3), + ) + + # Full sync (estimate based on current tip) + max_height = self.df["height"].max() + total_blocks = max_height # Approximate total blocks + full_current_gb = (avg_current * total_blocks) / 1_000_000_000 + full_with_gb = (avg_with_transparent * total_blocks) / 1_000_000_000 + full_delta_gb = full_with_gb - full_current_gb + + axes[0, 1].bar( + ["Current", "With\nTransparent"], + [full_current_gb, full_with_gb], + color=["steelblue", "coral"], + ) + axes[0, 1].set_ylabel("GB") + axes[0, 1].set_title(f"Full Chain Sync\n(~{total_blocks:,} blocks)") + axes[0, 1].grid(True, alpha=0.3, axis="y") + + for i, v in enumerate([full_current_gb, full_with_gb]): + axes[0, 1].text( + i, v + 0.1, f"{v:.2f} GB", ha="center", va="bottom", fontweight="bold" + ) + + # Add delta annotation + axes[0, 1].text( + 0.5, + max(full_current_gb, full_with_gb) * 0.5, + f"Δ = +{full_delta_gb:.2f} GB\n({(full_delta_gb/full_current_gb)*100:.1f}%)", + ha="center", + fontsize=12, + fontweight="bold", + bbox=dict(boxstyle="round", facecolor="yellow", alpha=0.3), + ) + + # Absolute delta chart - use MB consistently + x = np.arange(3) + width = 0.35 + + # Calculate for different time periods (all in MB) + daily_delta_mb = daily_delta_mb + weekly_delta_mb = daily_delta_mb * 7 + monthly_delta_mb = daily_delta_mb * 30 + + deltas = [daily_delta_mb, weekly_delta_mb, monthly_delta_mb] + labels = [ + f"Daily\n({blocks_per_day} blocks)", + "Weekly\n(7 days)", + "Monthly\n(30 days)", + ] + + bars = axes[1, 0].bar(x, deltas, color="coral", alpha=0.7) + axes[1, 0].set_ylabel("Additional Bandwidth (MB)") + axes[1, 0].set_title("Absolute Bandwidth Increase\n(Transparent Data Overhead)") + axes[1, 0].set_xticks(x) + axes[1, 0].set_xticklabels(labels) + axes[1, 0].grid(True, alpha=0.3, axis="y") + + # Add value labels + for i, (bar, val) in enumerate(zip(bars, deltas)): + height = bar.get_height() + axes[1, 0].text( + bar.get_x() + bar.get_width() / 2.0, + height, + f"+{val:.2f} MB", + ha="center", + va="bottom", + fontweight="bold", + ) + + # Sync time (assume 5 Mbps connection) + bandwidth_mbps = 5 + bandwidth_mb_per_sec = bandwidth_mbps / 8 + + daily_time_current = daily_current_mb / bandwidth_mb_per_sec / 60 # minutes + daily_time_with = daily_with_mb / bandwidth_mb_per_sec / 60 + daily_time_delta = daily_time_with - daily_time_current + + full_time_current = ( + full_current_gb * 1000 / bandwidth_mb_per_sec / 3600 + ) # hours + full_time_with = full_with_gb * 1000 / bandwidth_mb_per_sec / 3600 + full_time_delta = full_time_with - full_time_current + + x = np.arange(2) + axes[1, 1].bar( + x - width / 2, + [daily_time_current, full_time_current], + width, + label="Current", + color="steelblue", + ) + axes[1, 1].bar( + x + width / 2, + [daily_time_with, full_time_with], + width, + label="With Transparent", + color="coral", + ) + + axes[1, 1].set_ylabel("Time") + axes[1, 1].set_title(f"Sync Time\n(@{bandwidth_mbps} Mbps)") + axes[1, 1].set_xticks(x) + axes[1, 1].set_xticklabels(["Daily\n(minutes)", "Full\n(hours)"]) + axes[1, 1].legend() + axes[1, 1].grid(True, alpha=0.3, axis="y") + + # Add delta annotations + for i, delta in enumerate([daily_time_delta, full_time_delta]): + y_pos = ( + max( + [daily_time_current, full_time_current][i], + [daily_time_with, full_time_with][i], + ) + * 1.05 + ) + unit = "min" if i == 0 else "hrs" + axes[1, 1].text( + i, + y_pos, + f"+{delta:.1f} {unit}", + ha="center", + fontsize=10, + fontweight="bold", + bbox=dict(boxstyle="round", facecolor="yellow", alpha=0.3), + ) + + plt.tight_layout() + plt.savefig(filename, dpi=300, bbox_inches="tight") + plt.close() + print(f" ✓ Bandwidth impact chart saved: {filename}") + + def plot_heatmap(self, filename): + """Heatmap of overhead by era and transaction characteristics""" + fig, axes = plt.subplots(1, 2, figsize=(14, 5)) + + # Create bins for transaction count + self.df["tx_bin"] = pd.cut( + self.df["tx_count"], + bins=[0, 10, 50, 100, 500, 10000], + labels=["1-10", "11-50", "51-100", "101-500", "500+"], + ) + + # Pivot table: era vs tx_bin + if "era" in self.df.columns and not self.df["era"].isna().all(): + pivot = self.df.pivot_table( + values="delta_percent", index="era", columns="tx_bin", aggfunc="mean" + ) + + sns.heatmap( + pivot, + annot=True, + fmt=".1f", + cmap="YlOrRd", + ax=axes[0], + cbar_kws={"label": "Overhead (%)"}, + ) + axes[0].set_title("Average Overhead by Era and Transaction Count") + axes[0].set_ylabel("Era") + axes[0].set_xlabel("Transactions per Block") + + # Pivot table: transparent usage + self.df["transparent_bin"] = pd.cut( + self.df["transparent_inputs"] + self.df["transparent_outputs"], + bins=[0, 10, 50, 100, 500, 10000], + labels=["1-10", "11-50", "51-100", "101-500", "500+"], + ) + + if "era" in self.df.columns and not self.df["era"].isna().all(): + pivot2 = self.df.pivot_table( + values="delta_percent", + index="era", + columns="transparent_bin", + aggfunc="mean", + ) + + sns.heatmap( + pivot2, + annot=True, + fmt=".1f", + cmap="YlOrRd", + ax=axes[1], + cbar_kws={"label": "Overhead (%)"}, + ) + axes[1].set_title("Average Overhead by Era and Transparent I/O") + axes[1].set_ylabel("Era") + axes[1].set_xlabel("Transparent Inputs + Outputs") + + plt.tight_layout() + plt.savefig(filename, dpi=300, bbox_inches="tight") + plt.close() + print(f" ✓ Heatmap saved: {filename}") + + def generate_report(self, filename): + """Generate statistical report in Markdown format""" + with open(filename, "w") as f: + f.write("# Statistical Analysis Report\n\n") + f.write("**Zcash Compact Block Transparent Data Overhead**\n\n") + f.write("---\n\n") + + # Summary statistics + f.write("## Summary Statistics\n\n") + f.write(f"- **Total blocks analyzed:** {len(self.df):,}\n") + f.write( + f"- **Block height range:** {self.df['height'].min():,} - {self.df['height'].max():,}\n\n" + ) + + # Overhead statistics + f.write("## Overhead Percentage\n\n") + f.write("| Metric | Value |\n") + f.write("|--------|-------|\n") + f.write(f"| Mean | {self.df['delta_percent'].mean():.2f}% |\n") + f.write(f"| Median | {self.df['delta_percent'].median():.2f}% |\n") + f.write(f"| Std Dev| {self.df['delta_percent'].std():.2f}% |\n") + f.write(f"| Min | {self.df['delta_percent'].min():.2f}% |\n") + f.write(f"| Max | {self.df['delta_percent'].max():.2f}% |\n\n") + + # Percentiles + f.write("## Percentiles\n\n") + f.write("| Percentile | Overhead |\n") + f.write("|------------|----------|\n") + for p in [25, 50, 75, 90, 95, 99]: + value = np.percentile(self.df["delta_percent"], p) + f.write(f"| P{p} | {value:.2f}% |\n") + f.write("\n") + + # Confidence intervals + f.write("## Confidence Intervals (95%)\n\n") + mean = self.df["delta_percent"].mean() + std_err = stats.sem(self.df["delta_percent"]) + ci = stats.t.interval(0.95, len(self.df) - 1, loc=mean, scale=std_err) + f.write(f"- **Mean overhead:** {mean:.2f}% ± {(ci[1]-mean):.2f}%\n") + f.write(f"- **Range:** [{ci[0]:.2f}%, {ci[1]:.2f}%]\n\n") + + # By era + f.write("## Statistics by Era\n\n") + if "era" in self.df.columns: + f.write("| Era | Count | Mean | Std Dev | Median | Min | Max |\n") + f.write("|-----|-------|------|---------|--------|-----|-----|\n") + + era_order = ["sapling", "blossom", "heartwood", "canopy", "nu5", "nu6"] + for era in era_order: + if era in self.df["era"].values: + era_data = self.df[self.df["era"] == era]["delta_percent"] + f.write( + f"| {era.capitalize()} | {len(era_data):,} | {era_data.mean():.2f}% | " + f"{era_data.std():.2f}% | {era_data.median():.2f}% | " + f"{era_data.min():.2f}% | {era_data.max():.2f}% |\n" + ) + f.write("\n") + + # Bandwidth impact + f.write("## Practical Bandwidth Impact\n\n") + + avg_current = self.df["current_compact_size"].mean() + avg_with = self.df["estimated_with_transparent"].mean() + avg_delta = avg_with - avg_current + + f.write("### Average Block Sizes\n\n") + f.write(f"- **Current:** {avg_current/1000:.2f} KB\n") + f.write(f"- **With transparent:** {avg_with/1000:.2f} KB\n") + f.write(f"- **Delta:** {avg_delta/1000:.2f} KB\n\n") + + blocks_per_day = 1152 + daily_current_mb = (avg_current * blocks_per_day) / 1_000_000 + daily_with_mb = (avg_with * blocks_per_day) / 1_000_000 + daily_delta_mb = daily_with_mb - daily_current_mb + + f.write(f"### Daily Sync (~{blocks_per_day} blocks)\n\n") + f.write(f"- **Current:** {daily_current_mb:.2f} MB\n") + f.write(f"- **With transparent:** {daily_with_mb:.2f} MB\n") + f.write( + f"- **Additional:** {daily_delta_mb:.2f} MB ({(daily_delta_mb/daily_current_mb)*100:.1f}%)\n\n" + ) + + # Correlations + f.write("## Correlations\n\n") + corr_inputs = self.df["transparent_inputs"].corr(self.df["delta_bytes"]) + corr_outputs = self.df["transparent_outputs"].corr(self.df["delta_bytes"]) + corr_tx = self.df["tx_count"].corr(self.df["delta_percent"]) + + f.write("| Variables | Correlation (r) |\n") + f.write("|-----------|----------------|\n") + f.write(f"| Transparent inputs → delta bytes | {corr_inputs:.3f} |\n") + f.write(f"| Transparent outputs → delta bytes | {corr_outputs:.3f} |\n") + f.write(f"| Transaction count → overhead % | {corr_tx:.3f} |\n\n") + + # Recommendations + f.write("## Decision Framework\n\n") + median_overhead = self.df["delta_percent"].median() + p95_overhead = np.percentile(self.df["delta_percent"], 95) + + f.write(f"- **Median overhead:** {median_overhead:.1f}%\n") + f.write(f"- **95th percentile:** {p95_overhead:.1f}%\n\n") + + if median_overhead < 20: + f.write("### ✅ Recommendation: LOW IMPACT\n\n") + f.write( + "The overhead is relatively small (<20%). Consider making transparent " + ) + f.write( + "data part of the default GetBlockRange method. This would:\n\n" + ) + f.write("- Simplify the API (single method)\n") + f.write("- Provide feature parity with full nodes\n") + f.write("- Have minimal bandwidth impact on users\n") + elif median_overhead < 50: + f.write("### ⚠️ Recommendation: MODERATE IMPACT\n\n") + f.write("The overhead is significant (20-50%). Consider:\n\n") + f.write("- Separate opt-in method for transparent data\n") + f.write("- Pool-based filtering (as in librustzcash PR #1781)\n") + f.write("- Let clients choose based on their needs\n") + f.write("- Important for mobile/bandwidth-limited users\n") + else: + f.write("### 🚨 Recommendation: HIGH IMPACT\n\n") + f.write("The overhead is substantial (>50%). Strongly consider:\n\n") + f.write("- Separate method required\n") + f.write("- Clear opt-in for clients needing transparent data\n") + f.write("- Critical for mobile and bandwidth-limited users\n") + f.write("- May significantly impact sync times\n") + + f.write("\n---\n\n") + f.write("*Report generated by Compact Block Analyzer*\n") + + print(f" ✓ Statistical report saved: {filename}") + + +def main(): + parser = argparse.ArgumentParser( + description="Generate visualizations and statistical analysis for compact block data" + ) + parser.add_argument("csv_file", help="Path to CSV file with analysis results") + parser.add_argument( + "--output-dir", + "-o", + default="./charts", + help="Output directory for charts (default: ./charts)", + ) + + args = parser.parse_args() + + # Check if file exists + if not Path(args.csv_file).exists(): + print(f"Error: File not found: {args.csv_file}") + return 1 + + print(f"Loading data from: {args.csv_file}") + analyzer = CompactBlockAnalyzer(args.csv_file) + + print(f"Loaded {len(analyzer.df)} blocks") + print() + + # Generate all visualizations + analyzer.generate_all_charts(args.output_dir) + + print("\n✅ Analysis complete!") + print(f"\nView the statistical report: {args.output_dir}/statistical_report.txt") + print(f"View charts in: {args.output_dir}/") + + return 0 + + +if __name__ == "__main__": + exit(main())