From f2da554d4559399b5e675d5eeb7ac3e1dda3eed1 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Mon, 31 May 2021 13:44:48 +0200 Subject: [PATCH 1/6] use electrs without monitoring --- .github/workflows/cont_integration.yml | 19 +++---- Cargo.toml | 4 ++ src/lib.rs | 77 +++++++++++++++++--------- 3 files changed, 63 insertions(+), 37 deletions(-) diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index 6307395..ac48056 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -27,30 +27,25 @@ jobs: ~/.cargo/git ~/.cargo/registry target + electrs/target/release/electrs bitcoin-${{ env.BITCOIN_VER }} - key: ${{ runner.os }}-test-electrsd-${{ env.BITCOIN_VER }}-${{ env.ELECTRS_VER }}-${{ hashFiles('Cargo.toml','Cargo.lock') }} + key: ${{ runner.os }}-test-electrsd1-${{ env.BITCOIN_VER }}-${{ env.ELECTRS_VER }}-${{ hashFiles('Cargo.toml','Cargo.lock') }} - name: Setup rust toolchain uses: actions-rs/toolchain@v1 with: toolchain: stable override: true - name: Set ELECTRS_EXE env - run: echo "ELECTRS_EXE=$HOME/.cargo/bin/electrs" >> $GITHUB_ENV - - name: Install electrs + run: echo "ELECTRS_EXE=${{ github.workspace }}/electrs/target/release/electrs" >> $GITHUB_ENV + - name: Build electrs if: steps.cache-step.outputs.cache-hit != 'true' - uses: actions-rs/cargo@v1 - with: - command: install - args: electrs --version ${{ env.ELECTRS_VER }} + run: git clone https://github.com/romanz/electrs && cd electrs && git checkout df513cfb18297d3d76097bb62dba93676bbf9a94 && cargo build --release --no-default-features - name: Show electrs options - run: electrs --help + run: ${{ env.ELECTRS_EXE }} --help - name: Set BITCOIND_EXE env run: echo "BITCOIND_EXE=${{ github.workspace }}/bitcoin-${{ env.BITCOIN_VER }}/bin/bitcoind" >> $GITHUB_ENV - name: Install bitcoind if: steps.cache-step.outputs.cache-hit != 'true' run: curl https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VER/bitcoin-$BITCOIN_VER-x86_64-linux-gnu.tar.gz | tar -xvz bitcoin-$BITCOIN_VER/bin/bitcoind - name: Test electrsd - uses: actions-rs/cargo@v1 - with: - command: test - args: --verbose ${{ matrix.features }} \ No newline at end of file + run: RUST_LOG=electrs=debug cargo test --verbose ${{ matrix.features }} diff --git a/Cargo.toml b/Cargo.toml index 07e840b..10329f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,6 +12,10 @@ edition = "2018" bitcoind = { version = "0.7.0" } electrum-client = { version="0.7.0", default-features = false } nix = { version="0.20.0", optional = true } +log = "0.4.14" + +[dev-dependencies] +env_logger = "0.8.3" [features] trigger = ["nix"] \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 6e7f0bd..c5f1918 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,9 +6,11 @@ //! Utility to run a regtest electrsd process, useful in integration testing environment //! +use bitcoind::bitcoincore_rpc::RpcApi; use bitcoind::tempfile::TempDir; use bitcoind::{get_available_port, BitcoinD}; use electrum_client::raw_client::{ElectrumPlaintextStream, RawClient}; +use log::debug; use std::ffi::OsStr; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -45,6 +47,10 @@ pub enum Error { /// Wrapper of bitcoincore_rpc Error BitcoinCoreRpc(bitcoind::bitcoincore_rpc::Error), + /// Electrs requires bitcoind started with p2p networking, this error is thrown if the node + /// starts without p2p + BitcoinNodeHasNoP2P, + #[cfg(feature = "trigger")] /// Wrapper of nix Error Nix(nix::Error), @@ -52,15 +58,28 @@ pub enum Error { impl ElectrsD { /// Create a new electrs process connected with the given bitcoind + /// One block will be generated in bitcoind if in IBD pub fn new>( exe: S, bitcoind: BitcoinD, view_stderr: bool, http_enabled: bool, ) -> Result { - let mut args = vec![]; - - args.push("-vvv"); + if bitcoind + .client + .get_blockchain_info()? + .initial_block_download + { + // electrum will remain idle until bitcoind is in IBD + // bitcoind will remain in IBD if doesn't see a block from a long time, thus adding a block + let node_address = bitcoind.client.get_new_address(None, None).unwrap(); + bitcoind + .client + .generate_to_address(1, &node_address) + .unwrap(); + } + + let mut args = vec!["-vvv"]; let _db_dir = TempDir::new()?; let db_dir = format!("{}", _db_dir.path().display()); @@ -78,22 +97,28 @@ impl ElectrsD { let rpc_socket = bitcoind.rpc_socket.to_string(); args.push(&rpc_socket); + let p2p_socket = bitcoind + .p2p_socket + .ok_or(Error::BitcoinNodeHasNoP2P)? + .to_string(); + args.push("--daemon-p2p-addr"); + args.push(&p2p_socket); + + //args.push("--daemon-dir"); + //let rpc_socket = bitcoind._work_dir.to_string(); + args.push("--jsonrpc-import"); let electrum_url = format!("0.0.0.0:{}", get_available_port()?); args.push("--electrum-rpc-addr"); args.push(&electrum_url); - // would be better to disable it, didn't found a flag - let monitoring = format!("0.0.0.0:{}", get_available_port()?); - args.push("--monitoring-addr"); - args.push(&monitoring); - let esplora_url_string; let esplora_url = if http_enabled { esplora_url_string = format!("0.0.0.0:{}", get_available_port()?); args.push("--http-addr"); args.push(&esplora_url_string); + #[allow(clippy::redundant_clone)] Some(esplora_url_string.clone()) } else { None @@ -105,7 +130,7 @@ impl ElectrsD { Stdio::null() }; - eprintln!("args: {:?}", args); + debug!("args: {:?}", args); let process = Command::new(exe).args(args).stderr(view_stderr).spawn()?; let client = loop { @@ -116,9 +141,9 @@ impl ElectrsD { }; Ok(ElectrsD { - client, - bitcoind, process, + bitcoind, + client, _db_dir, electrum_url, esplora_url, @@ -182,22 +207,18 @@ mod test { #[test] fn test_electrsd() { + env_logger::try_init().unwrap(); + let bitcoind_exe = env::var("BITCOIND_EXE").expect("BITCOIND_EXE env var must be set"); let electrs_exe = env::var("ELECTRS_EXE").expect("ELECTRS_EXE env var must be set"); - let bitcoind = BitcoinD::with_args(bitcoind_exe, vec![], true, bitcoind::P2P::No).unwrap(); - let electrsd = ElectrsD::new(electrs_exe, bitcoind, true, false).unwrap(); + let bitcoind = + BitcoinD::with_args(bitcoind_exe.clone(), vec![], true, bitcoind::P2P::Yes).unwrap(); + let electrsd = ElectrsD::new(electrs_exe.clone(), bitcoind, true, false).unwrap(); let header = electrsd.client.block_headers_subscribe().unwrap(); - assert_eq!(header.height, 0); - let address = electrsd - .bitcoind - .client - .get_new_address(None, None) - .unwrap(); - electrsd - .bitcoind - .client - .generate_to_address(101, &address) - .unwrap(); + assert_eq!(header.height, 1); + let node_client = &electrsd.bitcoind.client; + let address = node_client.get_new_address(None, None).unwrap(); + node_client.generate_to_address(100, &address).unwrap(); #[cfg(feature = "trigger")] electrsd.trigger().unwrap(); @@ -205,10 +226,16 @@ mod test { let header = loop { std::thread::sleep(std::time::Duration::from_secs(1)); let header = electrsd.client.block_headers_subscribe().unwrap(); - if header.height > 0 { + if header.height > 100 { break header; } }; assert_eq!(header.height, 101); + + // launch another instance to check there are no fixed port used + let bitcoind = BitcoinD::with_args(bitcoind_exe, vec![], true, bitcoind::P2P::Yes).unwrap(); + let electrsd = ElectrsD::new(electrs_exe.clone(), bitcoind, true, false).unwrap(); + let header = electrsd.client.block_headers_subscribe().unwrap(); + assert_eq!(header.height, 1); } } From 166f90cfb4a0858186ccd487d230dfa2293949d8 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 1 Jun 2021 11:56:07 +0200 Subject: [PATCH 2/6] update bitcoind version, introduce features to use new/old electrs --- Cargo.toml | 8 ++++++-- src/lib.rs | 57 ++++++++++++++++++++++++++++++++++-------------------- 2 files changed, 42 insertions(+), 23 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 10329f4..5dee4b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT" edition = "2018" [dependencies] -bitcoind = { version = "0.7.0" } +bitcoind = { git = "https://github.com/RCasatta/bitcoind", rev="ff0792a3c8317195a65662866727145fde577348" } electrum-client = { version="0.7.0", default-features = false } nix = { version="0.20.0", optional = true } log = "0.4.14" @@ -18,4 +18,8 @@ log = "0.4.14" env_logger = "0.8.3" [features] -trigger = ["nix"] \ No newline at end of file +trigger = ["nix"] + +# electrs features +p2p-enabled = [] # requires electrs version >= 0.8.11 +monitoring-disabled = [] # requires electrs version >= 0.8.11 built without default features \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index c5f1918..655bd0a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,8 +19,6 @@ use std::time::Duration; pub struct ElectrsD { /// Process child handle, used to terminate the process when this struct is dropped process: Child, - /// bitcoind process connected to this electrs - pub bitcoind: BitcoinD, /// Electrum client connected to the electrs process pub client: RawClient, /// DB directory, where electrs store indexes. It is kept in the struct so that @@ -61,7 +59,7 @@ impl ElectrsD { /// One block will be generated in bitcoind if in IBD pub fn new>( exe: S, - bitcoind: BitcoinD, + bitcoind: &BitcoinD, view_stderr: bool, http_enabled: bool, ) -> Result { @@ -90,22 +88,40 @@ impl ElectrsD { args.push("regtest"); args.push("--cookie-file"); - let cookie_file = format!("{}", bitcoind.cookie_file.display()); + let cookie_file = format!("{}", bitcoind.config.cookie_file.display()); args.push(&cookie_file); args.push("--daemon-rpc-addr"); - let rpc_socket = bitcoind.rpc_socket.to_string(); + let rpc_socket = bitcoind.config.rpc_socket.to_string(); args.push(&rpc_socket); - let p2p_socket = bitcoind - .p2p_socket - .ok_or(Error::BitcoinNodeHasNoP2P)? - .to_string(); - args.push("--daemon-p2p-addr"); - args.push(&p2p_socket); + #[cfg(feature = "p2p-enabled")] + let p2p_socket; + #[cfg(feature = "p2p-enabled")] + { + p2p_socket = bitcoind + .config + .p2p_socket + .ok_or(Error::BitcoinNodeHasNoP2P)? + .to_string(); + args.push("--daemon-p2p-addr"); + args.push(&p2p_socket); + } + + #[cfg(not(feature = "monitoring-disabled"))] + let monitoring_address; + #[cfg(not(feature = "monitoring-disabled"))] + { + monitoring_address = format!("0.0.0.0:{}", get_available_port()?); + args.push("--monitoring-addr"); + args.push(&monitoring_address); + } - //args.push("--daemon-dir"); - //let rpc_socket = bitcoind._work_dir.to_string(); + // `--daemon-dir` isn't necessary since we use `--jsonrpc-import` however better to see the + // correct value in the logs and it may be used in the future + args.push("--daemon-dir"); + let daemon_dir = format!("{}", bitcoind.config.datadir.display()); + args.push(&daemon_dir); args.push("--jsonrpc-import"); @@ -142,7 +158,6 @@ impl ElectrsD { Ok(ElectrsD { process, - bitcoind, client, _db_dir, electrum_url, @@ -213,12 +228,11 @@ mod test { let electrs_exe = env::var("ELECTRS_EXE").expect("ELECTRS_EXE env var must be set"); let bitcoind = BitcoinD::with_args(bitcoind_exe.clone(), vec![], true, bitcoind::P2P::Yes).unwrap(); - let electrsd = ElectrsD::new(electrs_exe.clone(), bitcoind, true, false).unwrap(); + let electrsd = ElectrsD::new(electrs_exe.clone(), &bitcoind, true, false).unwrap(); let header = electrsd.client.block_headers_subscribe().unwrap(); assert_eq!(header.height, 1); - let node_client = &electrsd.bitcoind.client; - let address = node_client.get_new_address(None, None).unwrap(); - node_client.generate_to_address(100, &address).unwrap(); + let address = bitcoind.client.get_new_address(None, None).unwrap(); + bitcoind.client.generate_to_address(100, &address).unwrap(); #[cfg(feature = "trigger")] electrsd.trigger().unwrap(); @@ -232,10 +246,11 @@ mod test { }; assert_eq!(header.height, 101); + // launch another instance to check there are no fixed port used - let bitcoind = BitcoinD::with_args(bitcoind_exe, vec![], true, bitcoind::P2P::Yes).unwrap(); - let electrsd = ElectrsD::new(electrs_exe.clone(), bitcoind, true, false).unwrap(); + let electrsd = ElectrsD::new(electrs_exe.clone(), &bitcoind, true, false).unwrap(); let header = electrsd.client.block_headers_subscribe().unwrap(); - assert_eq!(header.height, 1); + assert_eq!(header.height, 101); + } } From ca6c6e632c5d6d22f66a22ae314265dedb75344f Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 1 Jun 2021 11:57:00 +0200 Subject: [PATCH 3/6] add CI cosmetics --- .github/workflows/cont_integration.yml | 15 +++++++++++++++ src/lib.rs | 2 -- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index ac48056..8810676 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -49,3 +49,18 @@ jobs: run: curl https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VER/bitcoin-$BITCOIN_VER-x86_64-linux-gnu.tar.gz | tar -xvz bitcoin-$BITCOIN_VER/bin/bitcoind - name: Test electrsd run: RUST_LOG=electrs=debug cargo test --verbose ${{ matrix.features }} + + cosmetics: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + profile: minimal + components: rustfmt, clippy + - name: fmt + run: cargo fmt -- --check + - name: clippy + run: cargo clippy -- -D warnings diff --git a/src/lib.rs b/src/lib.rs index 655bd0a..133c9da 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -246,11 +246,9 @@ mod test { }; assert_eq!(header.height, 101); - // launch another instance to check there are no fixed port used let electrsd = ElectrsD::new(electrs_exe.clone(), &bitcoind, true, false).unwrap(); let header = electrsd.client.block_headers_subscribe().unwrap(); assert_eq!(header.height, 101); - } } From a101e26f82d2add2d26a0f8f13a2df6153017868 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Tue, 1 Jun 2021 21:54:30 +0200 Subject: [PATCH 4/6] features to allow electrs 0.8.10 and 0.8.11* --- .github/workflows/cont_integration.yml | 4 ++-- Cargo.toml | 5 +++-- src/lib.rs | 8 ++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/cont_integration.yml b/.github/workflows/cont_integration.yml index 8810676..e97d9d6 100644 --- a/.github/workflows/cont_integration.yml +++ b/.github/workflows/cont_integration.yml @@ -13,7 +13,7 @@ jobs: ELECTRS_VER: 0.8.10 strategy: matrix: - features: ["", "--features trigger"] + features: ["--features use-p2p", "--features use-p2p,trigger"] steps: - name: Checkout @@ -48,7 +48,7 @@ jobs: if: steps.cache-step.outputs.cache-hit != 'true' run: curl https://bitcoincore.org/bin/bitcoin-core-$BITCOIN_VER/bitcoin-$BITCOIN_VER-x86_64-linux-gnu.tar.gz | tar -xvz bitcoin-$BITCOIN_VER/bin/bitcoind - name: Test electrsd - run: RUST_LOG=electrs=debug cargo test --verbose ${{ matrix.features }} + run: RUST_LOG=electrs=debug cargo test --verbose --no-default-features ${{ matrix.features }} cosmetics: runs-on: ubuntu-20.04 diff --git a/Cargo.toml b/Cargo.toml index 5dee4b6..567feeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,8 +18,9 @@ log = "0.4.14" env_logger = "0.8.3" [features] +default = ["use-monitoring"] # mandatory in electrs <= 0.8.10 trigger = ["nix"] # electrs features -p2p-enabled = [] # requires electrs version >= 0.8.11 -monitoring-disabled = [] # requires electrs version >= 0.8.11 built without default features \ No newline at end of file +use-p2p = [] # required in electrs version > 0.8.10 +use-monitoring = [] # required in electrs version <= 0.8.10 built without default features \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 133c9da..9dfac3a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -95,9 +95,9 @@ impl ElectrsD { let rpc_socket = bitcoind.config.rpc_socket.to_string(); args.push(&rpc_socket); - #[cfg(feature = "p2p-enabled")] + #[cfg(feature = "use-p2p")] let p2p_socket; - #[cfg(feature = "p2p-enabled")] + #[cfg(feature = "use-p2p")] { p2p_socket = bitcoind .config @@ -108,9 +108,9 @@ impl ElectrsD { args.push(&p2p_socket); } - #[cfg(not(feature = "monitoring-disabled"))] + #[cfg(feature = "use-monitoring")] let monitoring_address; - #[cfg(not(feature = "monitoring-disabled"))] + #[cfg(feature = "use-monitoring")] { monitoring_address = format!("0.0.0.0:{}", get_available_port()?); args.push("--monitoring-addr"); From 9a8850202c163ef84bcba55419b3b161c6641e20 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Fri, 4 Jun 2021 09:28:45 +0200 Subject: [PATCH 5/6] bump bitcoind version --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 567feeb..30f2c75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ license = "MIT" edition = "2018" [dependencies] -bitcoind = { git = "https://github.com/RCasatta/bitcoind", rev="ff0792a3c8317195a65662866727145fde577348" } +bitcoind = "0.10.0" electrum-client = { version="0.7.0", default-features = false } nix = { version="0.20.0", optional = true } log = "0.4.14" From 648dd29782cc8511bd8a24d2b529163245685b33 Mon Sep 17 00:00:00 2001 From: Riccardo Casatta Date: Fri, 4 Jun 2021 09:29:40 +0200 Subject: [PATCH 6/6] re-export bitcoind --- src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 9dfac3a..9b1673a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,9 @@ use std::ffi::OsStr; use std::process::{Child, Command, Stdio}; use std::time::Duration; +// re-export bitcoind +pub use bitcoind; + /// Struct representing the bitcoind process with related information pub struct ElectrsD { /// Process child handle, used to terminate the process when this struct is dropped