Bump test framework to v30.0

.github/workflows/test.yml

@@ -4,7 +4,7 @@ on:
   pull_request:
   push:
     branches:
-      - main
+      - '**'
 
 env:
   PYTHON_VERSION: "3.12"

resources/charts/bitcoincore/templates/servicemonitor.yaml

@@ -9,6 +9,7 @@ metadata:
 spec:
   endpoints:
     - port: prometheus-metrics
+      interval: {{ .Values.metricsScrapeInterval | default "15s" }}
   selector:
     matchLabels:
       app: {{ include "bitcoincore.fullname" . }}

resources/images/bitcoin/Dockerfile.dev

@@ -1,5 +1,5 @@
 # Setup deps stage
-FROM alpine AS deps
+FROM alpine:3.20 AS deps
 ARG REPO
 ARG COMMIT_SHA
 ARG BUILD_ARGS
@@ -20,7 +20,8 @@ RUN --mount=type=cache,target=/var/cache/apk \
     libtool \
     linux-headers \
     sqlite-dev \
-    zeromq-dev
+    zeromq-dev \
+    capnproto-dev
 
 COPY isroutable.patch /tmp/
 COPY addrman.patch /tmp/
@@ -51,7 +52,7 @@ RUN set -ex \
     && rm -f ${BITCOIN_PREFIX}/lib/libbitcoinconsensus.so.0.0.0
 
 # Final clean stage
-FROM alpine
+FROM alpine:3.20
 ARG UID=100
 ARG GID=101
 ENV BITCOIN_DATA=/root/.bitcoin
@@ -68,7 +69,8 @@ RUN --mount=type=cache,target=/var/cache/apk sed -i 's/http\:\/\/dl-cdn.alpineli
     libzmq \
     shadow \
     sqlite-dev \
-    su-exec
+    su-exec \
+    capnproto-dev
 
 COPY --from=build /opt/bitcoin /usr/local
 COPY entrypoint.sh /

resources/images/exporter/bitcoin-exporter.py

@@ -45,6 +45,13 @@ def make_metric_function(cmd):
         return None
 
 
+def make_counting_function(cmd, key, value):
+    try:
+        return lambda: sum(1 for x in eval(f"rpc.{cmd}") if x.get(key) == value)
+    except Exception:
+        return None
+
+
 # Parse RPC queries into metrics
 commands = METRICS.split(" ")
 for labeled_cmd in commands:
@@ -53,7 +60,12 @@ def make_metric_function(cmd):
     label, cmd = labeled_cmd.strip().split("=")
     # label, description i.e. ("bitcoin_conn_in", "Number of connections in")
     metric = Gauge(label, cmd)
-    metric.set_function(make_metric_function(cmd))
+    if "COUNT:" in cmd:
+        _, args = cmd.split(":")
+        cmd, key, value = args.split(",")
+        metric.set_function(make_counting_function(cmd, key, value))
+    else:
+        metric.set_function(make_metric_function(cmd))
     print(f"Metric created: {labeled_cmd}")
 
 # Start the server

resources/scenarios/commander.py

@@ -11,6 +11,7 @@
 import sys
 import tempfile
 import threading
+import types
 from time import sleep
 
 from kubernetes import client, config
@@ -240,6 +241,10 @@ def setup(self):
         self.lns: dict[str, LNNode] = {}
         self.channels = WARNET["channels"]
 
+        self.binary_paths = types.SimpleNamespace()
+        self.binary_paths.bitcoin_cmd = None
+        self.binary_paths.bitcoind = None
+
         for i, tank in enumerate(WARNET["tanks"]):
             self.log.info(
                 f"Adding TestNode #{i} from pod {tank['tank']} with IP {tank['rpc_host']}"
@@ -251,13 +256,12 @@ def setup(self):
                 rpchost=tank["rpc_host"],
                 timewait=60,
                 timeout_factor=self.options.timeout_factor,
-                bitcoind=None,
-                bitcoin_cli=None,
+                binaries=self.get_binaries(),
                 cwd=self.options.tmpdir,
                 coverage_dir=self.options.coveragedir,
             )
             node.tank = tank["tank"]
-            node.rpc = get_rpc_proxy(
+            node._rpc = get_rpc_proxy(
                 f"http://{tank['rpc_user']}:{tank['rpc_password']}@{tank['rpc_host']}:{tank['rpc_port']}",
                 i,
                 timeout=60,
@@ -317,7 +321,7 @@ def setup(self):
                 except Exception as e:
                     self.log.info(f"Failed to get signet network magic bytes from {node.tank}: {e}")
 
-    def parse_args(self):
+    def parse_args(self, _):
         # Only print "outer" args from parent class when using --help
         help_parser = argparse.ArgumentParser(usage="%(prog)s [options]")
         self.add_options(help_parser)
@@ -450,6 +454,12 @@ def parse_args(self):
             action="store_true",
             help="use BIP324 v2 connections between all nodes by default",
         )
+        parser.add_argument(
+            "--test_methods",
+            dest="test_methods",
+            nargs="*",
+            help="Run specified test methods sequentially instead of the full test. Use only for methods that do not depend on any context set up in run_test or other methods.",
+        )
 
         self.add_options(parser)
         # Running TestShell in a Jupyter notebook causes an additional -f argument
@@ -565,7 +575,7 @@ def connect_nodes(self, a, b, *, peer_advertises_v2=None, wait_for_connect: bool
 
     def generatetoaddress(self, generator, n, addr, sync_fun=None, **kwargs):
         if generator.chain == "regtest":
-            blocks = generator.generatetoaddress(n, addr, invalid_call=False, **kwargs)
+            blocks = generator.generatetoaddress(n, addr, called_by_framework=True, **kwargs)
             sync_fun() if sync_fun else self.sync_all()
             return blocks
         if generator.chain == "signet":
@@ -591,7 +601,7 @@ def bcli(method, *args, **kwargs):
                 ]
                 cbtx.vout = [CTxOut(tmpl["coinbasevalue"], reward_spk)]
                 cbtx.vin[0].nSequence = 2**32 - 2
-                cbtx.rehash()
+
                 # assemble block
                 block = CBlock()
                 block.nVersion = tmpl["version"]
@@ -616,26 +626,25 @@ def bcli(method, *args, **kwargs):
                 txs[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(SIGNET_HEADER)
                 hashes = []
                 for tx in txs:
-                    tx.rehash()
-                    hashes.append(ser_uint256(tx.sha256))
+                    hashes.append(ser_uint256(tx.txid_int))
                 mroot = block.get_merkle_root(hashes)
                 sd = b""
                 sd += struct.pack("<i", block.nVersion)
                 sd += ser_uint256(block.hashPrevBlock)
                 sd += ser_uint256(mroot)
                 sd += struct.pack("<I", block.nTime)
                 to_spend = CTransaction()
-                to_spend.nVersion = 0
+                to_spend.version = 0
                 to_spend.nLockTime = 0
                 to_spend.vin = [
                     CTxIn(COutPoint(0, 0xFFFFFFFF), b"\x00" + CScriptOp.encode_op_pushdata(sd), 0)
                 ]
                 to_spend.vout = [CTxOut(0, signet_spk_bin)]
-                to_spend.rehash()
+
                 spend = CTransaction()
-                spend.nVersion = 0
+                spend.version = 0
                 spend.nLockTime = 0
-                spend.vin = [CTxIn(COutPoint(to_spend.sha256, 0), b"", 0)]
+                spend.vin = [CTxIn(COutPoint(to_spend.txid_int, 0), b"", 0)]
                 spend.vout = [CTxOut(0, b"\x6a")]
                 # create PSBT for miner wallet signing
                 psbt = PSBT()
@@ -670,7 +679,7 @@ def bcli(method, *args, **kwargs):
                 signed_block.vtx[0].vout[-1].scriptPubKey += CScriptOp.encode_op_pushdata(
                     SIGNET_HEADER + signet_solution
                 )
-                signed_block.vtx[0].rehash()
+
                 signed_block.hashMerkleRoot = signed_block.calc_merkle_root()
                 try:
                     headhex = CBlockHeader.serialize(signed_block).hex()
@@ -690,7 +699,7 @@ def bcli(method, *args, **kwargs):
                         raise Exception(newheadhex)
                     newhead = from_hex(CBlockHeader(), newheadhex.strip())
                     signed_block.nNonce = newhead.nNonce
-                    signed_block.rehash()
+
                 except Exception as e:
                     self.log.info(
                         f"Error grinding signet PoW with bitcoin-util in {generator.tank}: {e}".strip()
@@ -699,7 +708,7 @@ def bcli(method, *args, **kwargs):
                     signed_block.solve()
                 # submit block
                 bcli("submitblock", signed_block.serialize().hex())
-                block_hashes.append(signed_block.hash)
+                block_hashes.append(signed_block.hash_hex)
                 mined_blocks += 1
                 self.log.info(f"Generated {mined_blocks} signet blocks")
 

resources/scenarios/ln_init.py

@@ -511,7 +511,7 @@ def matching_graph(self, expected, ln):
 
 
 def main():
-    LNInit().main()
+    LNInit("").main()
 
 
 if __name__ == "__main__":

resources/scenarios/miner_std.py

@@ -75,7 +75,7 @@ def run_test(self):
 
 
 def main():
-    MinerStd().main()
+    MinerStd("").main()
 
 
 if __name__ == "__main__":

resources/scenarios/reconnaissance.py

@@ -68,7 +68,14 @@ def run_test(self):
         # unusual or non-standard messages to a "victim" node.
         self.log.info(f"Attacking {dstaddr}:{dstport}")
         attacker = P2PInterface()
-        attacker.peer_connect(dstaddr=dstaddr, dstport=dstport, net=chain, timeout_factor=1)()
+        attacker.peer_connect(
+            dstaddr=dstaddr,
+            dstport=dstport,
+            net=chain,
+            timeout_factor=1,
+            send_version=True,
+            supports_v2_p2p=False,
+        )()
         attacker.wait_until(lambda: attacker.is_connected, check_connected=False)
 
         # Send a harmless network message we expect a response to and wait for it
@@ -81,7 +88,7 @@ def run_test(self):
 
 
 def main():
-    Reconnaissance().main()
+    Reconnaissance("").main()
 
 
 if __name__ == "__main__":