From 40ee895948c3d23a854e164396f1e75e87ea6f7d Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Fri, 26 Dec 2025 11:14:32 +0800 Subject: [PATCH 1/2] devnet 4-seq-cluster --- devnet/3-op-init.sh | 10 ++ devnet/4-op-start-service.sh | 6 +- devnet/docker-compose.yml | 147 +++++++++++++++++++++++++++++ devnet/scripts/active-sequencer.sh | 23 ++--- 4 files changed, 172 insertions(+), 14 deletions(-) diff --git a/devnet/3-op-init.sh b/devnet/3-op-init.sh index 2684f4d7..674420fa 100755 --- a/devnet/3-op-init.sh +++ b/devnet/3-op-init.sh @@ -181,9 +181,17 @@ if [ "$CONDUCTOR_ENABLED" = "true" ]; then OP_GETH_DATADIR2="$(pwd)/data/op-geth-seq2" rm -rf "$OP_GETH_DATADIR2" cp -r $OP_GETH_DATADIR $OP_GETH_DATADIR2 + + OP_GETH_DATADIR4="$(pwd)/data/op-geth-seq4" + rm -rf "$OP_GETH_DATADIR4" + cp -r $OP_GETH_DATADIR $OP_GETH_DATADIR4 elif [ "$SEQ_TYPE" = "reth" ]; then rm -rf "$OP_RETH_DATADIR2" cp -r $OP_RETH_DATADIR $OP_RETH_DATADIR2 + + OP_RETH_DATADIR4="$(pwd)/data/op-reth-seq4" + rm -rf "$OP_RETH_DATADIR4" + cp -r $OP_RETH_DATADIR $OP_RETH_DATADIR4 fi # op-seq3 default EL is always op-geth to ensure multiple seqs' geth and reth compatibilities @@ -196,9 +204,11 @@ if [ "$SEQ_TYPE" = "reth" ]; then echo -n "1aba031aeb5aa8aedadaf04159d20e7d58eeefb3280176c7d59040476c2ab21b" > $OP_RETH_DATADIR/discovery-secret if [ "$CONDUCTOR_ENABLED" = "true" ]; then echo -n "934ee1c6d37504aa6397b13348d2b5788a0bae5d3a77c71645f8b28be54590d9" > $OP_RETH_DATADIR2/discovery-secret + echo -n "a8b3c4d5e6f7081920a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8091a2b3c4d" > $OP_RETH_DATADIR4/discovery-secret if [ "$FLASHBLOCK_ENABLED" = "true" ]; then echo -n "60a4284707ef52c2b8486410be2bc7bf3bf803fcd85f0059b87b8b772eba62b421ef496e2a44135cfd9e74133e2e2b3e30a4a6c428d3f41e3537eea14eaf9ea3" > $OP_RETH_DATADIR/fb-p2p-key echo -n "6c899cb8b6dadfc34ddde60a57a61b3bdc655247a72feae16b851204fd41596f67a5e73ff50c90ec1755bcf640de7333322cce8612f722732f1244af23be007a" > $OP_RETH_DATADIR2/fb-p2p-key + echo -n "c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d23700" > $OP_RETH_DATADIR4/fb-p2p-key fi fi echo "✅ Set p2p nodekey for reth sequencer" diff --git a/devnet/4-op-start-service.sh b/devnet/4-op-start-service.sh index fb480cd6..79010a01 100755 --- a/devnet/4-op-start-service.sh +++ b/devnet/4-op-start-service.sh @@ -58,7 +58,7 @@ jq ".genesis.l2.hash = \"$NEW_BLOCK_HASH\"" config-op/rollup.json > config-op/ro mv config-op/rollup.json.tmp config-op/rollup.json if [ "$CONDUCTOR_ENABLED" = "true" ]; then - docker compose up -d op-conductor op-conductor2 op-conductor3 + docker compose up -d op-conductor op-conductor2 op-conductor3 op-conductor4 sleep 10 $SCRIPTS_DIR/active-sequencer.sh else @@ -77,8 +77,8 @@ fi if [ "$CONDUCTOR_ENABLED" = "true" ]; then echo "🔧 Configuring op-batcher for conductor mode with conductor RPC endpoints..." # Set conductor mode endpoints - export OP_BATCHER_L2_ETH_RPC="http://op-conductor:8547,http://op-conductor2:8547,http://op-conductor3:8547" - export OP_BATCHER_ROLLUP_RPC="http://op-conductor:8547,http://op-conductor2:8547,http://op-conductor3:8547" + export OP_BATCHER_L2_ETH_RPC="http://op-conductor:8547,http://op-conductor2:8547,http://op-conductor3:8547,http://op-conductor4:8547" + export OP_BATCHER_ROLLUP_RPC="http://op-conductor:8547,http://op-conductor2:8547,http://op-conductor3:8547,http://op-conductor4:8547" echo "✅ op-batcher configured for conductor mode (connecting to conductor RPC endpoints)" else echo "🔧 Configuring op-batcher for single sequencer mode..." diff --git a/devnet/docker-compose.yml b/devnet/docker-compose.yml index 7f3a5330..131f59e3 100644 --- a/devnet/docker-compose.yml +++ b/devnet/docker-compose.yml @@ -845,6 +845,153 @@ services: op-seq3: condition: service_healthy + op-geth-seq4: + image: "${OP_GETH_IMAGE_TAG}" + container_name: op-geth-seq4 + entrypoint: geth + volumes: + - ./data/op-geth-seq4:/datadir + - ./config-op/jwt.txt:/jwt.txt + - ./config-op/test.geth.seq.config.toml:/config.toml + ports: + - "8423:8545" + - "30307:30303" + - "30307:30303/udp" + command: + - --networkid=${CHAIN_ID} + - --verbosity=3 + - --datadir=/datadir + - --db.engine=${DB_ENGINE} + - --config=/config.toml + - --gcmode=archive + - --rollup.disabletxpoolgossip=false + - --nodekeyhex=a8b3c4d5e6f7081920a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8091a2b3c4d + healthcheck: + test: ["CMD", "wget", "--spider", "--quiet", "http://localhost:8545"] + interval: 3s + timeout: 3s + retries: 10 + start_period: 3s + networks: + default: + aliases: + - op-seq-el4 + + op-reth-seq4: + image: "${OP_RETH_IMAGE_TAG}" + container_name: op-reth-seq4 + entrypoint: /entrypoint/reth-seq.sh 4 + env_file: + - ./.env + volumes: + - ./data/op-reth-seq4:/datadir + - ./config-op/jwt.txt:/jwt.txt + - ./config-op/genesis-reth.json:/genesis.json + - ./entrypoint:/entrypoint + - ./.env:/.env + ports: + - "8423:8545" + - "30307:30303" + - "30307:30303/udp" + - "11114:1111" # outbound flashblocks ws port + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:8545" ] + interval: 3s + timeout: 3s + retries: 10 + start_period: 3s + networks: + default: + aliases: + - op-seq-el4 + + op-seq4: + image: "${OP_STACK_IMAGE_TAG}" + container_name: op-seq4 + volumes: + - ./config-op/rollup.json:/rollup.json + - ./config-op/jwt.txt:/jwt.txt + - ./data/op-seq4:/data + - ./l1-geth/execution/genesis.json:/l1-genesis.json + ports: + - "9548:9545" + - "9226:9223" + - "9226:9223/udp" + command: + - /app/op-node/bin/op-node + - --log.level=info + - --log.format=logfmtms + - --l2=http://op-${SEQ_TYPE}-seq4:8552 + - --l2.jwt-secret=/jwt.txt + - --sequencer.enabled + - --sequencer.stopped + - --sequencer.l1-confs=5 + - --verifier.l1-confs=1 + - --l1.epoch-poll-interval=10s + - --rollup.config=/rollup.json + - --rpc.addr=0.0.0.0 + - --p2p.listen.tcp=9223 + - --p2p.listen.udp=9223 + - --p2p.priv.raw=b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2 + - --p2p.peerstore.path=/data/p2p/opnode_peerstore_db + - --p2p.discovery.path=/data/p2p/opnode_discovery_db + - --p2p.static=/dns4/op-seq/tcp/9223/p2p/16Uiu2HAkzHdkbmS2VrCsccLibsu7MvGHpmFUMJnMTkKifrtS5m65,/dns4/op-seq2/tcp/9223/p2p/16Uiu2HAmDTjVuEF6V9DccV1JhrHg7DYc5SKm3bw2T75kAFPsGuSp,/dns4/op-seq3/tcp/9223/p2p/16Uiu2HAmRDGMm3UUrP8CfQ3YQo9aaXEXFXA7LeFNztdPMNK5moyD + - --p2p.no-discovery + - --rpc.enable-admin + - --p2p.sequencer.key=${SEQUENCER_P2P_KEY} + - --l1=${L1_RPC_URL_IN_DOCKER} + - --l1.beacon=${L1_BEACON_URL_IN_DOCKER} + - --l1.rpckind=standard + - --rollup.l1-chain-config=/l1-genesis.json + - --safedb.path=/data/safedb + - --conductor.enabled=${CONDUCTOR_ENABLED:-false} + - --conductor.rpc=http://op-conductor4:8547 + - --l2.enginekind=${SEQ_TYPE} + depends_on: + - op-${SEQ_TYPE}-seq4 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9545"] + interval: 3s + timeout: 3s + retries: 10 + start_period: 3s + + op-conductor4: + image: "${OP_STACK_IMAGE_TAG}" + container_name: op-conductor4 + volumes: + - ./data/op-conductor4:/data + - ./config-op/rollup.json:/rollup.json + ports: + - "8550:8547" # RPC port + - "50053:50050" # Consensus port + command: + - /app/op-conductor/bin/op-conductor + - --log.level=debug + # already existed service + - --node.rpc=http://op-seq4:9545 + - --execution.rpc=http://op-${SEQ_TYPE}-seq4:8545 + # Raft Config + - --raft.server.id=conductor-4 + - --raft.storage.dir=/data/raft + - --raft.bootstrap=false + - --consensus.addr=0.0.0.0 + - --consensus.port=50050 + - --consensus.advertised=op-conductor4:50050 + # RPC Config + - --rpc.addr=0.0.0.0 + - --rpc.port=8547 + - --rpc.enable-proxy=true + # Healthcheck Config + - --healthcheck.interval=1 + - --healthcheck.unsafe-interval=3 + - --healthcheck.min-peer-count=1 + - --rollup.config=/rollup.json + - --rpc.http-body-limit-mb=64 + depends_on: + op-seq4: + condition: service_healthy + mempool-rebroadcaster: image: xlayerdev/mempool-rebroadcaster container_name: mempool-rebroadcaster diff --git a/devnet/scripts/active-sequencer.sh b/devnet/scripts/active-sequencer.sh index 660143af..d8e07078 100755 --- a/devnet/scripts/active-sequencer.sh +++ b/devnet/scripts/active-sequencer.sh @@ -4,10 +4,10 @@ detect_leader() { echo "Detecting leader conductor..." - # Check all three conductors to find the leader - for i in 1 2 3; do - CONDUCTOR_PORT=$((8546 + i)) # 8547, 8548, 8549 - SEQUENCER_PORT=$((9544 + i)) # 9545, 9546, 9547 + # Check all four conductors to find the leader + for i in 1 2 3 4; do + CONDUCTOR_PORT=$((8546 + i)) # 8547, 8548, 8549, 8550 + SEQUENCER_PORT=$((9544 + i)) # 9545, 9546, 9547, 9548 IS_LEADER=$(curl -sS -X POST -H "Content-Type: application/json" \ --data '{"jsonrpc":"2.0","method":"conductor_leader","params":[],"id":1}' \ @@ -32,8 +32,8 @@ detect_leader # 1. check connected peers CONNECTED=$(curl -sS -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"opp2p_peerStats","params":[],"id":1}' http://localhost:$LEADER_SEQUENCER_PORT | jq .result.connected) -if (( CONNECTED < 2 )); then - echo "$CONNECTED peers connected, which is less than 2" +if (( CONNECTED < 3 )); then + echo "$CONNECTED peers connected, which is less than 3" echo 1 fi @@ -76,16 +76,17 @@ fi echo "Sequencer successfully activated" -# 5. try to add other two conductors to raft consensus cluster +# 5. try to add other three conductors to raft consensus cluster SERVER_COUNT=$(curl -sS -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_clusterMembership","params":[],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT | jq '.result.servers | length') -if (( $SERVER_COUNT < 3 )); then +if (( $SERVER_COUNT < 4 )); then curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_addServerAsVoter","params":["conductor-2", "op-conductor2:50050", 0],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_addServerAsVoter","params":["conductor-3", "op-conductor3:50050", 0],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT + curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_addServerAsVoter","params":["conductor-4", "op-conductor4:50050", 0],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT SERVER_COUNT=$(curl -sS -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_clusterMembership","params":[],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT | jq '.result.servers | length') - if (( $SERVER_COUNT != 3 )); then - echo "unexpected server count, expected: 3, real: $SERVER_COUNT" + if (( $SERVER_COUNT != 4 )); then + echo "unexpected server count, expected: 4, real: $SERVER_COUNT" exit 1 fi - echo "add 2 new voters to raft consensus cluster successfully!" + echo "add 3 new voters to raft consensus cluster successfully!" fi From d67828db8b1650824c9ff256ea80edf5491f35b5 Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Fri, 26 Dec 2025 11:35:07 +0800 Subject: [PATCH 2/2] add config for voter/nonvoter --- devnet/example.env | 5 ++++ devnet/scripts/active-sequencer.sh | 37 ++++++++++++++++++++++++++---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/devnet/example.env b/devnet/example.env index ac35584a..ecb89bda 100644 --- a/devnet/example.env +++ b/devnet/example.env @@ -165,6 +165,11 @@ NEW_BLOCK_HASH=0xddb9bdc86631494bab4b4749c4575035e2383da7c96d32d31341de862b1dd6c DB_ENGINE="pebble" CONDUCTOR_ENABLED=false +# Conductor voter configuration (true=voter, false=nonvoter) +# conductor-1 is always the bootstrap node and leader +CONDUCTOR2_VOTER=true +CONDUCTOR3_VOTER=true +CONDUCTOR4_VOTER=true LAUNCH_RPC_NODE=true # Trusted peers enode url configurations for the shared private sequencer txpool diff --git a/devnet/scripts/active-sequencer.sh b/devnet/scripts/active-sequencer.sh index d8e07078..68a723d4 100755 --- a/devnet/scripts/active-sequencer.sh +++ b/devnet/scripts/active-sequencer.sh @@ -1,5 +1,31 @@ #!/bin/bash +# Load environment variables +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ENV_FILE="$(dirname "$SCRIPT_DIR")/.env" +source "$ENV_FILE" + +# Function to add conductor to cluster (voter or nonvoter based on config) +add_conductor_to_cluster() { + local conductor_id=$1 + local conductor_name=$2 + local conductor_addr=$3 + local is_voter_var="CONDUCTOR${conductor_id}_VOTER" + local is_voter="${!is_voter_var:-true}" # default to voter if not set + + if [ "$is_voter" = "true" ]; then + echo "Adding $conductor_name as voter..." + curl -X POST -H "Content-Type: application/json" \ + --data "{\"jsonrpc\":\"2.0\",\"method\":\"conductor_addServerAsVoter\",\"params\":[\"$conductor_name\", \"$conductor_addr\", 0],\"id\":1}" \ + http://localhost:$LEADER_CONDUCTOR_PORT + else + echo "Adding $conductor_name as nonvoter..." + curl -X POST -H "Content-Type: application/json" \ + --data "{\"jsonrpc\":\"2.0\",\"method\":\"conductor_addServerAsNonvoter\",\"params\":[\"$conductor_name\", \"$conductor_addr\", 0],\"id\":1}" \ + http://localhost:$LEADER_CONDUCTOR_PORT + fi +} + # Function to detect leader conductor and set ports detect_leader() { echo "Detecting leader conductor..." @@ -79,14 +105,17 @@ echo "Sequencer successfully activated" # 5. try to add other three conductors to raft consensus cluster SERVER_COUNT=$(curl -sS -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_clusterMembership","params":[],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT | jq '.result.servers | length') if (( $SERVER_COUNT < 4 )); then - curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_addServerAsVoter","params":["conductor-2", "op-conductor2:50050", 0],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT - curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_addServerAsVoter","params":["conductor-3", "op-conductor3:50050", 0],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT - curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_addServerAsVoter","params":["conductor-4", "op-conductor4:50050", 0],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT + add_conductor_to_cluster 2 "conductor-2" "op-conductor2:50050" + add_conductor_to_cluster 3 "conductor-3" "op-conductor3:50050" + add_conductor_to_cluster 4 "conductor-4" "op-conductor4:50050" SERVER_COUNT=$(curl -sS -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"conductor_clusterMembership","params":[],"id":1}' http://localhost:$LEADER_CONDUCTOR_PORT | jq '.result.servers | length') if (( $SERVER_COUNT != 4 )); then echo "unexpected server count, expected: 4, real: $SERVER_COUNT" exit 1 fi - echo "add 3 new voters to raft consensus cluster successfully!" + echo "add 3 new conductors to raft consensus cluster successfully!" + echo " CONDUCTOR2_VOTER=${CONDUCTOR2_VOTER:-true}" + echo " CONDUCTOR3_VOTER=${CONDUCTOR3_VOTER:-true}" + echo " CONDUCTOR4_VOTER=${CONDUCTOR4_VOTER:-true}" fi