Skip to content

replay: make scheduler depth configurable #8261

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ripatel-fd merged 1 commit into from
Feb 12, 2026
+123 −39
Merged

replay: make scheduler depth configurable #8261

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/app/firedancer-dev/commands/backtest.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,7 @@ extern int * fd_log_private_shared_lock;

static void
backtest_cmd_topo( config_t * config ) {
config->firedancer.development.replay.scheduler_depth = 8192UL;
backtest_topo( config );
}

Expand Down
7 changes: 7 additions & 0 deletions src/app/firedancer/config/default.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1944,3 +1944,10 @@ telemetry = true
# the layout will be ignored.
[development.snapshots]
disable_lthash_verification = true

[development.replay]
# scheduler_depth controls the number of transactions tracked by
# the replay scheduler. Larger values can result in faster
# replay times (particularly during heavy forking) at the cost
# of greater memory use.
scheduler_depth = 1048576
1 change: 1 addition & 0 deletions src/app/firedancer/topology.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1534,6 +1534,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
tile->replay.expected_shred_version = config->consensus.expected_shred_version;
tile->replay.wait_for_vote_to_start_leader = config->consensus.wait_for_vote_to_start_leader;

tile->replay.sched_depth = config->firedancer.development.replay.scheduler_depth;
tile->replay.max_live_slots = config->firedancer.runtime.max_live_slots;
tile->replay.write_delay_slots = config->firedancer.vinyl.write_delay_slots;

Expand Down
4 changes: 4 additions & 0 deletions src/app/shared/fd_config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,10 @@ struct fd_configf {

struct {
int hard_fork_fatal;

struct {
ulong scheduler_depth;
} replay;
} development;

struct {
Expand Down
1 change: 1 addition & 0 deletions src/app/shared/fd_config_parse.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ fd_config_extract_podf( uchar * pod,
CFG_POP ( uint, snapshots.min_download_speed_mibs );

CFG_POP ( bool, development.hard_fork_fatal );
CFG_POP ( ulong, development.replay.scheduler_depth );

return config;
}
Expand Down
1 change: 1 addition & 0 deletions src/disco/topo/fd_topo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,6 +397,7 @@ struct fd_topo_tile {
int wait_for_vote_to_start_leader;

ulong heap_size_gib;
ulong sched_depth;
ulong max_live_slots;
ulong write_delay_slots;

Expand Down
6 changes: 3 additions & 3 deletions src/discof/replay/fd_replay_tile.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ scratch_footprint( fd_topo_tile_t const * tile ) {
l = FD_LAYOUT_APPEND( l, fd_block_id_map_align(), fd_block_id_map_footprint( chain_cnt ) );
l = FD_LAYOUT_APPEND( l, fd_txncache_align(), fd_txncache_footprint( tile->replay.max_live_slots ) );
l = FD_LAYOUT_APPEND( l, fd_reasm_align(), fd_reasm_footprint( tile->replay.fec_max ) );
l = FD_LAYOUT_APPEND( l, fd_sched_align(), fd_sched_footprint( tile->replay.max_live_slots ) );
l = FD_LAYOUT_APPEND( l, fd_sched_align(), fd_sched_footprint( tile->replay.sched_depth, tile->replay.max_live_slots ) );
l = FD_LAYOUT_APPEND( l, fd_vinyl_req_pool_align(), fd_vinyl_req_pool_footprint( 1UL, 1UL ) );
l = FD_LAYOUT_APPEND( l, fd_vote_tracker_align(), fd_vote_tracker_footprint() );
l = FD_LAYOUT_APPEND( l, fd_capture_ctx_align(), fd_capture_ctx_footprint() );
Expand Down Expand Up @@ -2500,7 +2500,7 @@ unprivileged_init( fd_topo_t * topo,
void * block_id_map_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_block_id_map_align(), fd_block_id_map_footprint( chain_cnt ) );
void * _txncache = FD_SCRATCH_ALLOC_APPEND( l, fd_txncache_align(), fd_txncache_footprint( tile->replay.max_live_slots ) );
void * reasm_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_reasm_align(), fd_reasm_footprint( tile->replay.fec_max ) );
void * sched_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_sched_align(), fd_sched_footprint( tile->replay.max_live_slots ) );
void * sched_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_sched_align(), fd_sched_footprint( tile->replay.sched_depth, tile->replay.max_live_slots ) );
void * vinyl_req_pool_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_vinyl_req_pool_align(), fd_vinyl_req_pool_footprint( 1UL, 1UL ) );
void * vote_tracker_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_vote_tracker_align(), fd_vote_tracker_footprint() );
void * _capture_ctx = FD_SCRATCH_ALLOC_APPEND( l, fd_capture_ctx_align(), fd_capture_ctx_footprint() );
Expand Down Expand Up @@ -2626,7 +2626,7 @@ unprivileged_init( fd_topo_t * topo,
ctx->reasm = fd_reasm_join( fd_reasm_new( reasm_mem, tile->replay.fec_max, ctx->reasm_seed ) );
FD_TEST( ctx->reasm );

ctx->sched = fd_sched_join( fd_sched_new( sched_mem, tile->replay.max_live_slots, ctx->exec_cnt ), tile->replay.max_live_slots );
ctx->sched = fd_sched_join( fd_sched_new( sched_mem, tile->replay.sched_depth, tile->replay.max_live_slots, ctx->exec_cnt ) );
FD_TEST( ctx->sched );

FD_TEST( fd_vinyl_req_pool_new( vinyl_req_pool_mem, 1UL, 1UL ) );
Expand Down
113 changes: 81 additions & 32 deletions src/discof/replay/fd_sched.c
View file
Open in desktop

Large diffs are not rendered by default.

28 changes: 24 additions & 4 deletions src/discof/replay/fd_sched.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,9 @@
more ingest, more ready, more done ...
... */

#define FD_SCHED_MIN_DEPTH 478
#define FD_SCHED_MAX_DEPTH FD_RDISP_MAX_DEPTH

struct fd_sched;
typedef struct fd_sched fd_sched_t;

Expand Down Expand Up @@ -154,16 +157,33 @@ FD_PROTOTYPES_BEGIN

/* fd_sched_{align,footprint} return the required alignment and
footprint in bytes for a region of memory to be used as a scheduler.
footprint silently returns 0 if params are invalid (thus convenient
to validate params).

depth controls the reorder buffer transaction count (~1 million
recommended for live replay, ~10k recommended for async replay).
block_cnt_max is the maximum number of blocks that will be tracked by
the scheduler. */
ulong fd_sched_align ( void );
ulong fd_sched_footprint( ulong block_cnt_max );

ulong
fd_sched_align( void );

ulong
fd_sched_footprint( ulong depth, /* in [FD_SCHED_MIN_DEPTH,FD_SCHED_MAX_DEPTH] */
ulong block_cnt_max ); /* >= 1 */

/* fd_sched_new creates a sched object backed by the given memory region
(conforming to align() and footprint()). Returns NULL if any
parameter is invalid. */

void *
fd_sched_new( void * mem, ulong block_cnt_max, ulong exec_cnt );
fd_sched_new( void * mem,
ulong depth,
ulong block_cnt_max,
ulong exec_cnt );

fd_sched_t *
fd_sched_join( void * mem, ulong block_cnt_max );
fd_sched_join( void * mem );

/* Add the data in the FEC set to the scheduler. If is_last_fec is 1,
then this is the last FEC set in the block. Transactions may span
Expand Down
Loading