Skip to content

feat: Custom Table concurrency #2346

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bbernays wants to merge 2 commits into from
Closed

feat: Custom Table concurrency #2346

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
78221e7
Update table.go
bbernays Dec 9, 2025
972e100
Update scheduler_dfs.go
bbernays Dec 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 22 additions & 4 deletions scheduler/scheduler_dfs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,17 @@ func (s *syncClient) resolveResourcesDfs(ctx context.Context, table *schema.Tabl
chunks = lo.Chunk(resourcesSlice, table.PreResourceChunkResolver.ChunkSize)
}
for i := range chunks {
resourceConcurrencyKey := table.Name + "-" + client.ID() + "-" + "resource"
resourceSemVal, _ := s.scheduler.singleTableConcurrency.LoadOrStore(resourceConcurrencyKey, semaphore.NewWeighted(s.scheduler.singleResourceMaxConcurrency))
resourceConcurrencyKey := table.Name
if table.ConcurrencySettings != nil && lo.FromPtr(table.ConcurrencySettings.ConcurrencyKey) != "" {
resourceConcurrencyKey = lo.FromPtr(table.ConcurrencySettings.ConcurrencyKey)
}
resourceConcurrencyKey = resourceConcurrencyKey + "-" + client.ID() + "-" + "resource"
resourceConcurrency := s.scheduler.singleResourceMaxConcurrency
if table.ConcurrencySettings != nil && lo.FromPtr(table.ConcurrencySettings.MaxResourceConcurrency) > 0 {
resourceConcurrency = int64(lo.FromPtr(table.ConcurrencySettings.MaxResourceConcurrency))
}

resourceSemVal, _ := s.scheduler.singleTableConcurrency.LoadOrStore(resourceConcurrencyKey, semaphore.NewWeighted(resourceConcurrency))
resourceSem := resourceSemVal.(*semaphore.Weighted)
if err := resourceSem.Acquire(ctx, 1); err != nil {
s.logger.Warn().Err(err).Msg("failed to acquire semaphore. context cancelled")
Expand Down Expand Up @@ -227,9 +236,18 @@ func (s *syncClient) resolveResourcesDfs(ctx context.Context, table *schema.Tabl
resolvedResources <- resource
for _, relation := range resource.Table.Relations {
relation := relation
tableConcurrencyKey := table.Name + "-" + client.ID()
tableConcurrencyKey := table.Name
if table.ConcurrencySettings != nil && lo.FromPtr(table.ConcurrencySettings.ConcurrencyKey) != "" {
tableConcurrencyKey = lo.FromPtr(table.ConcurrencySettings.ConcurrencyKey)
}
tableConcurrencyKey = tableConcurrencyKey + "-" + client.ID()
tableConcurrency := s.scheduler.singleNestedTableMaxConcurrency
if table.ConcurrencySettings != nil && lo.FromPtr(table.ConcurrencySettings.MaxTableConcurrency) > 0 {
tableConcurrency = int64(lo.FromPtr(table.ConcurrencySettings.MaxTableConcurrency))
}

// Acquire the semaphore for the table
tableSemVal, _ := s.scheduler.singleTableConcurrency.LoadOrStore(tableConcurrencyKey, semaphore.NewWeighted(s.scheduler.singleNestedTableMaxConcurrency))
tableSemVal, _ := s.scheduler.singleTableConcurrency.LoadOrStore(tableConcurrencyKey, semaphore.NewWeighted(tableConcurrency))
tableSem := tableSemVal.(*semaphore.Weighted)
if err := tableSem.Acquire(ctx, 1); err != nil {
// This means context was cancelled
Expand Down
8 changes: 8 additions & 0 deletions schema/table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,14 @@ type Table struct {

// IgnorePKComponentsMismatchValidation is a flag that indicates if the table should skip validating usage of both primary key components and primary keys
IgnorePKComponentsMismatchValidation bool `json:"ignore_pk_components_mismatch_validation"`

// ConcurrencySettings Enables configuring concurrency settings for specific table. This provides a mechanism for irregular APIs to have unique settings either because it shares a common rate limit pool or has different limits than the default settings.
ConcurrencySettings *ConcurrencySettings `json:"concurrency_settings,omitempty"`
}
type ConcurrencySettings struct {
ConcurrencyKey *string `json:"concurrency_key,omitempty"`
MaxResourceConcurrency *int `json:"max_resource_concurrency,omitempty"`
MaxTableConcurrency *int `json:"max_table_concurrency,omitempty"`
}

var (
Expand Down