From 4c6200f7c28bffb1fddb08acac4c9673308f7e3d Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Fri, 23 Jan 2026 12:59:08 -0600 Subject: [PATCH] pkg/settings: add yaml support --- go.mod | 2 +- go.sum | 4 +- pkg/settings/cresettings/defaults.yaml | 71 ++++++++ pkg/settings/cresettings/settings_test.go | 19 ++ pkg/settings/json.go | 2 +- pkg/settings/testdata/config.yaml | 32 ++++ pkg/settings/testdata/yaml/global.yaml | 4 + pkg/settings/testdata/yaml/org/123.yaml | 4 + pkg/settings/testdata/yaml/org/456.yaml | 4 + ...26b4aa7e57ca7b68ae1bf45653f56b656fd3a.yaml | 4 + ...112d3f8f92e41c861939545ad387307af9703.yaml | 4 + ...804bc4af13855687559d7ff6552ac6dbb2ce0.yaml | 4 + ...804bc4af1385568f9b3363f6552ac6dbb2cef.yaml | 4 + pkg/settings/yaml.go | 163 ++++++++++++++++++ pkg/settings/yaml_test.go | 74 ++++++++ 15 files changed, 391 insertions(+), 4 deletions(-) create mode 100644 pkg/settings/cresettings/defaults.yaml create mode 100644 pkg/settings/testdata/config.yaml create mode 100644 pkg/settings/testdata/yaml/global.yaml create mode 100644 pkg/settings/testdata/yaml/org/123.yaml create mode 100644 pkg/settings/testdata/yaml/org/456.yaml create mode 100644 pkg/settings/testdata/yaml/owner/00026b4aa7e57ca7b68ae1bf45653f56b656fd3a.yaml create mode 100644 pkg/settings/testdata/yaml/owner/8bd112d3f8f92e41c861939545ad387307af9703.yaml create mode 100644 pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0.yaml create mode 100644 pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef.yaml create mode 100644 pkg/settings/yaml.go create mode 100644 pkg/settings/yaml_test.go diff --git a/go.mod b/go.mod index c18f98adb..0e9a998bd 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/go-json-experiment/json v0.0.0-20250223041408-d3c622f1b874 github.com/go-playground/validator/v10 v10.26.0 github.com/go-viper/mapstructure/v2 v2.4.0 + github.com/goccy/go-yaml v1.19.2 github.com/golang-jwt/jwt/v5 v5.2.3 github.com/google/go-cmp v0.7.0 github.com/google/uuid v1.6.0 @@ -99,7 +100,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/goccy/go-yaml v1.12.0 // indirect github.com/gofrs/uuid v4.4.0+incompatible // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/flatbuffers v25.2.10+incompatible // indirect diff --git a/go.sum b/go.sum index 7de2285a3..68c0f4634 100644 --- a/go.sum +++ b/go.sum @@ -94,8 +94,8 @@ github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9L github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM= -github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA= github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= diff --git a/pkg/settings/cresettings/defaults.yaml b/pkg/settings/cresettings/defaults.yaml new file mode 100644 index 000000000..cb8d154e0 --- /dev/null +++ b/pkg/settings/cresettings/defaults.yaml @@ -0,0 +1,71 @@ +WorkflowLimit: "200" +WorkflowExecutionConcurrencyLimit: "200" +GatewayIncomingPayloadSizeLimit: 1mb +GatewayVaultManagementEnabled: "true" +VaultCiphertextSizeLimit: 2kb +VaultIdentifierKeySizeLimit: 64b +VaultIdentifierOwnerSizeLimit: 64b +VaultIdentifierNamespaceSizeLimit: 64b +VaultPluginBatchSizeLimit: "20" +VaultRequestBatchSizeLimit: "10" +PerOrg: + ZeroBalancePruningTimeout: 24h0m0s +PerOwner: + WorkflowExecutionConcurrencyLimit: "5" + VaultSecretsLimit: "100" +PerWorkflow: + TriggerRegistrationsTimeout: 10s + TriggerSubscriptionTimeout: 15s + TriggerSubscriptionLimit: "10" + TriggerEventQueueLimit: "50" + TriggerEventQueueTimeout: 10m0s + CapabilityConcurrencyLimit: "30" + CapabilityCallTimeout: 3m0s + SecretsConcurrencyLimit: "5" + ExecutionConcurrencyLimit: "5" + ExecutionTimeout: 5m0s + ExecutionResponseLimit: 100kb + WASMMemoryLimit: 100mb + WASMBinarySizeLimit: 100mb + WASMCompressedBinarySizeLimit: 20mb + WASMConfigSizeLimit: 1mb + WASMSecretsSizeLimit: 1mb + LogLineLimit: 1kb + LogEventLimit: "1000" + ChainAllowed: + Default: "false" + Values: + "12922642891491394802": "true" + "3379446385462418246": "true" + CRONTrigger: + FastestScheduleInterval: 30s + HTTPTrigger: + RateLimit: every30s:3 + LogTrigger: + EventRateLimit: every6s:10 + EventSizeLimit: 5kb + FilterAddressLimit: "5" + FilterTopicsPerSlotLimit: "10" + ChainWrite: + TargetsLimit: "10" + ReportSizeLimit: 5kb + EVM: + TransactionGasLimit: "5000000" + GasLimit: + Default: "5000000" + Values: + "12922642891491394802": "50000000" + "3379446385462418246": "10000000" + ChainRead: + CallLimit: "10" + LogQueryBlockLimit: "100" + PayloadSizeLimit: 5kb + Consensus: + ObservationSizeLimit: 100kb + CallLimit: "20" + HTTPAction: + CallLimit: "5" + CacheAgeLimit: 10m0s + ConnectionTimeout: 10s + RequestSizeLimit: 10kb + ResponseSizeLimit: 100kb diff --git a/pkg/settings/cresettings/settings_test.go b/pkg/settings/cresettings/settings_test.go index 916f17f15..d76214b4e 100644 --- a/pkg/settings/cresettings/settings_test.go +++ b/pkg/settings/cresettings/settings_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/goccy/go-yaml" "github.com/pelletier/go-toml/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -34,6 +35,8 @@ var ( defaultsJSON string //go:embed defaults.toml defaultsTOML string + //go:embed defaults.yaml + defaultsYAML string ) func TestDefault(t *testing.T) { @@ -60,6 +63,22 @@ func TestDefault(t *testing.T) { require.Equal(t, defaultsTOML, string(b)) } }) + + t.Run("yaml", func(t *testing.T) { + jb, err := json.MarshalIndent(Default, "", "\t") + if err != nil { + log.Fatal(err) + } + b, err := yaml.JSONToYAML(jb) + if err != nil { + log.Fatal(err) + } + if *update { + require.NoError(t, os.WriteFile("defaults.yaml", b, 0644)) + } else { + require.Equal(t, defaultsYAML, string(b)) + } + }) } func TestSchema_Unmarshal(t *testing.T) { diff --git a/pkg/settings/json.go b/pkg/settings/json.go index c7c29461f..e153a7e84 100644 --- a/pkg/settings/json.go +++ b/pkg/settings/json.go @@ -100,7 +100,7 @@ func readJSONMaps(files fs.FS, dir string) (jsonMap, error) { } m, err := readJSONMap(files, path) if err != nil { - return fmt.Errorf("failed to read toml file %s: %w", path, err) + return fmt.Errorf("failed to read json file %s: %w", path, err) } name := strings.TrimSuffix(d.Name(), ".json") ms[name] = m diff --git a/pkg/settings/testdata/config.yaml b/pkg/settings/testdata/config.yaml new file mode 100644 index 000000000..74c3c3ab8 --- /dev/null +++ b/pkg/settings/testdata/config.yaml @@ -0,0 +1,32 @@ +# File generated from source files. DO NOT EDIT. +global: + Bar: + Baz: "10" + Foo: "5" +org: + "123": + Bar: + Baz: "99" + Foo: "42" + "456": + Bar: + Baz: "500" + Foo: "99" +owner: + 00026b4aa7e57ca7b68ae1bf45653f56b656fd3a: + Bar: + Baz: "200" + Foo: "75" + 8bd112d3f8f92e41c861939545ad387307af9703: + Bar: + Baz: "43" + Foo: "13" +workflow: + 15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0: + Bar: + Baz: "13" + Foo: "17" + 15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef: + Bar: + Baz: "50" + Foo: "20" diff --git a/pkg/settings/testdata/yaml/global.yaml b/pkg/settings/testdata/yaml/global.yaml new file mode 100644 index 000000000..c662a2e9b --- /dev/null +++ b/pkg/settings/testdata/yaml/global.yaml @@ -0,0 +1,4 @@ +Foo: "5" + +Bar: + Baz: "10" \ No newline at end of file diff --git a/pkg/settings/testdata/yaml/org/123.yaml b/pkg/settings/testdata/yaml/org/123.yaml new file mode 100644 index 000000000..86b933881 --- /dev/null +++ b/pkg/settings/testdata/yaml/org/123.yaml @@ -0,0 +1,4 @@ +Foo: "42" + +Bar: + Baz: "99" \ No newline at end of file diff --git a/pkg/settings/testdata/yaml/org/456.yaml b/pkg/settings/testdata/yaml/org/456.yaml new file mode 100644 index 000000000..377fe4ed1 --- /dev/null +++ b/pkg/settings/testdata/yaml/org/456.yaml @@ -0,0 +1,4 @@ +Foo: "99" + +Bar: + Baz: "500" \ No newline at end of file diff --git a/pkg/settings/testdata/yaml/owner/00026b4aa7e57ca7b68ae1bf45653f56b656fd3a.yaml b/pkg/settings/testdata/yaml/owner/00026b4aa7e57ca7b68ae1bf45653f56b656fd3a.yaml new file mode 100644 index 000000000..266fbe108 --- /dev/null +++ b/pkg/settings/testdata/yaml/owner/00026b4aa7e57ca7b68ae1bf45653f56b656fd3a.yaml @@ -0,0 +1,4 @@ +Foo: "75" + +Bar: + Baz: "200" \ No newline at end of file diff --git a/pkg/settings/testdata/yaml/owner/8bd112d3f8f92e41c861939545ad387307af9703.yaml b/pkg/settings/testdata/yaml/owner/8bd112d3f8f92e41c861939545ad387307af9703.yaml new file mode 100644 index 000000000..e0991e91c --- /dev/null +++ b/pkg/settings/testdata/yaml/owner/8bd112d3f8f92e41c861939545ad387307af9703.yaml @@ -0,0 +1,4 @@ +Foo: "13" + +Bar: + Baz: "43" \ No newline at end of file diff --git a/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0.yaml b/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0.yaml new file mode 100644 index 000000000..005207e31 --- /dev/null +++ b/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af13855687559d7ff6552ac6dbb2ce0.yaml @@ -0,0 +1,4 @@ +Foo: "17" + +Bar: + Baz: "13" \ No newline at end of file diff --git a/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef.yaml b/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef.yaml new file mode 100644 index 000000000..1e233a861 --- /dev/null +++ b/pkg/settings/testdata/yaml/workflow/15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef.yaml @@ -0,0 +1,4 @@ +Foo: "20" + +Bar: + Baz: "50" \ No newline at end of file diff --git a/pkg/settings/yaml.go b/pkg/settings/yaml.go new file mode 100644 index 000000000..7bd4248e7 --- /dev/null +++ b/pkg/settings/yaml.go @@ -0,0 +1,163 @@ +package settings + +import ( + "bytes" + "context" + "fmt" + "io/fs" + "strconv" + "strings" + + "github.com/goccy/go-yaml" +) + +func CombineYAMLFiles(files fs.FS) ([]byte, error) { + m := make(map[string]any) + global, err := readYAMLMap(files, "global.yaml") + if err != nil { + return nil, err + } + if len(global) > 0 { + m["global"] = global + } + orgs, err := readYAMLMaps(files, "org") + if err != nil { + return nil, err + } + if len(orgs) > 0 { + m["org"] = orgs + } + owners, err := readYAMLMaps(files, "owner") + if err != nil { + return nil, err + } + if len(owners) > 0 { + m["owner"] = owners + } + workflows, err := readYAMLMaps(files, "workflow") + if err != nil { + return nil, err + } + if len(workflows) > 0 { + m["workflow"] = workflows + } + + var b bytes.Buffer + b.WriteString(generatedHeader) + e := yaml.NewEncoder(&b) + err = e.Encode(m) + return b.Bytes(), err +} + +func readYAMLMap(files fs.FS, name string) (map[string]any, error) { + f, err := files.Open(name) + if err != nil { + return nil, fmt.Errorf("failed to open %s: %w", name, err) + } + defer f.Close() + d := yaml.NewDecoder(f) + var m jsonMap + err = d.Decode(&m) + if err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", name, err) + } + + return m, nil +} + +func readYAMLMaps(files fs.FS, dir string) (jsonMap, error) { + ms := make(map[string]any) + if err := fs.WalkDir(files, dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil // ignore + } + m, err := readYAMLMap(files, path) + if err != nil { + return fmt.Errorf("failed to read yaml file %s: %w", path, err) + } + name := strings.TrimSuffix(d.Name(), ".yaml") + ms[name] = m + return nil + }); err != nil { + return nil, fmt.Errorf("failed to walk %s: %w", dir, err) + } + return ms, nil +} + +type yamlSettings struct { + m map[string]any +} + +func newYAMLSettings(b []byte) (*yamlSettings, error) { + var m map[string]any + err := yaml.Unmarshal(b, &m) + if err != nil { + return nil, err + } + return &yamlSettings{m: m}, nil +} + +func (y *yamlSettings) getFirst(keys ...string) (string, error) { + for _, k := range keys { + v, err := y.get(k) + if err != nil { + return "", err + } + if v != "" { + return v, nil + } + } + return "", nil // no values +} + +func (y *yamlSettings) get(key string) (string, error) { + m := y.m + parts := strings.Split(key, ".") + for i, part := range parts[:len(parts)-1] { + v := m[part] + if v == nil { + return "", nil + } + var ok bool + m, ok = v.(map[string]any) + if !ok { + return "", fmt.Errorf("invalid key %s: %s is a field", key, strings.Join(parts[:i+1], ".")) + } + } + + field := parts[len(parts)-1] + if val, ok := m[field]; ok { + switch t := val.(type) { + case string: + return t, nil + case bool: + return strconv.FormatBool(t), nil + default: + return "", fmt.Errorf("non-string value: %s: %t(%v)", key, val, val) + } + } + return "", nil // no value +} + +type yamlGetter struct { + settings *yamlSettings +} + +func NewYAMLGetter(b []byte) (Getter, error) { + s, err := newYAMLSettings(b) + if err != nil { + return nil, err + } + return &yamlGetter{settings: s}, nil +} + +func (y *yamlGetter) GetScoped(ctx context.Context, scope Scope, key string) (value string, err error) { + keys, err := scope.rawKeys(ctx, key) + if err != nil { + return "", fmt.Errorf("failed to get raw keys: %w", err) + } + return y.settings.getFirst(keys...) +} diff --git a/pkg/settings/yaml_test.go b/pkg/settings/yaml_test.go new file mode 100644 index 000000000..65cda3007 --- /dev/null +++ b/pkg/settings/yaml_test.go @@ -0,0 +1,74 @@ +package settings + +import ( + "embed" + "io/fs" + "os" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/smartcontractkit/chainlink-common/pkg/contexts" +) + +//go:embed testdata/yaml +var yamlFiles embed.FS + +//go:embed testdata/config.yaml +var configYAML string + +//go:generate go test -run TestCombineYAMLFiles -update +func TestCombineYAMLFiles(t *testing.T) { + sub, err := fs.Sub(yamlFiles, "testdata/yaml") + require.NoError(t, err) + b, err := CombineYAMLFiles(sub) + require.NoError(t, err) + if *update { + require.NoError(t, os.WriteFile("testdata/config.yaml", b, os.ModePerm)) + return + } + require.Equal(t, configYAML, string(b)) +} +func Test_yamlSettings_GetScoped(t *testing.T) { + s, err := newYAMLSettings([]byte(configYAML)) + require.NoError(t, err) + r := yamlGetter{s} + + ctx := contexts.WithCRE(t.Context(), contexts.CRE{ + Org: "123", + Owner: "8bd112d3f8f92e41c861939545ad387307af9703", + Workflow: "15c631d295ef5e32deb99a10ee6804bc4af1385568f9b3363f6552ac6dbb2cef", + }) + gotValue, err := r.GetScoped(ctx, ScopeGlobal, `Foo`) + require.NoError(t, err) + assert.Equal(t, "5", gotValue) + + gotValue, err = r.GetScoped(ctx, ScopeGlobal, "Bar.Baz") + require.NoError(t, err) + assert.Equal(t, "10", gotValue) + + gotValue, err = r.GetScoped(ctx, ScopeOrg, "Foo") + require.NoError(t, err) + assert.Equal(t, "42", gotValue) + + gotValue, err = r.GetScoped(ctx, ScopeOrg, "Bar.Baz") + require.NoError(t, err) + assert.Equal(t, "99", gotValue) + + gotValue, err = r.GetScoped(ctx, ScopeOwner, "Foo") + require.NoError(t, err) + assert.Equal(t, "13", gotValue) + + gotValue, err = r.GetScoped(ctx, ScopeOwner, "Bar.Baz") + require.NoError(t, err) + assert.Equal(t, "43", gotValue) + + gotValue, err = r.GetScoped(ctx, ScopeWorkflow, "Foo") + require.NoError(t, err) + assert.Equal(t, "20", gotValue) + + gotValue, err = r.GetScoped(ctx, ScopeWorkflow, "Bar.Baz") + require.NoError(t, err) + assert.Equal(t, "50", gotValue) +}