From 134cb0e59230e4c1836e260157b644bb5e886555 Mon Sep 17 00:00:00 2001 From: josill Date: Thu, 29 Jan 2026 14:52:09 +0200 Subject: [PATCH 1/4] init --- docs/docs.go | 87 ++++++++++++++++++++- docs/swagger.json | 87 ++++++++++++++++++++- docs/swagger.yaml | 67 +++++++++++++++- pkg/server/configdb_handlers.go | 126 ++++++++++++++++++++++++++++++ pkg/server/endpoints_test.go | 132 ++++++++++++++++++++++++++++++++ 5 files changed, 492 insertions(+), 7 deletions(-) diff --git a/docs/docs.go b/docs/docs.go index 9f049304..4d6f265b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -83,6 +83,12 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -238,6 +244,12 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -358,6 +370,12 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -444,6 +462,12 @@ const docTemplate = `{ "/api/pipeline/reports": { "get": { "description": "List all pipeline reports from the database", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], "tags": [ "Pipeline Reports" ], @@ -466,6 +490,18 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering reports (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering reports (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -701,6 +737,18 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering SCMs (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering SCMs (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -923,7 +971,15 @@ const docTemplate = `{ ] }, "pipelineID": { - "description": "PipelineID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same PipelineID.\nThis value is coming from the pipeline report to improve the search of reports.", + "description": "PipelineID represent the unique identifier of the pipeline.\nSeveral reports can be associated to the same PipelineID.", + "type": "string" + }, + "reportID": { + "description": "ReportID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same ReportID.\nThis value is coming from the pipeline report to improve the search of reports.", + "type": "string" + }, + "result": { + "description": "Result represent the result of the pipeline execution.", "type": "string" }, "sourceConfigIDs": { @@ -1563,6 +1619,10 @@ const docTemplate = `{ "description": "CaptureIndex defines which substring occurrence to retrieve. Note also that a value of ` + "`" + `0` + "`" + ` for ` + "`" + `captureIndex` + "`" + ` returns all submatches, and individual submatch indexes start at ` + "`" + `1` + "`" + `.", "type": "integer" }, + "capturePattern": { + "description": "Uses the match group(s) to generate the output using \\0, \\1, \\2, etc", + "type": "string" + }, "deprecatedCaptureIndex": { "type": "integer" }, @@ -1572,6 +1632,26 @@ const docTemplate = `{ } } }, + "transformer.JsonMatch": { + "type": "object", + "properties": { + "joinMultipleMatches": { + "description": "If we find multiple matches, join them by this", + "type": "string" + }, + "key": { + "type": "string" + }, + "multipleMatchSelector": { + "description": "If we find multiple matches, select the \"first\" or the \"last\"", + "type": "string" + }, + "noMatchResult": { + "description": "If we don't find a match then return the following string or the input value", + "type": "string" + } + } + }, "transformer.Replacer": { "type": "object", "properties": { @@ -1624,6 +1704,9 @@ const docTemplate = `{ } ] }, + "jsonMatch": { + "$ref": "#/definitions/transformer.JsonMatch" + }, "quote": { "description": "Quote add quote around the value", "type": "boolean" @@ -1644,7 +1727,7 @@ const docTemplate = `{ } }, "semVerInc": { - "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", + "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", "type": "string" }, "trimPrefix": { diff --git a/docs/swagger.json b/docs/swagger.json index d2419318..310cfc07 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -72,6 +72,12 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -227,6 +233,12 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -347,6 +359,12 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -433,6 +451,12 @@ "/api/pipeline/reports": { "get": { "description": "List all pipeline reports from the database", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], "tags": [ "Pipeline Reports" ], @@ -455,6 +479,18 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering reports (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering reports (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -690,6 +726,18 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering SCMs (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering SCMs (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -912,7 +960,15 @@ ] }, "pipelineID": { - "description": "PipelineID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same PipelineID.\nThis value is coming from the pipeline report to improve the search of reports.", + "description": "PipelineID represent the unique identifier of the pipeline.\nSeveral reports can be associated to the same PipelineID.", + "type": "string" + }, + "reportID": { + "description": "ReportID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same ReportID.\nThis value is coming from the pipeline report to improve the search of reports.", + "type": "string" + }, + "result": { + "description": "Result represent the result of the pipeline execution.", "type": "string" }, "sourceConfigIDs": { @@ -1552,6 +1608,10 @@ "description": "CaptureIndex defines which substring occurrence to retrieve. Note also that a value of `0` for `captureIndex` returns all submatches, and individual submatch indexes start at `1`.", "type": "integer" }, + "capturePattern": { + "description": "Uses the match group(s) to generate the output using \\0, \\1, \\2, etc", + "type": "string" + }, "deprecatedCaptureIndex": { "type": "integer" }, @@ -1561,6 +1621,26 @@ } } }, + "transformer.JsonMatch": { + "type": "object", + "properties": { + "joinMultipleMatches": { + "description": "If we find multiple matches, join them by this", + "type": "string" + }, + "key": { + "type": "string" + }, + "multipleMatchSelector": { + "description": "If we find multiple matches, select the \"first\" or the \"last\"", + "type": "string" + }, + "noMatchResult": { + "description": "If we don't find a match then return the following string or the input value", + "type": "string" + } + } + }, "transformer.Replacer": { "type": "object", "properties": { @@ -1613,6 +1693,9 @@ } ] }, + "jsonMatch": { + "$ref": "#/definitions/transformer.JsonMatch" + }, "quote": { "description": "Quote add quote around the value", "type": "boolean" @@ -1633,7 +1716,7 @@ } }, "semVerInc": { - "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", + "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", "type": "string" }, "trimPrefix": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6892e61d..ec84a4e8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -170,10 +170,18 @@ definitions: description: Pipeline represent the Updatecli pipeline report. pipelineID: description: |- - PipelineID represent the ID of the pipeline executed by Updatecli. - different execution of the same pipeline will have the same PipelineID. + PipelineID represent the unique identifier of the pipeline. + Several reports can be associated to the same PipelineID. + type: string + reportID: + description: |- + ReportID represent the ID of the pipeline executed by Updatecli. + different execution of the same pipeline will have the same ReportID. This value is coming from the pipeline report to improve the search of reports. type: string + result: + description: Result represent the result of the pipeline execution. + type: string sourceConfigIDs: additionalProperties: type: string @@ -702,12 +710,31 @@ definitions: Note also that a value of `0` for `captureIndex` returns all submatches, and individual submatch indexes start at `1`. type: integer + capturePattern: + description: Uses the match group(s) to generate the output using \0, \1, + \2, etc + type: string deprecatedCaptureIndex: type: integer pattern: description: Pattern defines regular expression to use for retrieving a submatch type: string type: object + transformer.JsonMatch: + properties: + joinMultipleMatches: + description: If we find multiple matches, join them by this + type: string + key: + type: string + multipleMatchSelector: + description: If we find multiple matches, select the "first" or the "last" + type: string + noMatchResult: + description: If we don't find a match then return the following string or + the input value + type: string + type: object transformer.Replacer: properties: from: @@ -745,6 +772,8 @@ definitions: - $ref: '#/definitions/transformer.FindSubMatch' description: Find searches for a specific value if it exists then return the value using regular expression + jsonMatch: + $ref: '#/definitions/transformer.JsonMatch' quote: description: Quote add quote around the value type: boolean @@ -758,7 +787,7 @@ definitions: $ref: '#/definitions/transformer.Replacer' type: array semVerInc: - description: SemvVerInc specifies a comma separated list semantic versioning + description: SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded. type: string trimPrefix: @@ -838,6 +867,10 @@ paths: in: query name: page type: string + - description: When true, returns only the Spec field from each Config object + in: query + name: spec_only + type: boolean responses: "200": description: OK @@ -940,6 +973,10 @@ paths: in: query name: page type: string + - description: When true, returns only the Spec field from each Config object + in: query + name: spec_only + type: boolean responses: "200": description: OK @@ -1019,6 +1056,10 @@ paths: in: query name: page type: string + - description: When true, returns only the Spec field from each Config object + in: query + name: spec_only + type: boolean responses: "200": description: OK @@ -1056,6 +1097,8 @@ paths: - Configuration Targets /api/pipeline/reports: get: + consumes: + - application/json description: List all pipeline reports from the database parameters: - description: SCM ID @@ -1070,6 +1113,16 @@ paths: in: query name: page type: string + - description: Start time for filtering reports (RFC3339 format) + in: query + name: start_time + type: string + - description: End time for filtering reports (RFC3339 format) + in: query + name: end_time + type: string + produces: + - application/json responses: "200": description: OK @@ -1226,6 +1279,14 @@ paths: in: query name: page type: string + - description: Start time for filtering SCMs (RFC3339 format) + in: query + name: start_time + type: string + - description: End time for filtering SCMs (RFC3339 format) + in: query + name: end_time + type: string responses: "200": description: OK diff --git a/pkg/server/configdb_handlers.go b/pkg/server/configdb_handlers.go index 318744c6..8014a664 100644 --- a/pkg/server/configdb_handlers.go +++ b/pkg/server/configdb_handlers.go @@ -10,6 +10,91 @@ import ( "github.com/updatecli/udash/pkg/model" ) +// specOnlyConfig represents a Config object with only the Spec field +type specOnlyConfig struct { + Spec interface{} `json:"spec"` +} + +// transformToSpecOnly transforms a Config object to only include the Spec field +func transformToSpecOnly(configJSON []byte) (json.RawMessage, error) { + var configMap map[string]interface{} + if err := json.Unmarshal(configJSON, &configMap); err != nil { + return nil, err + } + + // Extract only the Spec field + specOnly := specOnlyConfig{ + Spec: configMap["spec"], + } + + result, err := json.Marshal(specOnly) + if err != nil { + return nil, err + } + + return json.RawMessage(result), nil +} + +// transformSourceConfigsToSpecOnly transforms a slice of ConfigSource to spec-only format +func transformSourceConfigsToSpecOnly(configs []model.ConfigSource) ([]model.ConfigSource, error) { + result := make([]model.ConfigSource, len(configs)) + for i, config := range configs { + result[i] = config + configJSON, err := json.Marshal(config.Config) + if err != nil { + return nil, err + } + specOnlyJSON, err := transformToSpecOnly(configJSON) + if err != nil { + return nil, err + } + if err := json.Unmarshal(specOnlyJSON, &result[i].Config); err != nil { + return nil, err + } + } + return result, nil +} + +// transformConditionConfigsToSpecOnly transforms a slice of ConfigCondition to spec-only format +func transformConditionConfigsToSpecOnly(configs []model.ConfigCondition) ([]model.ConfigCondition, error) { + result := make([]model.ConfigCondition, len(configs)) + for i, config := range configs { + result[i] = config + configJSON, err := json.Marshal(config.Config) + if err != nil { + return nil, err + } + specOnlyJSON, err := transformToSpecOnly(configJSON) + if err != nil { + return nil, err + } + if err := json.Unmarshal(specOnlyJSON, &result[i].Config); err != nil { + return nil, err + } + } + return result, nil +} + +// transformTargetConfigsToSpecOnly transforms a slice of ConfigTarget to spec-only format +func transformTargetConfigsToSpecOnly(configs []model.ConfigTarget) ([]model.ConfigTarget, error) { + result := make([]model.ConfigTarget, len(configs)) + for i, config := range configs { + result[i] = config + configJSON, err := json.Marshal(config.Config) + if err != nil { + return nil, err + } + specOnlyJSON, err := transformToSpecOnly(configJSON) + if err != nil { + return nil, err + } + if err := json.Unmarshal(specOnlyJSON, &result[i].Config); err != nil { + return nil, err + } + } + return result, nil +} + // SourceConfigResponse represents a response containing configuration sources. type SourceConfigResponse struct { // Configs is a list of configuration sources. @@ -48,6 +133,7 @@ type ConfigKindResponse struct { // @Param config query string false "Configuration of the source" // @Param limit query string false "Limit the number of reports returned, default is 100" // @Param page query string false "Page number for pagination, default is 1" +// @Param spec_only query boolean false "When true, returns only the Spec field from each Config object" // @Success 200 {object} SourceConfigResponse // @Failure 500 {object} DefaultResponseModel // @Router /api/pipeline/config/sources [get] @@ -55,6 +141,7 @@ func ListConfigSources(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") + specOnly := c.Request.URL.Query().Get("spec_only") == "true" limit, page, err := getPaginationParamFromURLQuery(c) @@ -75,6 +162,17 @@ func ListConfigSources(c *gin.Context) { return } + if specOnly { + rows, err = transformSourceConfigsToSpecOnly(rows) + if err != nil { + logrus.Errorf("transforming config sources to spec-only: %s", err) + c.JSON(http.StatusInternalServerError, DefaultResponseModel{ + Err: "failed to transform configs: " + err.Error(), + }) + return + } + } + c.JSON(http.StatusOK, SourceConfigResponse{ Configs: rows, TotalCount: totalCount, @@ -194,6 +292,7 @@ func DeleteConfigSource(c *gin.Context) { // @Param config query string false "Configuration of the condition" // @Param limit query string false "Limit the number of reports returned, default is 100" // @Param page query string false "Page number for pagination, default is 1" +// @Param spec_only query boolean false "When true, returns only the Spec field from each Config object" // @Success 200 {object} ConditionConfigResponse // @Failure 500 {object} DefaultResponseModel // @Router /api/pipeline/config/conditions [get] @@ -201,6 +300,7 @@ func ListConfigConditions(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") + specOnly := c.Request.URL.Query().Get("spec_only") == "true" limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { @@ -219,6 +319,18 @@ func ListConfigConditions(c *gin.Context) { }) return } + + if specOnly { + rows, err = transformConditionConfigsToSpecOnly(rows) + if err != nil { + logrus.Errorf("transforming config conditions to spec-only: %s", err) + c.JSON(http.StatusInternalServerError, DefaultResponseModel{ + Err: "failed to transform configs: " + err.Error(), + }) + return + } + } + c.JSON(http.StatusOK, ConditionConfigResponse{ Configs: rows, TotalCount: totalCount, @@ -306,6 +418,7 @@ func DeleteConfigCondition(c *gin.Context) { // @Param config query string false "Configuration of the target" // @Param limit query string false "Limit the number of reports returned, default is 100" // @Param page query string false "Page number for pagination, default is 1" +// @Param spec_only query boolean false "When true, returns only the Spec field from each Config object" // @Success 200 {object} TargetConfigResponse // @Failure 500 {object} DefaultResponseModel // @Router /api/pipeline/config/targets [get] @@ -313,6 +426,7 @@ func ListConfigTargets(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") + specOnly := c.Request.URL.Query().Get("spec_only") == "true" limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { @@ -331,6 +445,18 @@ func ListConfigTargets(c *gin.Context) { }) return } + + if specOnly { + rows, err = transformTargetConfigsToSpecOnly(rows) + if err != nil { + logrus.Errorf("transforming config targets to spec-only: %s", err) + c.JSON(http.StatusInternalServerError, DefaultResponseModel{ + Err: "failed to transform configs: " + err.Error(), + }) + return + } + } + c.JSON(http.StatusOK, TargetConfigResponse{ Configs: rows, TotalCount: totalCount, diff --git a/pkg/server/endpoints_test.go b/pkg/server/endpoints_test.go index 27207340..fa2659db 100644 --- a/pkg/server/endpoints_test.go +++ b/pkg/server/endpoints_test.go @@ -396,6 +396,138 @@ func TestEndpoints(t *testing.T) { }, }, removeFieldsAsserter("configs", "Created_at", "Updated_at")) }) + + t.Run("with spec_only=true", func(t *testing.T) { + // Insert a config with Spec, Transformers, Name, etc. + fullConfig := map[string]any{ + "spec": map[string]any{ + "owner": "updatecli", + "repository": "updatecli", + "token": "test-token", + }, + "transformers": []any{ + map[string]any{ + "addPrefix": "v", + }, + }, + "name": "test-source", + "kind": "githubrelease", + } + configID, err := database.InsertConfigResource(ctx, "source", "githubrelease", fullConfig) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, database.DeleteConfigResource(ctx, "source", configID)) + }) + + // Test without spec_only - should return full config + resp := doGetRequest(t, srv, "/api/pipeline/config/sources?id="+configID) + var resultWithoutSpecOnly map[string]any + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &resultWithoutSpecOnly)) + configs := resultWithoutSpecOnly["configs"].([]any) + require.Len(t, configs, 1) + config := configs[0].(map[string]any) + configObj := config["Config"].(map[string]any) + // Should have transformers, name, etc. + assert.NotNil(t, configObj["Transformers"]) + assert.NotNil(t, configObj["Spec"]) + + // Test with spec_only=true - should return only Spec + resp = doGetRequest(t, srv, "/api/pipeline/config/sources?id="+configID+"&spec_only=true") + var resultWithSpecOnly map[string]any + b, _ = io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &resultWithSpecOnly)) + configs = resultWithSpecOnly["configs"].([]any) + require.Len(t, configs, 1) + config = configs[0].(map[string]any) + configObj = config["Config"].(map[string]any) + // Should only have Spec field + assert.NotNil(t, configObj["Spec"]) + // Transformers should be nil/empty + transformers, hasTransformers := configObj["Transformers"] + if hasTransformers { + assert.Nil(t, transformers) + } + // Name should be empty + name, hasName := configObj["Name"] + if hasName { + assert.Empty(t, name) + } + }) + }) + }) + + t.Run("GET /api/pipeline/config/conditions", func(t *testing.T) { + t.Run("with spec_only=true", func(t *testing.T) { + fullConfig := map[string]any{ + "spec": map[string]any{ + "file": "/path/to/file", + "key": "version", + }, + "transformers": []any{ + map[string]any{ + "addSuffix": ".0", + }, + }, + "name": "test-condition", + "kind": "file", + } + configID, err := database.InsertConfigResource(ctx, "condition", "file", fullConfig) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, database.DeleteConfigResource(ctx, "condition", configID)) + }) + + // Test with spec_only=true + resp := doGetRequest(t, srv, "/api/pipeline/config/conditions?id="+configID+"&spec_only=true") + var result map[string]any + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &result)) + configs := result["configs"].([]any) + require.Len(t, configs, 1) + config := configs[0].(map[string]any) + configObj := config["Config"].(map[string]any) + // Should only have Spec field + assert.NotNil(t, configObj["Spec"]) + }) + }) + + t.Run("GET /api/pipeline/config/targets", func(t *testing.T) { + t.Run("with spec_only=true", func(t *testing.T) { + fullConfig := map[string]any{ + "spec": map[string]any{ + "file": "/path/to/target", + "key": "version", + }, + "transformers": []any{ + map[string]any{ + "trimPrefix": "v", + }, + }, + "name": "test-target", + "kind": "yaml", + } + configID, err := database.InsertConfigResource(ctx, "target", "yaml", fullConfig) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, database.DeleteConfigResource(ctx, "target", configID)) + }) + + // Test with spec_only=true + resp := doGetRequest(t, srv, "/api/pipeline/config/targets?id="+configID+"&spec_only=true") + var result map[string]any + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &result)) + configs := result["configs"].([]any) + require.Len(t, configs, 1) + config := configs[0].(map[string]any) + configObj := config["Config"].(map[string]any) + // Should only have Spec field + assert.NotNil(t, configObj["Spec"]) }) }) } From 625c04e8795844b13907b43917be463f4b5a8454 Mon Sep 17 00:00:00 2001 From: josill Date: Thu, 29 Jan 2026 15:11:42 +0200 Subject: [PATCH 2/4] optimize queries at db level --- pkg/database/config.go | 118 ++++++++++++++++++--------- pkg/database/report.go | 6 +- pkg/server/configdb_handlers.go | 136 +++----------------------------- 3 files changed, 93 insertions(+), 167 deletions(-) diff --git a/pkg/database/config.go b/pkg/database/config.go index c8ee3473..cb93cc21 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -8,7 +8,9 @@ import ( "github.com/google/uuid" "github.com/sirupsen/logrus" + "github.com/stephenafamo/bob" "github.com/stephenafamo/bob/dialect/psql" + "github.com/stephenafamo/bob/dialect/psql/dialect" "github.com/stephenafamo/bob/dialect/psql/dm" "github.com/stephenafamo/bob/dialect/psql/im" "github.com/stephenafamo/bob/dialect/psql/sm" @@ -156,40 +158,54 @@ func GetConfigKind(ctx context.Context, resourceType string) ([]string, error) { } // GetSourceConfigs returns a list of resource configurations from the database. -func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page int) ([]model.ConfigSource, int, error) { +// If specOnly is true, only the Spec field is extracted from the config JSONB column. +func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page int, specOnly bool) ([]model.ConfigSource, int, error) { table := configSourceTableName - // SELECT id, kind, created_at, updated_at, config FROM " + table - query := psql.Select( - sm.Columns("id", "kind", "created_at", "updated_at", "config"), - sm.From(table), - ) + // When specOnly is true, use PostgreSQL JSONB to extract only the spec field + // jsonb_build_object('spec', config->'spec') creates a new JSONB object with only the spec field + var query *bob.BaseQuery[*dialect.SelectQuery] + if specOnly { + // SELECT id, kind, created_at, updated_at, jsonb_build_object('spec', config->'spec') as config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "jsonb_build_object('spec', config->'spec') as config"), + sm.From(table), + ) + query = &q + } else { + // SELECT id, kind, created_at, updated_at, config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "config"), + sm.From(table), + ) + query = &q + } if id != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("id").EQ(psql.Arg(id))), ) } if kind != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("kind").EQ(psql.Arg(kind))), ) } if config != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Raw("config @> ?", config)), ) } - query.Apply( + (*query).Apply( sm.OrderBy(psql.Quote("updated_at")).Desc(), ) // Get total count of results totalCount := 0 - totalQuery := psql.Select(sm.From(query), sm.Columns("count(*)")) + totalQuery := psql.Select(sm.From(*query), sm.Columns("count(*)")) totalQueryString, totalArgs, err := totalQuery.Build(ctx) if err != nil { logrus.Errorf("building total count query failed: %s\n\t%s", totalQueryString, err) @@ -203,13 +219,13 @@ func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page } if limit < totalCount && limit > 0 { - query.Apply( + (*query).Apply( sm.Limit(limit), sm.Offset((page-1)*limit), ) } - queryString, args, err := query.Build(ctx) + queryString, args, err := (*query).Build(ctx) if err != nil { logrus.Errorf("building query failed: %s\n\t%s", queryString, err) return nil, 0, err @@ -248,34 +264,48 @@ func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page } // GetConditionConfigs returns a list of resource configurations from the database. -func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, page int) ([]model.ConfigCondition, int, error) { +// If specOnly is true, only the Spec field is extracted from the config JSONB column. +func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, page int, specOnly bool) ([]model.ConfigCondition, int, error) { table := configConditionTableName - // SELECT id, kind, created_at, updated_at, config FROM " + table - query := psql.Select( - sm.Columns("id", "kind", "created_at", "updated_at", "config"), - sm.From(table), - ) + // When specOnly is true, use PostgreSQL JSONB to extract only the spec field + // jsonb_build_object('spec', config->'spec') creates a new JSONB object with only the spec field + var query *bob.BaseQuery[*dialect.SelectQuery] + if specOnly { + // SELECT id, kind, created_at, updated_at, jsonb_build_object('spec', config->'spec') as config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "jsonb_build_object('spec', config->'spec') as config"), + sm.From(table), + ) + query = &q + } else { + // SELECT id, kind, created_at, updated_at, config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "config"), + sm.From(table), + ) + query = &q + } if id != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("id").EQ(psql.Arg(id))), ) } if kind != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("kind").EQ(psql.Arg(kind))), ) } if config != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Raw("config @> ?", config)), ) } - query.Apply( + (*query).Apply( sm.OrderBy(psql.Quote("updated_at")).Desc(), ) @@ -295,13 +325,13 @@ func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, pa // Apply pagination if limit and page are set if limit < totalCount && limit > 0 { - query.Apply( + (*query).Apply( sm.Limit(limit), sm.Offset((page-1)*limit), ) } - queryString, args, err := query.Build(ctx) + queryString, args, err := (*query).Build(ctx) if err != nil { logrus.Errorf("building query failed: %s\n\t%s", queryString, err) return nil, 0, err @@ -343,34 +373,48 @@ func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, pa } // GetTargetConfigs returns a list of resource configurations from the database. -func GetTargetConfigs(ctx context.Context, kind, id, config string, limit, page int) ([]model.ConfigTarget, int, error) { +// If specOnly is true, only the Spec field is extracted from the config JSONB column. +func GetTargetConfigs(ctx context.Context, kind, id, config string, limit, page int, specOnly bool) ([]model.ConfigTarget, int, error) { table := configTargetTableName - // SELECT id, kind, created_at, updated_at, config FROM " + table - query := psql.Select( - sm.Columns("id", "kind", "created_at", "updated_at", "config"), - sm.From(table), - ) + // When specOnly is true, use PostgreSQL JSONB to extract only the spec field + // jsonb_build_object('spec', config->'spec') creates a new JSONB object with only the spec field + var query *bob.BaseQuery[*dialect.SelectQuery] + if specOnly { + // SELECT id, kind, created_at, updated_at, jsonb_build_object('spec', config->'spec') as config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "jsonb_build_object('spec', config->'spec') as config"), + sm.From(table), + ) + query = &q + } else { + // SELECT id, kind, created_at, updated_at, config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "config"), + sm.From(table), + ) + query = &q + } if id != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("id").EQ(psql.Arg(id))), ) } if kind != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("kind").EQ(psql.Arg(kind))), ) } if config != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Raw("config @> ?", config)), ) } - query.Apply( + (*query).Apply( sm.OrderBy(psql.Quote("updated_at")).Desc(), ) @@ -390,13 +434,13 @@ func GetTargetConfigs(ctx context.Context, kind, id, config string, limit, page // Apply pagination if limit and page are set if limit < totalCount && limit > 0 { - query.Apply( + (*query).Apply( sm.Limit(limit), sm.Offset((page-1)*limit), ) } - queryString, args, err := query.Build(ctx) + queryString, args, err := (*query).Build(ctx) if err != nil { logrus.Errorf("building query failed: %s\n\t%s", queryString, err) return nil, 0, err diff --git a/pkg/database/report.go b/pkg/database/report.go index 995537ac..55deebd8 100644 --- a/pkg/database/report.go +++ b/pkg/database/report.go @@ -344,7 +344,7 @@ func InsertReport(ctx context.Context, report reports.Report) (string, error) { continue } - results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1) + results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1, false) if err != nil { logrus.Errorf("failed: %s", err) continue @@ -429,7 +429,7 @@ func InsertReport(ctx context.Context, report reports.Report) (string, error) { continue } - results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1) + results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1, false) if err != nil { logrus.Errorf("failed: %s", err) continue @@ -532,7 +532,7 @@ func buildConfigSources(ctx context.Context, report reports.Report) pgtype.Hstor continue } - results, _, err := GetSourceConfigs(ctx, kind, "", string(data), 0, 1) + results, _, err := GetSourceConfigs(ctx, kind, "", string(data), 0, 1, false) if err != nil { logrus.Errorf("failed: %s", err) continue diff --git a/pkg/server/configdb_handlers.go b/pkg/server/configdb_handlers.go index 8014a664..b780886a 100644 --- a/pkg/server/configdb_handlers.go +++ b/pkg/server/configdb_handlers.go @@ -10,91 +10,6 @@ import ( "github.com/updatecli/udash/pkg/model" ) -// specOnlyConfig represents a Config object with only the Spec field -type specOnlyConfig struct { - Spec interface{} `json:"spec"` -} - -// transformToSpecOnly transforms a Config object to only include the Spec field -func transformToSpecOnly(configJSON []byte) (json.RawMessage, error) { - var configMap map[string]interface{} - if err := json.Unmarshal(configJSON, &configMap); err != nil { - return nil, err - } - - // Extract only the Spec field - specOnly := specOnlyConfig{ - Spec: configMap["spec"], - } - - result, err := json.Marshal(specOnly) - if err != nil { - return nil, err - } - - return json.RawMessage(result), nil -} - -// transformSourceConfigsToSpecOnly transforms a slice of ConfigSource to spec-only format -func transformSourceConfigsToSpecOnly(configs []model.ConfigSource) ([]model.ConfigSource, error) { - result := make([]model.ConfigSource, len(configs)) - for i, config := range configs { - result[i] = config - configJSON, err := json.Marshal(config.Config) - if err != nil { - return nil, err - } - specOnlyJSON, err := transformToSpecOnly(configJSON) - if err != nil { - return nil, err - } - if err := json.Unmarshal(specOnlyJSON, &result[i].Config); err != nil { - return nil, err - } - } - return result, nil -} - -// transformConditionConfigsToSpecOnly transforms a slice of ConfigCondition to spec-only format -func transformConditionConfigsToSpecOnly(configs []model.ConfigCondition) ([]model.ConfigCondition, error) { - result := make([]model.ConfigCondition, len(configs)) - for i, config := range configs { - result[i] = config - configJSON, err := json.Marshal(config.Config) - if err != nil { - return nil, err - } - specOnlyJSON, err := transformToSpecOnly(configJSON) - if err != nil { - return nil, err - } - if err := json.Unmarshal(specOnlyJSON, &result[i].Config); err != nil { - return nil, err - } - } - return result, nil -} - -// transformTargetConfigsToSpecOnly transforms a slice of ConfigTarget to spec-only format -func transformTargetConfigsToSpecOnly(configs []model.ConfigTarget) ([]model.ConfigTarget, error) { - result := make([]model.ConfigTarget, len(configs)) - for i, config := range configs { - result[i] = config - configJSON, err := json.Marshal(config.Config) - if err != nil { - return nil, err - } - specOnlyJSON, err := transformToSpecOnly(configJSON) - if err != nil { - return nil, err - } - if err := json.Unmarshal(specOnlyJSON, &result[i].Config); err != nil { - return nil, err - } - } - return result, nil -} - // SourceConfigResponse represents a response containing configuration sources. type SourceConfigResponse struct { // Configs is a list of configuration sources. @@ -141,7 +56,7 @@ func ListConfigSources(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") - specOnly := c.Request.URL.Query().Get("spec_only") == "true" + specOnly := c.Request.URL.Query().Get("spec_only") == "false" limit, page, err := getPaginationParamFromURLQuery(c) @@ -153,7 +68,7 @@ func ListConfigSources(c *gin.Context) { return } - rows, totalCount, err := database.GetSourceConfigs(c, kind, id, config, limit, page) + rows, totalCount, err := database.GetSourceConfigs(c, kind, id, config, limit, page, specOnly) if err != nil { logrus.Errorf("searching for config source: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -162,17 +77,6 @@ func ListConfigSources(c *gin.Context) { return } - if specOnly { - rows, err = transformSourceConfigsToSpecOnly(rows) - if err != nil { - logrus.Errorf("transforming config sources to spec-only: %s", err) - c.JSON(http.StatusInternalServerError, DefaultResponseModel{ - Err: "failed to transform configs: " + err.Error(), - }) - return - } - } - c.JSON(http.StatusOK, SourceConfigResponse{ Configs: rows, TotalCount: totalCount, @@ -208,7 +112,7 @@ func SearchConfigSources(c *gin.Context) { return } - rows, totalCount, err := database.GetSourceConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page) + rows, totalCount, err := database.GetSourceConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page, false) if err != nil { logrus.Errorf("searching for config source: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -300,7 +204,7 @@ func ListConfigConditions(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") - specOnly := c.Request.URL.Query().Get("spec_only") == "true" + specOnly := c.Request.URL.Query().Get("spec_only") == "false" limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { @@ -311,7 +215,7 @@ func ListConfigConditions(c *gin.Context) { return } - rows, totalCount, err := database.GetConditionConfigs(c, kind, id, config, limit, page) + rows, totalCount, err := database.GetConditionConfigs(c, kind, id, config, limit, page, specOnly) if err != nil { logrus.Errorf("searching for config condition: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -320,17 +224,6 @@ func ListConfigConditions(c *gin.Context) { return } - if specOnly { - rows, err = transformConditionConfigsToSpecOnly(rows) - if err != nil { - logrus.Errorf("transforming config conditions to spec-only: %s", err) - c.JSON(http.StatusInternalServerError, DefaultResponseModel{ - Err: "failed to transform configs: " + err.Error(), - }) - return - } - } - c.JSON(http.StatusOK, ConditionConfigResponse{ Configs: rows, TotalCount: totalCount, @@ -371,7 +264,7 @@ func SearchConfigConditions(c *gin.Context) { return } - configs, totalCount, err := database.GetConditionConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page) + configs, totalCount, err := database.GetConditionConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page, false) if err != nil { logrus.Errorf("searching for config condition: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -426,7 +319,7 @@ func ListConfigTargets(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") - specOnly := c.Request.URL.Query().Get("spec_only") == "true" + specOnly := c.Request.URL.Query().Get("spec_only") == "false" limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { @@ -437,7 +330,7 @@ func ListConfigTargets(c *gin.Context) { return } - rows, totalCount, err := database.GetTargetConfigs(c, kind, id, config, limit, page) + rows, totalCount, err := database.GetTargetConfigs(c, kind, id, config, limit, page, specOnly) if err != nil { logrus.Errorf("searching for config target: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -446,17 +339,6 @@ func ListConfigTargets(c *gin.Context) { return } - if specOnly { - rows, err = transformTargetConfigsToSpecOnly(rows) - if err != nil { - logrus.Errorf("transforming config targets to spec-only: %s", err) - c.JSON(http.StatusInternalServerError, DefaultResponseModel{ - Err: "failed to transform configs: " + err.Error(), - }) - return - } - } - c.JSON(http.StatusOK, TargetConfigResponse{ Configs: rows, TotalCount: totalCount, @@ -492,7 +374,7 @@ func SearchConfigTargets(c *gin.Context) { return } - configs, totalCount, err := database.GetTargetConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page) + configs, totalCount, err := database.GetTargetConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page, false) if err != nil { logrus.Errorf("searching for config target: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ From c8e6ee7f372c2d8a0c30f622ae2c180573c1d713 Mon Sep 17 00:00:00 2001 From: josill Date: Thu, 29 Jan 2026 15:20:16 +0200 Subject: [PATCH 3/4] CR for myself --- pkg/database/config.go | 8 ++++---- pkg/server/configdb_handlers.go | 22 +++++++++++++++++++--- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/database/config.go b/pkg/database/config.go index cb93cc21..d7564158 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -137,7 +137,7 @@ func GetConfigKind(ctx context.Context, resourceType string) ([]string, error) { return nil, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) return nil, err @@ -231,7 +231,7 @@ func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page return nil, 0, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) @@ -337,7 +337,7 @@ func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, pa return nil, 0, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) @@ -446,7 +446,7 @@ func GetTargetConfigs(ctx context.Context, kind, id, config string, limit, page return nil, 0, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) diff --git a/pkg/server/configdb_handlers.go b/pkg/server/configdb_handlers.go index b780886a..1ab80bf8 100644 --- a/pkg/server/configdb_handlers.go +++ b/pkg/server/configdb_handlers.go @@ -3,6 +3,7 @@ package server import ( "encoding/json" "net/http" + "strconv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" @@ -56,7 +57,12 @@ func ListConfigSources(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") - specOnly := c.Request.URL.Query().Get("spec_only") == "false" + + specOnlyStr := c.Request.URL.Query().Get("spec_only") + specOnly, err := strconv.ParseBool(specOnlyStr) + if err != nil { + logrus.Warningf("ignoring spec_only param due to: %s", err) + } limit, page, err := getPaginationParamFromURLQuery(c) @@ -204,7 +210,12 @@ func ListConfigConditions(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") - specOnly := c.Request.URL.Query().Get("spec_only") == "false" + + specOnlyStr := c.Request.URL.Query().Get("spec_only") + specOnly, err := strconv.ParseBool(specOnlyStr) + if err != nil { + logrus.Warningf("ignoring spec_only param due to: %s", err) + } limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { @@ -319,7 +330,12 @@ func ListConfigTargets(c *gin.Context) { id := c.Request.URL.Query().Get("id") kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") - specOnly := c.Request.URL.Query().Get("spec_only") == "false" + + specOnlyStr := c.Request.URL.Query().Get("spec_only") + specOnly, err := strconv.ParseBool(specOnlyStr) + if err != nil { + logrus.Warningf("ignoring spec_only param due to: %s", err) + } limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { From 54ee238e256a3a82d0d25c225fd0537cb4bdd2dd Mon Sep 17 00:00:00 2001 From: josill Date: Thu, 29 Jan 2026 15:27:03 +0200 Subject: [PATCH 4/4] add db index --- .../000007_add_config_jsonb_indexes.down.sql | 9 +++++++++ .../000007_add_config_jsonb_indexes.up.sql | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+) create mode 100644 pkg/database/migrations/000007_add_config_jsonb_indexes.down.sql create mode 100644 pkg/database/migrations/000007_add_config_jsonb_indexes.up.sql diff --git a/pkg/database/migrations/000007_add_config_jsonb_indexes.down.sql b/pkg/database/migrations/000007_add_config_jsonb_indexes.down.sql new file mode 100644 index 00000000..b9f66434 --- /dev/null +++ b/pkg/database/migrations/000007_add_config_jsonb_indexes.down.sql @@ -0,0 +1,9 @@ +BEGIN; + +-- Drop GIN indexes on config JSONB columns + +DROP INDEX IF EXISTS idx_config_sources_config_gin; +DROP INDEX IF EXISTS idx_config_conditions_config_gin; +DROP INDEX IF EXISTS idx_config_targets_config_gin; + +COMMIT; diff --git a/pkg/database/migrations/000007_add_config_jsonb_indexes.up.sql b/pkg/database/migrations/000007_add_config_jsonb_indexes.up.sql new file mode 100644 index 00000000..4ccfa5ab --- /dev/null +++ b/pkg/database/migrations/000007_add_config_jsonb_indexes.up.sql @@ -0,0 +1,19 @@ +BEGIN; + +-- Add GIN indexes on config JSONB columns for faster queries +-- Using jsonb_path_ops for optimal performance on containment queries (config @> ?) +-- These indexes support: +-- - config @> ? (containment queries) - optimized with jsonb_path_ops +-- - config->'spec' (field extraction) +-- - Any JSONB operations on the config column + +CREATE INDEX IF NOT EXISTS idx_config_sources_config_gin +ON config_sources USING gin (config jsonb_path_ops); + +CREATE INDEX IF NOT EXISTS idx_config_conditions_config_gin +ON config_conditions USING gin (config jsonb_path_ops); + +CREATE INDEX IF NOT EXISTS idx_config_targets_config_gin +ON config_targets USING gin (config jsonb_path_ops); + +COMMIT;