diff --git a/docs/docs.go b/docs/docs.go index 9f049304..4d6f265b 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -83,6 +83,12 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -238,6 +244,12 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -358,6 +370,12 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -444,6 +462,12 @@ const docTemplate = `{ "/api/pipeline/reports": { "get": { "description": "List all pipeline reports from the database", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], "tags": [ "Pipeline Reports" ], @@ -466,6 +490,18 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering reports (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering reports (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -701,6 +737,18 @@ const docTemplate = `{ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering SCMs (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering SCMs (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -923,7 +971,15 @@ const docTemplate = `{ ] }, "pipelineID": { - "description": "PipelineID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same PipelineID.\nThis value is coming from the pipeline report to improve the search of reports.", + "description": "PipelineID represent the unique identifier of the pipeline.\nSeveral reports can be associated to the same PipelineID.", + "type": "string" + }, + "reportID": { + "description": "ReportID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same ReportID.\nThis value is coming from the pipeline report to improve the search of reports.", + "type": "string" + }, + "result": { + "description": "Result represent the result of the pipeline execution.", "type": "string" }, "sourceConfigIDs": { @@ -1563,6 +1619,10 @@ const docTemplate = `{ "description": "CaptureIndex defines which substring occurrence to retrieve. Note also that a value of ` + "`" + `0` + "`" + ` for ` + "`" + `captureIndex` + "`" + ` returns all submatches, and individual submatch indexes start at ` + "`" + `1` + "`" + `.", "type": "integer" }, + "capturePattern": { + "description": "Uses the match group(s) to generate the output using \\0, \\1, \\2, etc", + "type": "string" + }, "deprecatedCaptureIndex": { "type": "integer" }, @@ -1572,6 +1632,26 @@ const docTemplate = `{ } } }, + "transformer.JsonMatch": { + "type": "object", + "properties": { + "joinMultipleMatches": { + "description": "If we find multiple matches, join them by this", + "type": "string" + }, + "key": { + "type": "string" + }, + "multipleMatchSelector": { + "description": "If we find multiple matches, select the \"first\" or the \"last\"", + "type": "string" + }, + "noMatchResult": { + "description": "If we don't find a match then return the following string or the input value", + "type": "string" + } + } + }, "transformer.Replacer": { "type": "object", "properties": { @@ -1624,6 +1704,9 @@ const docTemplate = `{ } ] }, + "jsonMatch": { + "$ref": "#/definitions/transformer.JsonMatch" + }, "quote": { "description": "Quote add quote around the value", "type": "boolean" @@ -1644,7 +1727,7 @@ const docTemplate = `{ } }, "semVerInc": { - "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", + "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", "type": "string" }, "trimPrefix": { diff --git a/docs/swagger.json b/docs/swagger.json index d2419318..310cfc07 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -72,6 +72,12 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -227,6 +233,12 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -347,6 +359,12 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "boolean", + "description": "When true, returns only the Spec field from each Config object", + "name": "spec_only", + "in": "query" } ], "responses": { @@ -433,6 +451,12 @@ "/api/pipeline/reports": { "get": { "description": "List all pipeline reports from the database", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], "tags": [ "Pipeline Reports" ], @@ -455,6 +479,18 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering reports (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering reports (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -690,6 +726,18 @@ "description": "Page number for pagination, default is 1", "name": "page", "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering SCMs (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering SCMs (RFC3339 format)", + "name": "end_time", + "in": "query" } ], "responses": { @@ -912,7 +960,15 @@ ] }, "pipelineID": { - "description": "PipelineID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same PipelineID.\nThis value is coming from the pipeline report to improve the search of reports.", + "description": "PipelineID represent the unique identifier of the pipeline.\nSeveral reports can be associated to the same PipelineID.", + "type": "string" + }, + "reportID": { + "description": "ReportID represent the ID of the pipeline executed by Updatecli.\ndifferent execution of the same pipeline will have the same ReportID.\nThis value is coming from the pipeline report to improve the search of reports.", + "type": "string" + }, + "result": { + "description": "Result represent the result of the pipeline execution.", "type": "string" }, "sourceConfigIDs": { @@ -1552,6 +1608,10 @@ "description": "CaptureIndex defines which substring occurrence to retrieve. Note also that a value of `0` for `captureIndex` returns all submatches, and individual submatch indexes start at `1`.", "type": "integer" }, + "capturePattern": { + "description": "Uses the match group(s) to generate the output using \\0, \\1, \\2, etc", + "type": "string" + }, "deprecatedCaptureIndex": { "type": "integer" }, @@ -1561,6 +1621,26 @@ } } }, + "transformer.JsonMatch": { + "type": "object", + "properties": { + "joinMultipleMatches": { + "description": "If we find multiple matches, join them by this", + "type": "string" + }, + "key": { + "type": "string" + }, + "multipleMatchSelector": { + "description": "If we find multiple matches, select the \"first\" or the \"last\"", + "type": "string" + }, + "noMatchResult": { + "description": "If we don't find a match then return the following string or the input value", + "type": "string" + } + } + }, "transformer.Replacer": { "type": "object", "properties": { @@ -1613,6 +1693,9 @@ } ] }, + "jsonMatch": { + "$ref": "#/definitions/transformer.JsonMatch" + }, "quote": { "description": "Quote add quote around the value", "type": "boolean" @@ -1633,7 +1716,7 @@ } }, "semVerInc": { - "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", + "description": "SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded.", "type": "string" }, "trimPrefix": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 6892e61d..ec84a4e8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -170,10 +170,18 @@ definitions: description: Pipeline represent the Updatecli pipeline report. pipelineID: description: |- - PipelineID represent the ID of the pipeline executed by Updatecli. - different execution of the same pipeline will have the same PipelineID. + PipelineID represent the unique identifier of the pipeline. + Several reports can be associated to the same PipelineID. + type: string + reportID: + description: |- + ReportID represent the ID of the pipeline executed by Updatecli. + different execution of the same pipeline will have the same ReportID. This value is coming from the pipeline report to improve the search of reports. type: string + result: + description: Result represent the result of the pipeline execution. + type: string sourceConfigIDs: additionalProperties: type: string @@ -702,12 +710,31 @@ definitions: Note also that a value of `0` for `captureIndex` returns all submatches, and individual submatch indexes start at `1`. type: integer + capturePattern: + description: Uses the match group(s) to generate the output using \0, \1, + \2, etc + type: string deprecatedCaptureIndex: type: integer pattern: description: Pattern defines regular expression to use for retrieving a submatch type: string type: object + transformer.JsonMatch: + properties: + joinMultipleMatches: + description: If we find multiple matches, join them by this + type: string + key: + type: string + multipleMatchSelector: + description: If we find multiple matches, select the "first" or the "last" + type: string + noMatchResult: + description: If we don't find a match then return the following string or + the input value + type: string + type: object transformer.Replacer: properties: from: @@ -745,6 +772,8 @@ definitions: - $ref: '#/definitions/transformer.FindSubMatch' description: Find searches for a specific value if it exists then return the value using regular expression + jsonMatch: + $ref: '#/definitions/transformer.JsonMatch' quote: description: Quote add quote around the value type: boolean @@ -758,7 +787,7 @@ definitions: $ref: '#/definitions/transformer.Replacer' type: array semVerInc: - description: SemvVerInc specifies a comma separated list semantic versioning + description: SemvVerInc specifies a comma separated list semantic versioning component that needs to be upgraded. type: string trimPrefix: @@ -838,6 +867,10 @@ paths: in: query name: page type: string + - description: When true, returns only the Spec field from each Config object + in: query + name: spec_only + type: boolean responses: "200": description: OK @@ -940,6 +973,10 @@ paths: in: query name: page type: string + - description: When true, returns only the Spec field from each Config object + in: query + name: spec_only + type: boolean responses: "200": description: OK @@ -1019,6 +1056,10 @@ paths: in: query name: page type: string + - description: When true, returns only the Spec field from each Config object + in: query + name: spec_only + type: boolean responses: "200": description: OK @@ -1056,6 +1097,8 @@ paths: - Configuration Targets /api/pipeline/reports: get: + consumes: + - application/json description: List all pipeline reports from the database parameters: - description: SCM ID @@ -1070,6 +1113,16 @@ paths: in: query name: page type: string + - description: Start time for filtering reports (RFC3339 format) + in: query + name: start_time + type: string + - description: End time for filtering reports (RFC3339 format) + in: query + name: end_time + type: string + produces: + - application/json responses: "200": description: OK @@ -1226,6 +1279,14 @@ paths: in: query name: page type: string + - description: Start time for filtering SCMs (RFC3339 format) + in: query + name: start_time + type: string + - description: End time for filtering SCMs (RFC3339 format) + in: query + name: end_time + type: string responses: "200": description: OK diff --git a/pkg/database/config.go b/pkg/database/config.go index c8ee3473..d7564158 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -8,7 +8,9 @@ import ( "github.com/google/uuid" "github.com/sirupsen/logrus" + "github.com/stephenafamo/bob" "github.com/stephenafamo/bob/dialect/psql" + "github.com/stephenafamo/bob/dialect/psql/dialect" "github.com/stephenafamo/bob/dialect/psql/dm" "github.com/stephenafamo/bob/dialect/psql/im" "github.com/stephenafamo/bob/dialect/psql/sm" @@ -135,7 +137,7 @@ func GetConfigKind(ctx context.Context, resourceType string) ([]string, error) { return nil, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) return nil, err @@ -156,40 +158,54 @@ func GetConfigKind(ctx context.Context, resourceType string) ([]string, error) { } // GetSourceConfigs returns a list of resource configurations from the database. -func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page int) ([]model.ConfigSource, int, error) { +// If specOnly is true, only the Spec field is extracted from the config JSONB column. +func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page int, specOnly bool) ([]model.ConfigSource, int, error) { table := configSourceTableName - // SELECT id, kind, created_at, updated_at, config FROM " + table - query := psql.Select( - sm.Columns("id", "kind", "created_at", "updated_at", "config"), - sm.From(table), - ) + // When specOnly is true, use PostgreSQL JSONB to extract only the spec field + // jsonb_build_object('spec', config->'spec') creates a new JSONB object with only the spec field + var query *bob.BaseQuery[*dialect.SelectQuery] + if specOnly { + // SELECT id, kind, created_at, updated_at, jsonb_build_object('spec', config->'spec') as config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "jsonb_build_object('spec', config->'spec') as config"), + sm.From(table), + ) + query = &q + } else { + // SELECT id, kind, created_at, updated_at, config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "config"), + sm.From(table), + ) + query = &q + } if id != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("id").EQ(psql.Arg(id))), ) } if kind != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("kind").EQ(psql.Arg(kind))), ) } if config != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Raw("config @> ?", config)), ) } - query.Apply( + (*query).Apply( sm.OrderBy(psql.Quote("updated_at")).Desc(), ) // Get total count of results totalCount := 0 - totalQuery := psql.Select(sm.From(query), sm.Columns("count(*)")) + totalQuery := psql.Select(sm.From(*query), sm.Columns("count(*)")) totalQueryString, totalArgs, err := totalQuery.Build(ctx) if err != nil { logrus.Errorf("building total count query failed: %s\n\t%s", totalQueryString, err) @@ -203,19 +219,19 @@ func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page } if limit < totalCount && limit > 0 { - query.Apply( + (*query).Apply( sm.Limit(limit), sm.Offset((page-1)*limit), ) } - queryString, args, err := query.Build(ctx) + queryString, args, err := (*query).Build(ctx) if err != nil { logrus.Errorf("building query failed: %s\n\t%s", queryString, err) return nil, 0, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) @@ -248,34 +264,48 @@ func GetSourceConfigs(ctx context.Context, kind, id, config string, limit, page } // GetConditionConfigs returns a list of resource configurations from the database. -func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, page int) ([]model.ConfigCondition, int, error) { +// If specOnly is true, only the Spec field is extracted from the config JSONB column. +func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, page int, specOnly bool) ([]model.ConfigCondition, int, error) { table := configConditionTableName - // SELECT id, kind, created_at, updated_at, config FROM " + table - query := psql.Select( - sm.Columns("id", "kind", "created_at", "updated_at", "config"), - sm.From(table), - ) + // When specOnly is true, use PostgreSQL JSONB to extract only the spec field + // jsonb_build_object('spec', config->'spec') creates a new JSONB object with only the spec field + var query *bob.BaseQuery[*dialect.SelectQuery] + if specOnly { + // SELECT id, kind, created_at, updated_at, jsonb_build_object('spec', config->'spec') as config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "jsonb_build_object('spec', config->'spec') as config"), + sm.From(table), + ) + query = &q + } else { + // SELECT id, kind, created_at, updated_at, config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "config"), + sm.From(table), + ) + query = &q + } if id != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("id").EQ(psql.Arg(id))), ) } if kind != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("kind").EQ(psql.Arg(kind))), ) } if config != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Raw("config @> ?", config)), ) } - query.Apply( + (*query).Apply( sm.OrderBy(psql.Quote("updated_at")).Desc(), ) @@ -295,19 +325,19 @@ func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, pa // Apply pagination if limit and page are set if limit < totalCount && limit > 0 { - query.Apply( + (*query).Apply( sm.Limit(limit), sm.Offset((page-1)*limit), ) } - queryString, args, err := query.Build(ctx) + queryString, args, err := (*query).Build(ctx) if err != nil { logrus.Errorf("building query failed: %s\n\t%s", queryString, err) return nil, 0, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) @@ -343,34 +373,48 @@ func GetConditionConfigs(ctx context.Context, kind, id, config string, limit, pa } // GetTargetConfigs returns a list of resource configurations from the database. -func GetTargetConfigs(ctx context.Context, kind, id, config string, limit, page int) ([]model.ConfigTarget, int, error) { +// If specOnly is true, only the Spec field is extracted from the config JSONB column. +func GetTargetConfigs(ctx context.Context, kind, id, config string, limit, page int, specOnly bool) ([]model.ConfigTarget, int, error) { table := configTargetTableName - // SELECT id, kind, created_at, updated_at, config FROM " + table - query := psql.Select( - sm.Columns("id", "kind", "created_at", "updated_at", "config"), - sm.From(table), - ) + // When specOnly is true, use PostgreSQL JSONB to extract only the spec field + // jsonb_build_object('spec', config->'spec') creates a new JSONB object with only the spec field + var query *bob.BaseQuery[*dialect.SelectQuery] + if specOnly { + // SELECT id, kind, created_at, updated_at, jsonb_build_object('spec', config->'spec') as config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "jsonb_build_object('spec', config->'spec') as config"), + sm.From(table), + ) + query = &q + } else { + // SELECT id, kind, created_at, updated_at, config FROM " + table + q := psql.Select( + sm.Columns("id", "kind", "created_at", "updated_at", "config"), + sm.From(table), + ) + query = &q + } if id != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("id").EQ(psql.Arg(id))), ) } if kind != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Quote("kind").EQ(psql.Arg(kind))), ) } if config != "" { - query.Apply( + (*query).Apply( sm.Where(psql.Raw("config @> ?", config)), ) } - query.Apply( + (*query).Apply( sm.OrderBy(psql.Quote("updated_at")).Desc(), ) @@ -390,19 +434,19 @@ func GetTargetConfigs(ctx context.Context, kind, id, config string, limit, page // Apply pagination if limit and page are set if limit < totalCount && limit > 0 { - query.Apply( + (*query).Apply( sm.Limit(limit), sm.Offset((page-1)*limit), ) } - queryString, args, err := query.Build(ctx) + queryString, args, err := (*query).Build(ctx) if err != nil { logrus.Errorf("building query failed: %s\n\t%s", queryString, err) return nil, 0, err } - rows, err := DB.Query(context.Background(), queryString, args...) + rows, err := DB.Query(ctx, queryString, args...) if err != nil { logrus.Errorf("query failed: %q\n\t%s", queryString, err) diff --git a/pkg/database/migrations/000007_add_config_jsonb_indexes.down.sql b/pkg/database/migrations/000007_add_config_jsonb_indexes.down.sql new file mode 100644 index 00000000..b9f66434 --- /dev/null +++ b/pkg/database/migrations/000007_add_config_jsonb_indexes.down.sql @@ -0,0 +1,9 @@ +BEGIN; + +-- Drop GIN indexes on config JSONB columns + +DROP INDEX IF EXISTS idx_config_sources_config_gin; +DROP INDEX IF EXISTS idx_config_conditions_config_gin; +DROP INDEX IF EXISTS idx_config_targets_config_gin; + +COMMIT; diff --git a/pkg/database/migrations/000007_add_config_jsonb_indexes.up.sql b/pkg/database/migrations/000007_add_config_jsonb_indexes.up.sql new file mode 100644 index 00000000..4ccfa5ab --- /dev/null +++ b/pkg/database/migrations/000007_add_config_jsonb_indexes.up.sql @@ -0,0 +1,19 @@ +BEGIN; + +-- Add GIN indexes on config JSONB columns for faster queries +-- Using jsonb_path_ops for optimal performance on containment queries (config @> ?) +-- These indexes support: +-- - config @> ? (containment queries) - optimized with jsonb_path_ops +-- - config->'spec' (field extraction) +-- - Any JSONB operations on the config column + +CREATE INDEX IF NOT EXISTS idx_config_sources_config_gin +ON config_sources USING gin (config jsonb_path_ops); + +CREATE INDEX IF NOT EXISTS idx_config_conditions_config_gin +ON config_conditions USING gin (config jsonb_path_ops); + +CREATE INDEX IF NOT EXISTS idx_config_targets_config_gin +ON config_targets USING gin (config jsonb_path_ops); + +COMMIT; diff --git a/pkg/database/report.go b/pkg/database/report.go index 995537ac..55deebd8 100644 --- a/pkg/database/report.go +++ b/pkg/database/report.go @@ -344,7 +344,7 @@ func InsertReport(ctx context.Context, report reports.Report) (string, error) { continue } - results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1) + results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1, false) if err != nil { logrus.Errorf("failed: %s", err) continue @@ -429,7 +429,7 @@ func InsertReport(ctx context.Context, report reports.Report) (string, error) { continue } - results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1) + results, _, err := GetTargetConfigs(ctx, kind, "", string(data), 0, 1, false) if err != nil { logrus.Errorf("failed: %s", err) continue @@ -532,7 +532,7 @@ func buildConfigSources(ctx context.Context, report reports.Report) pgtype.Hstor continue } - results, _, err := GetSourceConfigs(ctx, kind, "", string(data), 0, 1) + results, _, err := GetSourceConfigs(ctx, kind, "", string(data), 0, 1, false) if err != nil { logrus.Errorf("failed: %s", err) continue diff --git a/pkg/server/configdb_handlers.go b/pkg/server/configdb_handlers.go index 318744c6..1ab80bf8 100644 --- a/pkg/server/configdb_handlers.go +++ b/pkg/server/configdb_handlers.go @@ -3,6 +3,7 @@ package server import ( "encoding/json" "net/http" + "strconv" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" @@ -48,6 +49,7 @@ type ConfigKindResponse struct { // @Param config query string false "Configuration of the source" // @Param limit query string false "Limit the number of reports returned, default is 100" // @Param page query string false "Page number for pagination, default is 1" +// @Param spec_only query boolean false "When true, returns only the Spec field from each Config object" // @Success 200 {object} SourceConfigResponse // @Failure 500 {object} DefaultResponseModel // @Router /api/pipeline/config/sources [get] @@ -56,6 +58,12 @@ func ListConfigSources(c *gin.Context) { kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") + specOnlyStr := c.Request.URL.Query().Get("spec_only") + specOnly, err := strconv.ParseBool(specOnlyStr) + if err != nil { + logrus.Warningf("ignoring spec_only param due to: %s", err) + } + limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { @@ -66,7 +74,7 @@ func ListConfigSources(c *gin.Context) { return } - rows, totalCount, err := database.GetSourceConfigs(c, kind, id, config, limit, page) + rows, totalCount, err := database.GetSourceConfigs(c, kind, id, config, limit, page, specOnly) if err != nil { logrus.Errorf("searching for config source: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -110,7 +118,7 @@ func SearchConfigSources(c *gin.Context) { return } - rows, totalCount, err := database.GetSourceConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page) + rows, totalCount, err := database.GetSourceConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page, false) if err != nil { logrus.Errorf("searching for config source: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -194,6 +202,7 @@ func DeleteConfigSource(c *gin.Context) { // @Param config query string false "Configuration of the condition" // @Param limit query string false "Limit the number of reports returned, default is 100" // @Param page query string false "Page number for pagination, default is 1" +// @Param spec_only query boolean false "When true, returns only the Spec field from each Config object" // @Success 200 {object} ConditionConfigResponse // @Failure 500 {object} DefaultResponseModel // @Router /api/pipeline/config/conditions [get] @@ -202,6 +211,12 @@ func ListConfigConditions(c *gin.Context) { kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") + specOnlyStr := c.Request.URL.Query().Get("spec_only") + specOnly, err := strconv.ParseBool(specOnlyStr) + if err != nil { + logrus.Warningf("ignoring spec_only param due to: %s", err) + } + limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { logrus.Errorf("invalid pagination parameters: %s", err) @@ -211,7 +226,7 @@ func ListConfigConditions(c *gin.Context) { return } - rows, totalCount, err := database.GetConditionConfigs(c, kind, id, config, limit, page) + rows, totalCount, err := database.GetConditionConfigs(c, kind, id, config, limit, page, specOnly) if err != nil { logrus.Errorf("searching for config condition: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -219,6 +234,7 @@ func ListConfigConditions(c *gin.Context) { }) return } + c.JSON(http.StatusOK, ConditionConfigResponse{ Configs: rows, TotalCount: totalCount, @@ -259,7 +275,7 @@ func SearchConfigConditions(c *gin.Context) { return } - configs, totalCount, err := database.GetConditionConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page) + configs, totalCount, err := database.GetConditionConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page, false) if err != nil { logrus.Errorf("searching for config condition: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -306,6 +322,7 @@ func DeleteConfigCondition(c *gin.Context) { // @Param config query string false "Configuration of the target" // @Param limit query string false "Limit the number of reports returned, default is 100" // @Param page query string false "Page number for pagination, default is 1" +// @Param spec_only query boolean false "When true, returns only the Spec field from each Config object" // @Success 200 {object} TargetConfigResponse // @Failure 500 {object} DefaultResponseModel // @Router /api/pipeline/config/targets [get] @@ -314,6 +331,12 @@ func ListConfigTargets(c *gin.Context) { kind := c.Request.URL.Query().Get("kind") config := c.Request.URL.Query().Get("config") + specOnlyStr := c.Request.URL.Query().Get("spec_only") + specOnly, err := strconv.ParseBool(specOnlyStr) + if err != nil { + logrus.Warningf("ignoring spec_only param due to: %s", err) + } + limit, page, err := getPaginationParamFromURLQuery(c) if err != nil { logrus.Errorf("invalid pagination parameters: %s", err) @@ -323,7 +346,7 @@ func ListConfigTargets(c *gin.Context) { return } - rows, totalCount, err := database.GetTargetConfigs(c, kind, id, config, limit, page) + rows, totalCount, err := database.GetTargetConfigs(c, kind, id, config, limit, page, specOnly) if err != nil { logrus.Errorf("searching for config target: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ @@ -331,6 +354,7 @@ func ListConfigTargets(c *gin.Context) { }) return } + c.JSON(http.StatusOK, TargetConfigResponse{ Configs: rows, TotalCount: totalCount, @@ -366,7 +390,7 @@ func SearchConfigTargets(c *gin.Context) { return } - configs, totalCount, err := database.GetTargetConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page) + configs, totalCount, err := database.GetTargetConfigs(c, queryConfig.Kind, queryConfig.ID, string(queryConfig.Config), queryConfig.Limit, queryConfig.Page, false) if err != nil { logrus.Errorf("searching for config target: %s", err) c.JSON(http.StatusInternalServerError, DefaultResponseModel{ diff --git a/pkg/server/endpoints_test.go b/pkg/server/endpoints_test.go index 27207340..fa2659db 100644 --- a/pkg/server/endpoints_test.go +++ b/pkg/server/endpoints_test.go @@ -396,6 +396,138 @@ func TestEndpoints(t *testing.T) { }, }, removeFieldsAsserter("configs", "Created_at", "Updated_at")) }) + + t.Run("with spec_only=true", func(t *testing.T) { + // Insert a config with Spec, Transformers, Name, etc. + fullConfig := map[string]any{ + "spec": map[string]any{ + "owner": "updatecli", + "repository": "updatecli", + "token": "test-token", + }, + "transformers": []any{ + map[string]any{ + "addPrefix": "v", + }, + }, + "name": "test-source", + "kind": "githubrelease", + } + configID, err := database.InsertConfigResource(ctx, "source", "githubrelease", fullConfig) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, database.DeleteConfigResource(ctx, "source", configID)) + }) + + // Test without spec_only - should return full config + resp := doGetRequest(t, srv, "/api/pipeline/config/sources?id="+configID) + var resultWithoutSpecOnly map[string]any + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &resultWithoutSpecOnly)) + configs := resultWithoutSpecOnly["configs"].([]any) + require.Len(t, configs, 1) + config := configs[0].(map[string]any) + configObj := config["Config"].(map[string]any) + // Should have transformers, name, etc. + assert.NotNil(t, configObj["Transformers"]) + assert.NotNil(t, configObj["Spec"]) + + // Test with spec_only=true - should return only Spec + resp = doGetRequest(t, srv, "/api/pipeline/config/sources?id="+configID+"&spec_only=true") + var resultWithSpecOnly map[string]any + b, _ = io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &resultWithSpecOnly)) + configs = resultWithSpecOnly["configs"].([]any) + require.Len(t, configs, 1) + config = configs[0].(map[string]any) + configObj = config["Config"].(map[string]any) + // Should only have Spec field + assert.NotNil(t, configObj["Spec"]) + // Transformers should be nil/empty + transformers, hasTransformers := configObj["Transformers"] + if hasTransformers { + assert.Nil(t, transformers) + } + // Name should be empty + name, hasName := configObj["Name"] + if hasName { + assert.Empty(t, name) + } + }) + }) + }) + + t.Run("GET /api/pipeline/config/conditions", func(t *testing.T) { + t.Run("with spec_only=true", func(t *testing.T) { + fullConfig := map[string]any{ + "spec": map[string]any{ + "file": "/path/to/file", + "key": "version", + }, + "transformers": []any{ + map[string]any{ + "addSuffix": ".0", + }, + }, + "name": "test-condition", + "kind": "file", + } + configID, err := database.InsertConfigResource(ctx, "condition", "file", fullConfig) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, database.DeleteConfigResource(ctx, "condition", configID)) + }) + + // Test with spec_only=true + resp := doGetRequest(t, srv, "/api/pipeline/config/conditions?id="+configID+"&spec_only=true") + var result map[string]any + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &result)) + configs := result["configs"].([]any) + require.Len(t, configs, 1) + config := configs[0].(map[string]any) + configObj := config["Config"].(map[string]any) + // Should only have Spec field + assert.NotNil(t, configObj["Spec"]) + }) + }) + + t.Run("GET /api/pipeline/config/targets", func(t *testing.T) { + t.Run("with spec_only=true", func(t *testing.T) { + fullConfig := map[string]any{ + "spec": map[string]any{ + "file": "/path/to/target", + "key": "version", + }, + "transformers": []any{ + map[string]any{ + "trimPrefix": "v", + }, + }, + "name": "test-target", + "kind": "yaml", + } + configID, err := database.InsertConfigResource(ctx, "target", "yaml", fullConfig) + require.NoError(t, err) + t.Cleanup(func() { + assert.NoError(t, database.DeleteConfigResource(ctx, "target", configID)) + }) + + // Test with spec_only=true + resp := doGetRequest(t, srv, "/api/pipeline/config/targets?id="+configID+"&spec_only=true") + var result map[string]any + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + require.NoError(t, json.Unmarshal(b, &result)) + configs := result["configs"].([]any) + require.Len(t, configs, 1) + config := configs[0].(map[string]any) + configObj := config["Config"].(map[string]any) + // Should only have Spec field + assert.NotNil(t, configObj["Spec"]) }) }) }