From 2183f3ff09f0eff39d519fe22181b34084422ac3 Mon Sep 17 00:00:00 2001 From: Andy Li Date: Mon, 26 May 2025 14:32:39 -0400 Subject: [PATCH] feat(dora): add issue lead time metrics calculation [DX-85] --- .github/workflows/build-push-ecr.yml | 118 +++++++++++++ backend/plugins/dora/impl/impl.go | 2 + .../dora/models/issue_lead_time_metric.go | 38 ++++ ...50424_add_issue_lead_time_metrics_table.go | 55 ++++++ .../dora/models/migrationscripts/register.go | 1 + .../tasks/deployment_commits_generator.go | 6 +- .../dora/tasks/deployment_generator.go | 6 +- .../dora/tasks/issue_lead_time_calculator.go | 163 ++++++++++++++++++ .../tasks/prev_deployment_commit_enricher.go | 4 +- backend/plugins/dora/tasks/task_data.go | 61 +++++-- 10 files changed, 436 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/build-push-ecr.yml create mode 100644 backend/plugins/dora/models/issue_lead_time_metric.go create mode 100644 backend/plugins/dora/models/migrationscripts/20250424_add_issue_lead_time_metrics_table.go create mode 100644 backend/plugins/dora/tasks/issue_lead_time_calculator.go diff --git a/.github/workflows/build-push-ecr.yml b/.github/workflows/build-push-ecr.yml new file mode 100644 index 00000000000..becec6c7668 --- /dev/null +++ b/.github/workflows/build-push-ecr.yml @@ -0,0 +1,118 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: DevLake Build and Push to ECR +run-name: DevLake build and push to ECR by @${{ github.actor }} + +on: + workflow_dispatch: + push: + branches: + - master + - andy/dx-85 + +permissions: + id-token: write # Required for JWT + contents: read # Required for checkout + +env: + LATEST_TAG: v1.0.2-beta4 + COMMIT_TAG: ${{ github.sha }} + AWS_REGION: us-east-1 + IAM_ROLE_ARN: arn:aws:iam::130726505375:role/github-actions-ecr + ECR: 130726505375.dkr.ecr.us-east-1.amazonaws.com + IMAGE_NAME_SERVER: devlake-server + IMAGE_NAME_CONFIG_UI: devlake-config-ui + +jobs: + build-and-push-server: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 #v3.5.1 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0 + with: + role-to-assume: ${{ env.IAM_ROLE_ARN }} + role-session-name: github-action-devlake-server-build + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + registry: ${{ env.ECR }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c + + - name: Build and push container image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: ./backend + push: true + file: ./backend/Dockerfile + tags: | + ${{ env.ECR }}/${{ env.IMAGE_NAME_SERVER }}:${{ env.LATEST_TAG }} + ${{ env.ECR }}/${{ env.IMAGE_NAME_SERVER }}:${{ env.COMMIT_TAG }} + platforms: linux/amd64,linux/arm64 + build-args: | + TAG=${{ github.ref_name }} + SHA=${{ github.sha }} + + build-and-push-config-ui: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@83b7061638ee4956cf7545a6f7efe594e5ad0247 #v3.5.1 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e1e17a757e536f70e52b5a12b2e8d1d1c60e04ef # v2.0.0 + with: + role-to-assume: ${{ env.IAM_ROLE_ARN }} + role-session-name: github-action-devlake-config-ui-build + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a + with: + registry: ${{ env.ECR }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c + + - name: Build and push container image + uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 + with: + context: ./config-ui + push: true + file: ./config-ui/Dockerfile + tags: | + ${{ env.ECR }}/${{ env.IMAGE_NAME_CONFIG_UI }}:${{ env.LATEST_TAG }} + ${{ env.ECR }}/${{ env.IMAGE_NAME_CONFIG_UI }}:${{ env.COMMIT_TAG }} + platforms: linux/amd64,linux/arm64 + build-args: | + TAG=${{ github.ref_name }} + SHA=${{ github.sha }} diff --git a/backend/plugins/dora/impl/impl.go b/backend/plugins/dora/impl/impl.go index 24609739811..b584763ce97 100644 --- a/backend/plugins/dora/impl/impl.go +++ b/backend/plugins/dora/impl/impl.go @@ -94,6 +94,7 @@ func (p Dora) SubTaskMetas() []plugin.SubTaskMeta { tasks.EnrichPrevSuccessDeploymentCommitMeta, tasks.EnrichTaskEnvMeta, tasks.CalculateChangeLeadTimeMeta, + tasks.CalculateIssueLeadTimeMeta, tasks.IssuesToIncidentsMeta, tasks.ConnectIncidentToDeploymentMeta, } @@ -160,6 +161,7 @@ func (p Dora) MakeMetricPluginPipelinePlanV200(projectName string, options json. }, Subtasks: []string{ "calculateChangeLeadTime", + "calculateIssueLeadTime", tasks.IssuesToIncidentsMeta.Name, "ConnectIncidentToDeployment", }, diff --git a/backend/plugins/dora/models/issue_lead_time_metric.go b/backend/plugins/dora/models/issue_lead_time_metric.go new file mode 100644 index 00000000000..3d55f8fd67b --- /dev/null +++ b/backend/plugins/dora/models/issue_lead_time_metric.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" +) + +// IssueLeadTimeMetric tracks lead time for issues from in-progress to done +type IssueLeadTimeMetric struct { + ProjectName string `json:"projectName" gorm:"primaryKey;type:varchar(255)"` + IssueId string `json:"issueId" gorm:"primaryKey;type:varchar(255)"` + InProgressDate *time.Time `json:"InProgressDate"` + DoneDate *time.Time `json:"DoneDate"` + + // Lead time in minutes from first 'In Progress' to first 'Done' + InProgressToDoneMinutes *int64 `json:"inProgressToDoneMinutes"` +} + +// TableName specifies the database table name +func (IssueLeadTimeMetric) TableName() string { + return "_tool_dora_issue_lead_time_metrics" +} diff --git a/backend/plugins/dora/models/migrationscripts/20250424_add_issue_lead_time_metrics_table.go b/backend/plugins/dora/models/migrationscripts/20250424_add_issue_lead_time_metrics_table.go new file mode 100644 index 00000000000..67891c58bdd --- /dev/null +++ b/backend/plugins/dora/models/migrationscripts/20250424_add_issue_lead_time_metrics_table.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" +) + +// Define the actual table structure directly in the migration script +type issueLeadTimeMetric struct { + ProjectName string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"primaryKey;type:varchar(255)"` + InProgressDate *time.Time + DoneDate *time.Time + InProgressToDoneMinutes *int64 +} + +// TableName specifies the table name +func (issueLeadTimeMetric) TableName() string { + return "_tool_dora_issue_lead_time_metrics" +} + +type addIssueLeadTimeMetricsTable struct{} + +func (script *addIssueLeadTimeMetricsTable) Up(baseRes context.BasicRes) errors.Error { + db := baseRes.GetDal() + // Use our directly defined model instead of importing from models + return db.AutoMigrate(&issueLeadTimeMetric{}) +} + +func (*addIssueLeadTimeMetricsTable) Version() uint64 { + return 2025042401 +} + +func (*addIssueLeadTimeMetricsTable) Name() string { + return "dora add _tool_dora_issue_lead_time_metrics table" +} diff --git a/backend/plugins/dora/models/migrationscripts/register.go b/backend/plugins/dora/models/migrationscripts/register.go index 6a12d071963..a6ba46b06b4 100644 --- a/backend/plugins/dora/models/migrationscripts/register.go +++ b/backend/plugins/dora/models/migrationscripts/register.go @@ -27,5 +27,6 @@ func All() []plugin.MigrationScript { new(addDoraBenchmark), new(fixDoraBenchmarkMetric), new(adddoraBenchmark2023), + new(addIssueLeadTimeMetricsTable), } } diff --git a/backend/plugins/dora/tasks/deployment_commits_generator.go b/backend/plugins/dora/tasks/deployment_commits_generator.go index 826e177288f..388e4fa784d 100644 --- a/backend/plugins/dora/tasks/deployment_commits_generator.go +++ b/backend/plugins/dora/tasks/deployment_commits_generator.go @@ -103,11 +103,11 @@ func GenerateDeploymentCommits(taskCtx plugin.SubTaskContext) errors.Error { noneSkippedResult, ), } - if data.Options.ScopeId != nil { - clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`, data.Options.ScopeId)) + if data.ScopeId != "" { + clauses = append(clauses, dal.Where(`p.cicd_scope_id = ?`, data.ScopeId)) // Clear previous results from the project deleteSql := `DELETE FROM cicd_deployment_commits WHERE cicd_scope_id = ? and subtask_name = ?;` - err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeploymentCommits) + err := db.Exec(deleteSql, data.ScopeId, DORAGenerateDeploymentCommits) if err != nil { return errors.Default.Wrap(err, "error deleting previous cicd_deployment_commits") } diff --git a/backend/plugins/dora/tasks/deployment_generator.go b/backend/plugins/dora/tasks/deployment_generator.go index 0defd4205fe..9a183db80a9 100644 --- a/backend/plugins/dora/tasks/deployment_generator.go +++ b/backend/plugins/dora/tasks/deployment_generator.go @@ -76,13 +76,13 @@ func GenerateDeployment(taskCtx plugin.SubTaskContext) errors.Error { noneSkippedResult, ), } - if data.Options.ScopeId != nil { + if data.ScopeId != "" { clauses = append(clauses, - dal.Where("p.cicd_scope_id = ?", data.Options.ScopeId), + dal.Where("p.cicd_scope_id = ?", data.ScopeId), ) // Clear previous results from the cicd_scope_id deleteSql := `DELETE FROM cicd_deployments WHERE cicd_scope_id = ? and subtask_name = ?;` - err := db.Exec(deleteSql, data.Options.ScopeId, DORAGenerateDeployment) + err := db.Exec(deleteSql, data.ScopeId, DORAGenerateDeployment) if err != nil { return errors.Default.Wrap(err, "error deleting previous deployments") } diff --git a/backend/plugins/dora/tasks/issue_lead_time_calculator.go b/backend/plugins/dora/tasks/issue_lead_time_calculator.go new file mode 100644 index 00000000000..494f47c8e6a --- /dev/null +++ b/backend/plugins/dora/tasks/issue_lead_time_calculator.go @@ -0,0 +1,163 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "database/sql" + "fmt" + "strconv" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/dora/models" + jiraModels "github.com/apache/incubator-devlake/plugins/jira/models" +) + +// CalculateIssueLeadTimeMeta contains metadata for the CalculateIssueLeadTime subtask. +var CalculateIssueLeadTimeMeta = plugin.SubTaskMeta{ + Name: "calculateIssueLeadTime", + EntryPoint: CalculateIssueLeadTime, + EnabledByDefault: true, + Description: "Calculate issue lead time from first 'In Progress' to first 'Done'", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +// CalculateIssueLeadTime calculates the lead time for issues from first 'In Progress' status to 'Done' status. +func CalculateIssueLeadTime(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + logger := taskCtx.GetLogger() + data := taskCtx.GetData().(*DoraTaskData) + + logger.Info("Starting calculateIssueLeadTime task for project %s", data.Options.ProjectName) + + // Delete any old metrics for this project + err := db.Delete( + &models.IssueLeadTimeMetric{}, + dal.Where("project_name = ?", data.Options.ProjectName), + ) + if err != nil { + return errors.Default.Wrap(err, "failed to delete old issue lead time metrics") + } + logger.Info("Deleted old issue lead time metrics for project %s", data.Options.ProjectName) + + // Get the actual _tool_jira_* table names + rawItems := jiraModels.JiraIssueChangelogItems{}.TableName() + rawChgs := jiraModels.JiraIssueChangelogs{}.TableName() + rawIss := jiraModels.JiraIssue{}.TableName() + + // Build the SQL query, filter out null timestamps and use only latest resolution per issue + query := fmt.Sprintf(` + SELECT + c.issue_id AS issue_id, + MIN(CASE WHEN UPPER(TRIM(i.to_string)) IN ('IN PROGRESS', 'INPROGRESS') THEN c.created END) AS in_progress_timestamp, + u.resolution_date AS done_timestamp + FROM %s i + JOIN %s c + ON i.connection_id = c.connection_id + AND i.changelog_id = c.changelog_id + JOIN %s u + ON c.connection_id = u.connection_id + AND c.issue_id = u.issue_id + JOIN _tool_jira_board_issues bi + ON u.connection_id = bi.connection_id + AND u.issue_id = bi.issue_id + JOIN project_mapping pm + ON pm.row_id = CONCAT('jira:JiraBoard:', bi.connection_id, ':', bi.board_id) + AND pm.table = 'boards' + WHERE i.field = 'status' + AND pm.project_name = ? + AND u.resolution_date IS NOT NULL + AND u.resolution_date = ( + SELECT MAX(u2.resolution_date) + FROM %s u2 + WHERE u2.connection_id = u.connection_id + AND u2.issue_id = u.issue_id + ) + GROUP BY c.issue_id, u.resolution_date + HAVING in_progress_timestamp IS NOT NULL + `, rawItems, rawChgs, rawIss, rawIss) + + logger.Info("Executing SQL query for DevLake project: %s", data.Options.ProjectName) + + // Execute query and stream results + rows, err := db.RawCursor(query, data.Options.ProjectName) + if err != nil { + return errors.Default.Wrap(err, "failed to run lead time aggregation query") + } + defer rows.Close() + + rowCount := 0 + logger.Info("Starting to process SQL query results...") + + for rows.Next() { + var ( + rawIssueID uint64 + rawInProgress sql.NullTime + rawDone sql.NullTime + ) + + if scanErr := rows.Scan(&rawIssueID, &rawInProgress, &rawDone); scanErr != nil { + return errors.Default.Wrap(scanErr, "failed to scan lead time row") + } + + logger.Debug("Scanned row: issueID=%d, inProgress=%v, done=%v", rawIssueID, rawInProgress, rawDone) + + // Skip if null timestamps + if !rawInProgress.Valid || !rawDone.Valid { + logger.Debug("Skipping row with null timestamp: issueID=%d", rawIssueID) + continue + } + + start := rawInProgress.Time + end := rawDone.Time + mins := int64(end.Sub(start).Minutes()) + + // Skip negative lead times + if mins < 0 { + logger.Debug("Skipping row with negative lead time: issueID=%d", rawIssueID) + continue + } + + // Create and save the metric + metric := &models.IssueLeadTimeMetric{ + ProjectName: data.Options.ProjectName, + IssueId: strconv.FormatUint(rawIssueID, 10), + InProgressDate: &start, + DoneDate: &end, + InProgressToDoneMinutes: &mins, + } + + logger.Debug("Upserting metric: projectName=%s, issueId=%s, minutes=%d", + metric.ProjectName, metric.IssueId, *metric.InProgressToDoneMinutes) + + if upsertErr := db.CreateOrUpdate(metric); upsertErr != nil { + return errors.Default.Wrap(upsertErr, "failed to upsert issue lead time metric") + } + + rowCount++ + } + + logger.Info("Completed calculateIssueLeadTime task: processed %d records", rowCount) + + if err := rows.Err(); err != nil && err != sql.ErrNoRows { + return errors.Default.Wrap(err, "error iterating lead time rows") + } + + return nil +} diff --git a/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go b/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go index 69c68a9f68a..ef7c507c47a 100644 --- a/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go +++ b/backend/plugins/dora/tasks/prev_deployment_commit_enricher.go @@ -61,9 +61,9 @@ func EnrichPrevSuccessDeploymentCommit(taskCtx plugin.SubTaskContext) errors.Err devops.RESULT_SUCCESS, ), } - if data.Options.ScopeId != nil { + if data.ScopeId != "" { clauses = append(clauses, - dal.Where("dc.cicd_scope_id = ?", data.Options.ScopeId), + dal.Where("dc.cicd_scope_id = ?", data.ScopeId), dal.Orderby("dc.repo_url, dc.environment, dc.finished_date"), ) } else { diff --git a/backend/plugins/dora/tasks/task_data.go b/backend/plugins/dora/tasks/task_data.go index 3beaf339f6b..b49e03a4644 100644 --- a/backend/plugins/dora/tasks/task_data.go +++ b/backend/plugins/dora/tasks/task_data.go @@ -18,8 +18,11 @@ limitations under the License. package tasks import ( + "strings" + "time" + "github.com/apache/incubator-devlake/core/errors" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" ) type DoraApiParams struct { @@ -27,23 +30,61 @@ type DoraApiParams struct { } type DoraOptions struct { - Tasks []string `json:"tasks,omitempty"` - Since string - ProjectName string `json:"projectName"` - ScopeId *string `json:"scopeId,omitempty"` + // options for dora plugin, required when activate dora + ProjectName string `json:"projectName"` + RepoUrl string `json:"repoUrl"` // optional, for locating repo + // Time after which the data represents the real history, like github cloud is 2015-01-01, or the project start date + TimeAfter *time.Time `json:"timeAfter"` + + // Specify the issue statuses that indicate 'In Progress'. Can be multiple comma-separated values. + // e.g., "In Progress,In Development,In Review" + InProgressStatus string `json:"inProgressStatus"` + + // Specify the issue statuses that indicate 'Done'. Can be multiple comma-separated values. + // e.g., "Done,Closed,Resolved" + DoneStatus string `json:"doneStatus"` + + // --- Keep original fields needed by other DORA tasks --- + ScopeConfigId uint64 `json:"scopeConfigId"` + ScopeConfigName string `json:"scopeConfigName"` +} + +func (o *DoraOptions) GetInProgressStatuses() []string { + if o.InProgressStatus == "" { + return []string{} + } + return strings.Split(o.InProgressStatus, ",") +} + +func (o *DoraOptions) GetDoneStatuses() []string { + if o.DoneStatus == "" { + return []string{} + } + return strings.Split(o.DoneStatus, ",") } type DoraTaskData struct { - Options *DoraOptions - DisableIssueToIncidentGenerator bool + Options *DoraOptions + TimeAfter *time.Time + RepoUrl string `json:"repoUrl"` // optional, for locating repo + + // --- Keep fields needed by other DORA tasks --- + ScopeId string `json:"scopeId"` + ScopeName string `json:"scopeName"` + DisableIssueToIncidentGenerator bool `json:"disableIssueToIncidentGenerator"` + + RemoteArgs interface{} // Temporary workaround to let it compile } func DecodeAndValidateTaskOptions(options map[string]interface{}) (*DoraOptions, errors.Error) { var op DoraOptions - err := helper.Decode(options, &op, nil) + err := api.Decode(options, &op, nil) if err != nil { - return nil, errors.Default.Wrap(err, "error decoding DORA task options") + return nil, err + } + // find the scope config if owner/repo is not specified + if op.ProjectName == "" { + return nil, errors.BadInput.New("projectName is required for dora plugin") } - return &op, nil }