Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,3 +51,27 @@ func TestConnectIncidentToDeploymentDataFlow(t *testing.T) {
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
}

// Incidents whose component names a CI/CD scope of the project (by name or id, any case) are tied to that scope's
// latest production deployment; any other component, or none, keeps the project-wide lookup.
func TestConnectIncidentToDeploymentByComponentDataFlow(t *testing.T) {
var plugin impl.Dora
dataflowTester := e2ehelper.NewDataFlowTester(t, "dora", plugin)

taskData := &tasks.DoraTaskData{
Options: &tasks.DoraOptions{
ProjectName: "project1",
},
}
dataflowTester.ImportCsvIntoTabler("./connect_incident_to_deployment/prev_success_deployment_commit/cicd_deployment_commits_after.csv", &devops.CicdDeploymentCommit{})
dataflowTester.ImportCsvIntoTabler("./connect_incident_to_deployment/raw_tables/project_mapping.csv", &crossdomain.ProjectMapping{})
dataflowTester.ImportCsvIntoTabler("./connect_incident_to_deployment/by_component/cicd_scopes.csv", &devops.CicdScope{})
dataflowTester.ImportCsvIntoTabler("./connect_incident_to_deployment/by_component/incidents.csv", &ticket.Incident{})

dataflowTester.FlushTabler(&crossdomain.ProjectIncidentDeploymentRelationship{})
dataflowTester.Subtask(tasks.ConnectIncidentToDeploymentMeta, taskData)
dataflowTester.VerifyTableWithOptions(&crossdomain.ProjectIncidentDeploymentRelationship{}, e2ehelper.TableOptions{
CSVRelPath: "./connect_incident_to_deployment/by_component/project_incident_deployment_relationships.csv",
IgnoreTypes: []interface{}{common.NoPKModel{}},
})
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
id,name
cicd1,owner/repo-one
cicd2,owner/repo-two
cicd3,owner/repo-three
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
id,created_date,updated_date,table,scope_id,component
incident:by-name,2022-11-13 23:37:21,2022-11-15 22:37:21,repos,repo1,owner/repo-one
incident:by-name-other-case,2022-11-13 01:07:21,2022-11-14 00:07:21,repos,repo1,OWNER/Repo-One
incident:by-id,2022-11-02 08:21:09,2022-11-04 07:21:09,repos,repo1,cicd1
incident:unknown-component,2022-11-16 21:34:01,2022-11-19 20:34:01,repos,repo2,someone/else
incident:no-component,2022-11-28 01:46:47,2022-11-29 00:46:47,repos,repo2,
incident:scope-earlier-than-project,2022-09-14 12:00:00,2022-09-15 12:00:00,repos,repo1,owner/repo-one
incident:scope-has-no-earlier-deploy,2022-09-13 12:00:00,2022-09-14 12:00:00,repos,repo1,owner/repo-two
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
id,project_name,deployment_id
incident:by-id,project1,pipeline6
incident:by-name,project1,pipeline6
incident:by-name-other-case,project1,pipeline6
incident:no-component,project1,pipeline7
incident:scope-earlier-than-project,project1,pipeline1
incident:scope-has-no-earlier-deploy,project1,pipeline1
incident:unknown-component,project1,pipeline7
89 changes: 58 additions & 31 deletions backend/plugins/dora/tasks/incident_deploy_connector.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package tasks

import (
"reflect"
"strings"
"time"

"github.com/apache/devlake/core/dal"
Expand Down Expand Up @@ -99,40 +100,17 @@ func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {
ProjectName: data.Options.ProjectName,
}
logger.Debug("get incident: %+v", incident.Id)
cicdDeploymentCommit := &devops.CicdDeploymentCommit{}
cicdDeploymentCommitClauses := []dal.Clause{
dal.Select("cicd_deployment_commits.cicd_deployment_id as id, cicd_deployment_commits.finished_date as finished_date"),
dal.From(cicdDeploymentCommit),
dal.Join("left join project_mapping pm on cicd_deployment_commits.cicd_scope_id = pm.row_id"),
dal.Where(
`cicd_deployment_commits.finished_date < ?
and cicd_deployment_commits.result = ?
and cicd_deployment_commits.environment = ?
and pm.table = ?
and pm.project_name = ?`,
incident.CreatedDate, devops.RESULT_SUCCESS, devops.PRODUCTION, "cicd_scopes", data.Options.ProjectName,
),
dal.Orderby("finished_date DESC"),
dal.Limit(1),
}

scdc := &simpleCicdDeploymentCommit{}
err = db.All(scdc, cicdDeploymentCommitClauses...)
scdc, err := findIncidentDeployment(db, incident, data.Options.ProjectName)
if err != nil {
if db.IsErrorNotFound(err) {
logger.Warn(err, "deployment commit not found")
return nil, nil
} else {
logger.Error(err, "get all deployment commits")
return nil, err
}
logger.Error(err, "get all deployment commits")
return nil, err
}
if scdc.Id != "" {
projectIssueMetric.DeploymentId = scdc.Id
return []interface{}{projectIssueMetric}, nil
if scdc == nil {
logger.Warn(nil, "deployment commit not found for incident %s", incident.Id)
return nil, nil
}
logger.Debug("scdc.id is empty, incident will be ignored: %+v", incident.Id)
return nil, nil
projectIssueMetric.DeploymentId = scdc.Id
return []interface{}{projectIssueMetric}, nil
},
})
if err != nil {
Expand All @@ -141,3 +119,52 @@ func ConnectIncidentToDeployment(taskCtx plugin.SubTaskContext) errors.Error {

return enricher.Execute()
}

// findIncidentDeployment returns the latest successful production deployment of the project that finished before
// the incident started. When the incident names a component that matches a CI/CD scope of the project (its name,
// such as "owner/repo", or its id, case-insensitive), only that scope's deployments are considered, so the incident
// is tied to the release of the repository that caused it rather than to whichever repository deployed last. If the
// component matches no scope, or that scope has no earlier production deployment, the project-wide lookup is used.
func findIncidentDeployment(db dal.Dal, incident *ticket.Incident, projectName string) (*simpleCicdDeploymentCommit, errors.Error) {
component := strings.TrimSpace(incident.Component)
if component != "" {
scdc, err := latestProductionDeployment(db, incident, projectName, component)
if err != nil || scdc != nil {
return scdc, err
}
}
return latestProductionDeployment(db, incident, projectName, "")
}

func latestProductionDeployment(db dal.Dal, incident *ticket.Incident, projectName string, component string) (*simpleCicdDeploymentCommit, errors.Error) {
clauses := []dal.Clause{
dal.Select("cicd_deployment_commits.cicd_deployment_id as id, cicd_deployment_commits.finished_date as finished_date"),
dal.From(&devops.CicdDeploymentCommit{}),
dal.Join("left join project_mapping pm on cicd_deployment_commits.cicd_scope_id = pm.row_id"),
}
where := `cicd_deployment_commits.finished_date < ?
and cicd_deployment_commits.result = ?
and cicd_deployment_commits.environment = ?
and pm.table = ?
and pm.project_name = ?`
args := []interface{}{incident.CreatedDate, devops.RESULT_SUCCESS, devops.PRODUCTION, "cicd_scopes", projectName}
if component != "" {
clauses = append(clauses, dal.Join("left join cicd_scopes cs on cs.id = cicd_deployment_commits.cicd_scope_id"))
where += ` and (lower(cs.name) = ? or lower(cs.id) = ?)`
lowered := strings.ToLower(component)
args = append(args, lowered, lowered)
}
clauses = append(clauses, dal.Where(where, args...), dal.Orderby("finished_date DESC"), dal.Limit(1))
scdc := &simpleCicdDeploymentCommit{}
err := db.All(scdc, clauses...)
if err != nil {
if db.IsErrorNotFound(err) {
return nil, nil
}
return nil, err
}
if scdc.Id == "" {
return nil, nil
}
return scdc, nil
}
Loading