Skip to content

Commit 35bb875

Browse files
danielemoraschitamas-la
danielemoraschi
authored and
tamas-la
committed
fix(linker): scope clearHistoryData to current project only (#8814) (#8815)
The clearHistoryData() function used a LEFT JOIN with project_name in the ON clause, causing the subquery to return all PR IDs regardless of project. This effectively wiped the entire pull_request_issues table on every linker run, deleting links from other projects sharing the same repos and links created by the GitHub converter. Fix: - Use INNER JOIN + WHERE for proper project scoping - Add issue-side subquery scoped to current project's boards - Filter by _raw_data_table/_raw_data_remark to only delete linker-created rows Add e2e test for cross-project shared repo scenario.
1 parent 6799e00 commit 35bb875

8 files changed

Lines changed: 81 additions & 4 deletions

‎backend/plugins/linker/e2e/link_pr_and_issue_test.go‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,48 @@ import (
2929
"github.com/apache/incubator-devlake/plugins/linker/tasks"
3030
)
3131

32+
func TestLinkPrToIssueWithSharedRepo(t *testing.T) {
33+
var plugin impl.Linker
34+
dataflowTester := e2ehelper.NewDataFlowTester(t, "linker", plugin)
35+
36+
regexpStr := "#(\\d+)"
37+
re, err := regexp.Compile(regexpStr)
38+
if err != nil {
39+
panic(err)
40+
}
41+
taskData := &tasks.LinkerTaskData{
42+
Options: &tasks.LinkerOptions{
43+
PrToIssueRegexp: regexpStr,
44+
ProjectName: "GitHub1",
45+
},
46+
PrToIssueRegexp: re,
47+
}
48+
49+
// Import input tables for two projects sharing repo R1
50+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/cross_project_issues.csv", &ticket.Issue{})
51+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/cross_project_pull_requests.csv", &code.PullRequest{})
52+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/cross_project_project_mapping.csv", &crossdomain.ProjectMapping{})
53+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/cross_project_board_issues.csv", &ticket.BoardIssue{})
54+
55+
// Pre-populate pull_request_issues with data from GitHub2's linker and from GitHub converter.
56+
// These rows must survive when we run the linker for GitHub1.
57+
dataflowTester.ImportCsvIntoTabler("./snapshot_tables/cross_project_pull_request_issues_before.csv", &crossdomain.PullRequestIssue{})
58+
59+
dataflowTester.Subtask(tasks.LinkPrToIssueMeta, taskData)
60+
61+
// Verify: GitHub1's links are created, GitHub2's linker link and converter link survive
62+
dataflowTester.VerifyTable(
63+
crossdomain.PullRequestIssue{},
64+
"./snapshot_tables/cross_project_pull_request_issues_after.csv",
65+
e2ehelper.ColumnWithRawData(
66+
"pull_request_id",
67+
"pull_request_key",
68+
"issue_id",
69+
"issue_key",
70+
),
71+
)
72+
}
73+
3274
func TestLinkPrToIssue(t *testing.T) {
3375
var plugin impl.Linker
3476
dataflowTester := e2ehelper.NewDataFlowTester(t, "linker", plugin)

‎backend/plugins/linker/e2e/snapshot_tables/cross_project_board_issues.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
"board_id","issue_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark"
2+
"github:GithubRepo:1:384111310","github:GithubIssue:1:1237324696","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",69,""
3+
"github:GithubRepo:1:384111310","github:GithubIssue:1:1237324697","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",69,""
4+
"github:GithubRepo:2:999999999","github:GithubIssue:2:2001","2024-05-14 10:42:37.541","2024-05-28 00:25:41.436","{""ConnectionId"":2,""Name"":""other/repo""}","_raw_github_graphql_issues",70,""

‎backend/plugins/linker/e2e/snapshot_tables/cross_project_issues.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","url","icon_url","issue_key","title","description","epic_key","type","original_type","status","original_status","resolution_date","created_date","updated_date","lead_time_minutes","parent_issue_id","priority","story_point","original_estimate_minutes","time_spent_minutes","time_remaining_minutes","creator_id","creator_name","assignee_id","assignee_name","severity","component","original_project","urgency"
2+
"github:GithubIssue:1:1237324696","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1884","","1884","Issue 1884","desc","","","type/feature-request","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","",""
3+
"github:GithubIssue:1:1237324697","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_graphql_issues",59,"","https://github.com/apache/incubator-devlake/issues/1885","","1885","Issue 1885","desc","","","type/feature-request","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","",""
4+
"github:GithubIssue:2:2001","2024-05-14 10:42:37.529","2024-05-15 12:07:36.450","{""ConnectionId"":2,""Name"":""other/repo""}","_raw_github_graphql_issues",60,"","https://github.com/other/repo/issues/2001","","2001","Issue 2001","desc","","","type/bug","TODO","OPEN","2032-05-16 15:23:21.000","2022-05-16 15:23:21.000","2024-05-11 00:17:21.000",10,"","",11,1,12,11,"github:GithubAccount:1:14050754","Startrekzky","","","","","",""

‎backend/plugins/linker/e2e/snapshot_tables/cross_project_project_mapping.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
"project_name","table","row_id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark"
2+
"GitHub1","repos","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,""
3+
"GitHub1","boards","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub1","",0,""
4+
"GitHub2","repos","github:GithubRepo:1:384111310","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub2","",0,""
5+
"GitHub2","boards","github:GithubRepo:2:999999999","2024-05-15 12:02:13.590","2024-05-15 12:02:13.590","GitHub2","",0,""

‎backend/plugins/linker/e2e/snapshot_tables/cross_project_pull_request_issues_after.csv‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pull_request_id,issue_id,pull_request_key,issue_key,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
2+
github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324696,7317,1884,,,0,"pull_requests,"
3+
github:GithubPullRequest:1:1819250573,github:GithubIssue:1:1237324697,7317,1885,,,0,"pull_requests,"
4+
github:GithubPullRequest:1:1819250574,github:GithubIssue:2:2001,7318,2001,,,0,"pull_requests,"
5+
github:GithubPullRequest:1:1819250574,github:GithubIssue:1:1237324696,7318,1884,"{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}",_raw_github_api_pull_requests,191,""

‎backend/plugins/linker/e2e/snapshot_tables/cross_project_pull_request_issues_before.csv‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
pull_request_id,issue_id,pull_request_key,issue_key,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
2+
github:GithubPullRequest:1:1819250574,github:GithubIssue:1:1237324697,7318,1885,,,0,"pull_requests,"
3+
github:GithubPullRequest:1:1819250574,github:GithubIssue:2:2001,7318,2001,,,0,"pull_requests,"
4+
github:GithubPullRequest:1:1819250574,github:GithubIssue:1:1237324696,7318,1884,"{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}",_raw_github_api_pull_requests,191,""

‎backend/plugins/linker/e2e/snapshot_tables/cross_project_pull_requests.csv‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
"id","created_at","updated_at","_raw_data_params","_raw_data_table","_raw_data_id","_raw_data_remark","base_repo_id","base_ref","base_commit_sha","head_repo_id","head_ref","head_commit_sha","merge_commit_sha","status","original_status","type","component","title","description","url","author_name","author_id","parent_pr_id","pull_request_key","created_date","merged_date","closed_date"
2+
"github:GithubPullRequest:1:1819250573","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",191,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#7275","14fb6488f2208e6a65374a86efce12dd460987e0","91dbce48759da14a4a030124c3ef751f1c5d8389","CLOSED","closed","","","fix: can't GET projects which have / in their name #1884 #1885","desc","https://github.com/apache/incubator-devlake/pull/7317","abeizn","github:GithubAccount:1:101256042","",7317,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000"
3+
"github:GithubPullRequest:1:1819250574","2024-05-15 12:07:36.778","2024-05-15 12:07:36.778","{""ConnectionId"":1,""Name"":""apache/incubator-devlake""}","_raw_github_api_pull_requests",192,"","github:GithubRepo:1:384111310","main","64c52748f3529784cb6c8a372691aa0f638fa73d","github:GithubRepo:1:384111310","fix#2001","14fb6488f2208e6a65374a86efce12dd460987e0","","MERGED","merged","","","fix: something related to #2001","desc","https://github.com/apache/incubator-devlake/pull/7318","abeizn","github:GithubAccount:1:101256042","",7318,"2024-04-12 05:31:43.000","2024-04-13 05:31:43.000","2024-04-12 06:44:27.000"

‎backend/plugins/linker/tasks/link_pr_and_issue.go‎

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,23 @@ func clearHistoryData(db dal.Dal, data *LinkerTaskData) errors.Error {
5252
WHERE pull_request_id IN (
5353
SELECT pr.id
5454
FROM pull_requests pr
55-
LEFT JOIN project_mapping pm
55+
INNER JOIN project_mapping pm
5656
ON pm.table = 'repos'
5757
AND pm.row_id = pr.base_repo_id
58-
AND pm.project_name = ?
59-
)
58+
WHERE pm.project_name = ?
59+
)
60+
AND issue_id IN (
61+
SELECT bi.issue_id
62+
FROM board_issues bi
63+
INNER JOIN project_mapping pm2
64+
ON pm2.table = 'boards'
65+
AND pm2.row_id = bi.board_id
66+
WHERE pm2.project_name = ?
67+
)
68+
AND (_raw_data_table = '' OR _raw_data_table IS NULL)
69+
AND _raw_data_remark LIKE '%pull_requests,%'
6070
`
61-
return db.Exec(sql, data.Options.ProjectName)
71+
return db.Exec(sql, data.Options.ProjectName, data.Options.ProjectName)
6272
}
6373

6474
func LinkPrToIssue(taskCtx plugin.SubTaskContext) errors.Error {

0 commit comments

Comments
 (0)