|
| 1 | +/* |
| 2 | +Licensed to the Apache Software Foundation (ASF) under one or more |
| 3 | +contributor license agreements. See the NOTICE file distributed with |
| 4 | +this work for additional information regarding copyright ownership. |
| 5 | +The ASF licenses this file to You under the Apache License, Version 2.0 |
| 6 | +(the "License"); you may not use this file except in compliance with |
| 7 | +the License. You may obtain a copy of the License at |
| 8 | +
|
| 9 | + http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +
|
| 11 | +Unless required by applicable law or agreed to in writing, software |
| 12 | +distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | +See the License for the specific language governing permissions and |
| 15 | +limitations under the License. |
| 16 | +*/ |
| 17 | + |
| 18 | +package e2e |
| 19 | + |
| 20 | +import ( |
| 21 | + "sort" |
| 22 | + "testing" |
| 23 | + "time" |
| 24 | + |
| 25 | + "github.com/apache/incubator-devlake/core/dal" |
| 26 | + "github.com/apache/incubator-devlake/core/models/common" |
| 27 | + "github.com/apache/incubator-devlake/helpers/e2ehelper" |
| 28 | + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" |
| 29 | + "github.com/apache/incubator-devlake/plugins/circleci/impl" |
| 30 | + "github.com/apache/incubator-devlake/plugins/circleci/models" |
| 31 | + "github.com/apache/incubator-devlake/plugins/circleci/tasks" |
| 32 | + "github.com/stretchr/testify/assert" |
| 33 | +) |
| 34 | + |
| 35 | +// TestCircleciWorkflowIncremental verifies that the workflow collector's incremental |
| 36 | +// logic correctly filters pipelines by created_date. This is a regression test for |
| 37 | +// the created_at -> created_date column rename. |
| 38 | +func TestCircleciWorkflowIncremental(t *testing.T) { |
| 39 | + var circleci impl.Circleci |
| 40 | + |
| 41 | + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci) |
| 42 | + taskData := &tasks.CircleciTaskData{ |
| 43 | + Options: &tasks.CircleciOptions{ |
| 44 | + ConnectionId: 1, |
| 45 | + ProjectSlug: "github/coldgust/coldgust.github.io", |
| 46 | + }, |
| 47 | + RegexEnricher: api.NewRegexEnricher(), |
| 48 | + } |
| 49 | + |
| 50 | + // seed pipelines table via extraction |
| 51 | + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_pipelines.csv", "_raw_circleci_api_pipelines") |
| 52 | + dataflowTester.FlushTabler(&models.CircleciPipeline{}) |
| 53 | + dataflowTester.Subtask(tasks.ExtractPipelinesMeta, taskData) |
| 54 | + |
| 55 | + // Part 1: verify the SQL query used by BuildInputIterator works with created_date. |
| 56 | + // Pipelines #4-10 have created_date > 17:45 — assert the exact IDs returned. |
| 57 | + createdAfter := time.Date(2023, 3, 25, 17, 45, 0, 0, time.UTC) |
| 58 | + var pipelines []models.CircleciPipeline |
| 59 | + assert.Nil(t, dataflowTester.Dal.All(&pipelines, |
| 60 | + dal.Where("connection_id = ? AND project_slug = ? AND created_date > ?", |
| 61 | + 1, "github/coldgust/coldgust.github.io", createdAfter), |
| 62 | + )) |
| 63 | + pipelineIds := make([]string, len(pipelines)) |
| 64 | + for i, p := range pipelines { |
| 65 | + pipelineIds[i] = p.Id |
| 66 | + } |
| 67 | + sort.Strings(pipelineIds) |
| 68 | + assert.Equal(t, []string{ |
| 69 | + "23622ee4-e150-4920-9d66-81533fa765a4", // pipeline #5 |
| 70 | + "2c45280f-7fb3-4025-b703-a547c4a94916", // pipeline #4 |
| 71 | + "70f3eb15-3b94-4f80-b65e-f23f4b74c33a", // pipeline #6 |
| 72 | + "7fcc1623-edcc-4a76-ad20-cd81aa83519f", // pipeline #9 |
| 73 | + "866e967d-f826-4470-aed6-fc0c92e98703", // pipeline #7 |
| 74 | + "afe0cabe-e7ee-4eb7-bf13-bb6170d139f0", // pipeline #8 |
| 75 | + "d323f088-02fa-4ed5-9696-fc2f89a27150", // pipeline #10 |
| 76 | + }, pipelineIds) |
| 77 | + |
| 78 | + // Part 2: verify extraction with only the incrementally-collected workflow raw data |
| 79 | + // (workflows for pipelines #4-9 only). |
| 80 | + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_workflows_incremental.csv", "_raw_circleci_api_workflows") |
| 81 | + dataflowTester.FlushTabler(&models.CircleciWorkflow{}) |
| 82 | + dataflowTester.Subtask(tasks.ExtractWorkflowsMeta, taskData) |
| 83 | + dataflowTester.VerifyTableWithOptions( |
| 84 | + models.CircleciWorkflow{}, |
| 85 | + e2ehelper.TableOptions{ |
| 86 | + CSVRelPath: "./snapshot_tables/_tool_circleci_workflows_incremental.csv", |
| 87 | + IgnoreTypes: []interface{}{common.NoPKModel{}}, |
| 88 | + IgnoreFields: []string{"started_at", "stopped_at"}, |
| 89 | + }, |
| 90 | + ) |
| 91 | +} |
| 92 | + |
| 93 | +// TestCircleciJobIncremental verifies that the job collector's incremental logic |
| 94 | +// correctly filters workflows by created_date. Regression test for the column rename. |
| 95 | +func TestCircleciJobIncremental(t *testing.T) { |
| 96 | + var circleci impl.Circleci |
| 97 | + |
| 98 | + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci) |
| 99 | + taskData := &tasks.CircleciTaskData{ |
| 100 | + Options: &tasks.CircleciOptions{ |
| 101 | + ConnectionId: 1, |
| 102 | + ProjectSlug: "github/coldgust/coldgust.github.io", |
| 103 | + }, |
| 104 | + RegexEnricher: api.NewRegexEnricher(), |
| 105 | + Project: &models.CircleciProject{ |
| 106 | + Id: "abcd", |
| 107 | + }, |
| 108 | + } |
| 109 | + |
| 110 | + // seed workflows table via extraction (all 10 workflows including b3b77371 with null created_date) |
| 111 | + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_workflows.csv", "_raw_circleci_api_workflows") |
| 112 | + dataflowTester.FlushTabler(&models.CircleciWorkflow{}) |
| 113 | + dataflowTester.Subtask(tasks.ExtractWorkflowsMeta, taskData) |
| 114 | + |
| 115 | + // Part 1: verify the SQL query used by BuildInputIterator works with created_date. |
| 116 | + // Workflows for pipelines #4-9 have created_date > 17:45 — assert the exact IDs returned. |
| 117 | + // Workflow b3b77371 (null created_date) is excluded by the > comparison. |
| 118 | + createdAfter := time.Date(2023, 3, 25, 17, 45, 0, 0, time.UTC) |
| 119 | + var workflows []models.CircleciWorkflow |
| 120 | + assert.Nil(t, dataflowTester.Dal.All(&workflows, |
| 121 | + dal.Where("connection_id = ? AND project_slug = ? AND created_date > ?", |
| 122 | + 1, "github/coldgust/coldgust.github.io", createdAfter), |
| 123 | + )) |
| 124 | + workflowIds := make([]string, len(workflows)) |
| 125 | + for i, w := range workflows { |
| 126 | + workflowIds[i] = w.Id |
| 127 | + } |
| 128 | + sort.Strings(workflowIds) |
| 129 | + assert.Equal(t, []string{ |
| 130 | + "6731159f-5275-4bfa-ba70-39d343d63814", // pipeline #5 |
| 131 | + "7370985a-9de3-4a47-acbc-e6a1fe8e5812", // pipeline #7 |
| 132 | + "b9ab7bbe-2f30-4c59-b4e2-eb2005bffb14", // pipeline #6 |
| 133 | + "c7df82a6-0d2b-4e19-a36a-3f3aa9fd3943", // pipeline #4 |
| 134 | + "fc76deef-bcdd-4856-8e96-a8e2d1c5a85f", // pipeline #8 |
| 135 | + "fd0bd4f5-264f-4e3c-a151-06153c018f78", // pipeline #9 |
| 136 | + }, workflowIds) |
| 137 | + |
| 138 | + // Part 2: verify extraction with only the incrementally-collected job raw data |
| 139 | + // (jobs for workflows from pipelines #4-9 only). |
| 140 | + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_projects.csv", "_raw_circleci_api_projects") |
| 141 | + dataflowTester.FlushTabler(&models.CircleciProject{}) |
| 142 | + dataflowTester.Subtask(tasks.ExtractProjectsMeta, taskData) |
| 143 | + |
| 144 | + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_circleci_api_jobs_incremental.csv", "_raw_circleci_api_jobs") |
| 145 | + dataflowTester.FlushTabler(&models.CircleciJob{}) |
| 146 | + dataflowTester.Subtask(tasks.ExtractJobsMeta, taskData) |
| 147 | + dataflowTester.VerifyTableWithOptions( |
| 148 | + models.CircleciJob{}, |
| 149 | + e2ehelper.TableOptions{ |
| 150 | + CSVRelPath: "./snapshot_tables/_tool_circleci_jobs_incremental.csv", |
| 151 | + IgnoreTypes: []interface{}{common.NoPKModel{}}, |
| 152 | + }, |
| 153 | + ) |
| 154 | +} |
0 commit comments