diff --git a/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go b/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go index 07d4b7627a3..9ac7669178e 100644 --- a/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go +++ b/backend/plugins/azuredevops_go/tasks/ci_cd_timeline_records_collector.go @@ -49,7 +49,7 @@ func CollectRecords(taskCtx plugin.SubTaskContext) errors.Error { cursor, err := db.Cursor( dal.Select("azuredevops_id"), dal.From(models.AzuredevopsBuild{}.TableName()), - dal.Where("repository_id = ? and connection_id=? and result != ?", data.Options.RepositoryId, data.Options.ConnectionId, "failed"), + dal.Where("repository_id = ? and connection_id=? and result NOT IN ?", data.Options.RepositoryId, data.Options.ConnectionId, []string{"failed", "none"}), ) if err != nil { return err @@ -68,7 +68,7 @@ func CollectRecords(taskCtx plugin.SubTaskContext) errors.Error { UrlTemplate: "{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/build/builds/{{ .Input.AzuredevopsId }}/Timeline?api-version=7.1", Query: BuildPaginator(true), ResponseParser: ParseRawMessageFromRecords, - AfterResponse: ignoreDeletedBuilds, // Ignore the 404 response if builds are deleted during the collection + AfterResponse: ignoreInvalidTimelineResponse, // Skip builds with missing/malformed timelines (e.g. YAML syntax errors) }) if err != nil { return err diff --git a/backend/plugins/azuredevops_go/tasks/shared.go b/backend/plugins/azuredevops_go/tasks/shared.go index 3ceae7c9853..e6fe0eca1e2 100644 --- a/backend/plugins/azuredevops_go/tasks/shared.go +++ b/backend/plugins/azuredevops_go/tasks/shared.go @@ -18,14 +18,17 @@ limitations under the License. package tasks import ( + "bytes" "encoding/json" "fmt" + "io" + "net/http" + "net/url" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "net/http" - "net/url" ) // Build and TimeLine Record State and Result types can be found here: @@ -145,9 +148,29 @@ func change203To401(res *http.Response) errors.Error { return nil } -func ignoreDeletedBuilds(res *http.Response) errors.Error { +// ignoreInvalidTimelineResponse is an AfterResponse handler for the Timeline API. +// It skips builds whose timeline response is missing or unparseable (e.g. builds +// that failed due to a YAML syntax error never produce a usable timeline), instead +// of aborting the entire subtask. +func ignoreInvalidTimelineResponse(res *http.Response) errors.Error { + // Keep existing behaviour: treat 404 as a graceful skip (build was deleted). if res.StatusCode == http.StatusNotFound { return api.ErrIgnoreAndContinue } + + // Read the body so we can inspect it, then restore it for the ResponseParser. + body, err := io.ReadAll(res.Body) + _ = res.Body.Close() + if err != nil { + return api.ErrIgnoreAndContinue + } + res.Body = io.NopCloser(bytes.NewReader(body)) + + // An empty body or non-JSON body means the timeline is not available. + // Return ErrIgnoreAndContinue so the build is skipped without failing the subtask. + if len(body) == 0 || !json.Valid(body) { + return api.ErrIgnoreAndContinue + } + return nil } diff --git a/backend/plugins/azuredevops_go/tasks/shared_test.go b/backend/plugins/azuredevops_go/tasks/shared_test.go new file mode 100644 index 00000000000..353ae3f4c91 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/shared_test.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "bytes" + "io" + "net/http" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func makeResponse(statusCode int, body string) *http.Response { + return &http.Response{ + StatusCode: statusCode, + Body: io.NopCloser(bytes.NewBufferString(body)), + Request: &http.Request{}, + } +} + +func TestIgnoreInvalidTimelineResponse_404(t *testing.T) { + res := makeResponse(http.StatusNotFound, "") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "404 should return ErrIgnoreAndContinue") +} + +func TestIgnoreInvalidTimelineResponse_EmptyBody(t *testing.T) { + res := makeResponse(http.StatusOK, "") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "empty body should return ErrIgnoreAndContinue") +} + +func TestIgnoreInvalidTimelineResponse_NonJSONBody(t *testing.T) { + res := makeResponse(http.StatusOK, "not json at all") + err := ignoreInvalidTimelineResponse(res) + assert.Equal(t, api.ErrIgnoreAndContinue, err, "non-JSON body should return ErrIgnoreAndContinue") +} + +func TestIgnoreInvalidTimelineResponse_ValidJSON_ReturnsNil(t *testing.T) { + validJSON := `{"records":[{"id":"abc","type":"Job","name":"Build"}]}` + res := makeResponse(http.StatusOK, validJSON) + err := ignoreInvalidTimelineResponse(res) + assert.Nil(t, err, "valid JSON body should return nil") + + // Verify body is still readable after the handler restored it. + remaining, readErr := io.ReadAll(res.Body) + assert.NoError(t, readErr) + assert.Equal(t, validJSON, string(remaining), "body should be restored for downstream parser") +} + +func TestIgnoreInvalidTimelineResponse_ValidEmptyJSONObject(t *testing.T) { + res := makeResponse(http.StatusOK, "{}") + err := ignoreInvalidTimelineResponse(res) + assert.Nil(t, err, "valid empty JSON object should return nil") +}