diff --git a/api/stovepipe/proto/stovepipe.proto b/api/stovepipe/proto/stovepipe.proto index 4bd02acad..8de6c2c90 100644 --- a/api/stovepipe/proto/stovepipe.proto +++ b/api/stovepipe/proto/stovepipe.proto @@ -116,7 +116,7 @@ message GetProjectStatusByIDRequest { string queue = 1; // Canonical decimal request identifier returned from Ingest. string request_id = 2; - // Required consumer-defined project IDs whose recorded results are requested. + // Optional consumer-defined project IDs. Empty returns all recorded project results. repeated string projects = 3; // Maximum requested projects to return in one page. Zero selects the server default. int32 page_size = 4; @@ -130,7 +130,7 @@ message GetProjectStatusByURIRequest { string queue = 1; // Exact VCS-agnostic URI of the commit under validation. string change_uri = 2; - // Required consumer-defined project IDs whose recorded results are requested. + // Optional consumer-defined project IDs. Empty returns all recorded project results. repeated string projects = 3; // Maximum requested projects to return in one page. Zero selects the server default. int32 page_size = 4; diff --git a/api/stovepipe/protopb/stovepipe.pb.go b/api/stovepipe/protopb/stovepipe.pb.go index a4777db93..ff308230d 100644 --- a/api/stovepipe/protopb/stovepipe.pb.go +++ b/api/stovepipe/protopb/stovepipe.pb.go @@ -696,7 +696,7 @@ type GetProjectStatusByIDRequest struct { Queue string `protobuf:"bytes,1,opt,name=queue,proto3" json:"queue,omitempty"` // Canonical decimal request identifier returned from Ingest. RequestId string `protobuf:"bytes,2,opt,name=request_id,json=requestId,proto3" json:"request_id,omitempty"` - // Required consumer-defined project IDs whose recorded results are requested. + // Optional consumer-defined project IDs. Empty returns all recorded project results. Projects []string `protobuf:"bytes,3,rep,name=projects,proto3" json:"projects,omitempty"` // Maximum requested projects to return in one page. Zero selects the server default. PageSize int32 `protobuf:"varint,4,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` @@ -778,7 +778,7 @@ type GetProjectStatusByURIRequest struct { Queue string `protobuf:"bytes,1,opt,name=queue,proto3" json:"queue,omitempty"` // Exact VCS-agnostic URI of the commit under validation. ChangeUri string `protobuf:"bytes,2,opt,name=change_uri,json=changeUri,proto3" json:"change_uri,omitempty"` - // Required consumer-defined project IDs whose recorded results are requested. + // Optional consumer-defined project IDs. Empty returns all recorded project results. Projects []string `protobuf:"bytes,3,rep,name=projects,proto3" json:"projects,omitempty"` // Maximum requested projects to return in one page. Zero selects the server default. PageSize int32 `protobuf:"varint,4,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` diff --git a/doc/rfc/stovepipe/project-status-api.md b/doc/rfc/stovepipe/project-status-api.md index 514a97866..a92332496 100644 --- a/doc/rfc/stovepipe/project-status-api.md +++ b/doc/rfc/stovepipe/project-status-api.md @@ -17,7 +17,7 @@ The request and response fields have the following API-level meaning. The publis | `queue` | Yes | Identifies the queue in which to find the validation request. | | `request_id` | ID lookup | Canonical positive decimal request ID returned by `Ingest`. | | `change_uri` | URI lookup | Identifies the exact commit URI under validation. | -| `projects` | Yes | Consumer-defined project IDs whose results are requested. At least one ID is required. | +| `projects` | No | Consumer-defined project IDs whose results are requested. Omitted or empty returns all recorded project results. | | `page_size` | No | Limits one page of results for the requested projects. | | `page_token` | No | Continues a previous page of results for the requested projects. | @@ -29,7 +29,7 @@ The request and response fields have the following API-level meaning. The publis | `updated_at_ms` | Records, in Unix milliseconds, the newest durable lifecycle or result record represented by the response. | | `repository_breakage_degree` | Returns the whole-repository result when it is durably recorded. | | `project_results_complete` | Indicates whether the implementation has durably finished producing results for the supplied project IDs. | -| `projects` | Returns recorded results for requested project IDs. | +| `projects` | Returns recorded results for requested project IDs, or all recorded project results when none are supplied. | | `next_page_token` | Continues requested-project result pagination when another page exists. | `request_state` is a stable public lifecycle vocabulary: `accepted`, `processing`, `succeeded`, `failed`, `cancelled`, or `superseded`. Clients must tolerate a future value. A terminal request state does not by itself mean that project results are complete. @@ -56,13 +56,15 @@ Both lookups are queue-scoped and use the same durable request summary and valid The request-URI mapping admits one request per `(queue, change_uri)`, so the endpoint returns one authoritative request. Revalidation support must introduce an explicit authoritative-request rule; it must not silently change this lookup's meaning. -Queue, URI, request ID, and project ID comparisons are byte-exact and are limited to 255 bytes. An empty queue, lookup selector, project list, or project ID is invalid. A request cannot contain duplicate project IDs. `page_size=0` selects the default of 50; the maximum is 200. Request IDs use the same canonical positive decimal format as `Ingest` and request-history lookups. Page tokens are opaque and bound to the selected request and requested project list; either endpoint can continue a page for the same queue, request, and projects. +Queue, URI, request ID, and project ID comparisons are byte-exact and are limited to 255 bytes. An empty queue, lookup selector, or individual project ID is invalid. A request cannot contain duplicate project IDs. `page_size=0` selects the default of 50; the maximum is 200. Request IDs use the same canonical positive decimal format as `Ingest` and request-history lookups. Page tokens are opaque and bound to the selected request and requested project list; either endpoint can continue a page for the same queue, request, and projects. ## Project Results and Pagination -The caller supplies the project IDs for which it wants recorded results. The API does not define project discovery, validation selection, or attribution. A project absent from the response has no universal meaning and consumers must not infer that it is green. +The caller may supply the project IDs for which it wants recorded results. The API does not define project discovery, validation selection, or attribution. A project absent from the response has no universal meaning and consumers must not infer that it is green. -The response includes only results for the supplied project IDs, in the supplied order, and paginates that list. An implementation may record results for every requested project, only requested projects with a non-green result, or another documented subset. +When project IDs are supplied, the response includes only results for those IDs, in the supplied order, and paginates that list. An implementation may record results for every requested project, only requested projects with a non-green result, or another documented subset. + +With no project IDs supplied, the response lists all recorded project results in project-ID order, excluding the repository fact. Results are a live view; restart pagination to discover facts recorded behind the cursor. Each project result is an immutable validation fact keyed by queue, commit URI, and project ID. A result must identify the selected request. `project_results_complete` is false until the implementation has durably finished producing results for the supplied project IDs; a completed empty set is valid. @@ -70,11 +72,11 @@ Callers use `project_results_complete`, rather than the presence or absence of a ## Implementation -Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and any recorded facts for requested projects in paginated request order. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page. +Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and recorded facts for requested projects in paginated request order, or all recorded project results in project-ID order when none are requested. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page. ## Errors and Authorization -- Invalid selectors, an empty project list, an invalid project ID, duplicate project IDs, or malformed page tokens are user errors. +- Invalid selectors, an invalid project ID, duplicate project IDs, or malformed page tokens are user errors. - An unknown queue-scoped request ID or commit URI is not found. - A request-URI mapping whose summary is not visible is retryable. - A selector and summary that disagree, a result belonging to another request, or a completion record with a missing project result is an internal consistency error. diff --git a/stovepipe/controller/project_status.go b/stovepipe/controller/project_status.go index 4eb15937a..5f9c63abe 100644 --- a/stovepipe/controller/project_status.go +++ b/stovepipe/controller/project_status.go @@ -23,7 +23,7 @@ import ( "errors" "fmt" "math" - "reflect" + "slices" "github.com/uber-go/tally" "github.com/uber/submitqueue/platform/errs" @@ -39,9 +39,11 @@ const ( ) type projectStatusPageToken struct { - RequestID string `json:"request_id"` - Projects []string `json:"projects"` - NextIndex int `json:"next_index"` + Queue string `json:"queue"` + RequestID string `json:"request_id"` + Projects []string `json:"projects"` + NextIndex int `json:"next_index"` + AfterProject string `json:"after_project,omitempty"` } // ProjectStatusNotFoundError indicates that no validation request matches a lookup selector. @@ -102,7 +104,7 @@ func NewProjectStatusController(logger *zap.SugaredLogger, scope tally.Scope, st } } -// GetProjectStatus returns the selected request summary and its repository validation fact, if recorded. +// GetProjectStatus returns the selected request summary and recorded validation facts. func (c *projectStatusController) GetProjectStatus(ctx context.Context, req entity.GetProjectStatusRequest) (result entity.GetProjectStatusResult, retErr error) { op := metrics.Begin(c.metricsScope, "get_project_status", metrics.StorageLatencyBuckets, metrics.TagsFromContext(ctx)...) defer func() { op.Complete(retErr) }() @@ -158,26 +160,19 @@ func (c *projectStatusController) GetProjectStatus(ctx context.Context, req enti result.UpdatedAtMs = fact.CreatedAt } } - page, err := selectProjectStatusPage(req, requestSummary.RequestID) + + page, err := loadProjectStatusPage(ctx, req, requestSummary, factStore) if err != nil { return entity.GetProjectStatusResult{}, err } - for _, project := range page.projects { - projectFact, err := factStore.Get(ctx, requestSummary.URI, project) - if err != nil { - if storage.IsNotFound(err) { - continue - } - return entity.GetProjectStatusResult{}, fmt.Errorf("GetProjectStatus failed to load project fact for request %q and project %q: %w", requestSummary.RequestID, project, err) - } - if err := validateProjectFact(projectFact, requestSummary, project); err != nil { - return entity.GetProjectStatusResult{}, err - } - result.ProjectValidationFacts = append(result.ProjectValidationFacts, projectFact) + + result.ProjectValidationFacts = page.facts + for _, projectFact := range page.facts { if projectFact.CreatedAt > result.UpdatedAtMs { result.UpdatedAtMs = projectFact.CreatedAt } } + result.NextPageToken = page.nextToken c.logger.Debugw("project status retrieved", "request_id", requestSummary.RequestID, "queue", requestSummary.Queue, "change_uri", requestSummary.URI, "has_repository_result", result.HasRepositoryValidationFact, "project_results", len(result.ProjectValidationFacts)) @@ -200,9 +195,6 @@ func validateProjectStatusRequest(req entity.GetProjectStatusRequest) error { return fmt.Errorf("GetProjectStatus invalid change_uri=%q: %w", req.ChangeURI, err) } } - if len(req.Projects) == 0 { - return fmt.Errorf("GetProjectStatus projects must be non-empty: %w", ErrInvalidRequest) - } seen := make(map[string]struct{}, len(req.Projects)) for _, project := range req.Projects { if err := validateHistoryIdentifier("project", project); err != nil { @@ -220,55 +212,153 @@ func validateProjectStatusRequest(req entity.GetProjectStatusRequest) error { } type projectStatusPage struct { - projects []string + facts []entity.ValidationFact nextToken string } -func selectProjectStatusPage(req entity.GetProjectStatusRequest, requestID string) (projectStatusPage, error) { - start := 0 - if req.PageToken != "" { - token, err := decodeProjectStatusPageToken(req.PageToken) - if err != nil { - return projectStatusPage{}, err - } - if token.RequestID != requestID || !reflect.DeepEqual(token.Projects, req.Projects) || token.NextIndex <= 0 || token.NextIndex >= len(req.Projects) { - return projectStatusPage{}, fmt.Errorf("GetProjectStatus page_token does not match the selected request and projects: %w", ErrInvalidRequest) - } - start = token.NextIndex +func loadProjectStatusPage( + ctx context.Context, + req entity.GetProjectStatusRequest, + summary entity.RequestSummary, + store storage.ValidationFactStore, +) (projectStatusPage, error) { + token, err := parseProjectStatusPageToken(req, summary) + if err != nil { + return projectStatusPage{}, err } + pageSize := int(req.PageSize) if pageSize == 0 { pageSize = defaultProjectStatusPageSize } - end := min(start+pageSize, len(req.Projects)) - page := projectStatusPage{projects: req.Projects[start:end]} - if end < len(req.Projects) { - token, err := encodeProjectStatusPageToken(projectStatusPageToken{RequestID: requestID, Projects: req.Projects, NextIndex: end}) + + if len(req.Projects) == 0 { + return listProjectStatusPage(ctx, summary, store, token, pageSize) + } + + return getRequestedProjectStatusPage(ctx, summary, store, token, pageSize) +} + +func getRequestedProjectStatusPage( + ctx context.Context, + summary entity.RequestSummary, + store storage.ValidationFactStore, + token projectStatusPageToken, + pageSize int, +) (projectStatusPage, error) { + end := min(token.NextIndex+pageSize, len(token.Projects)) + var page projectStatusPage + + for _, project := range token.Projects[token.NextIndex:end] { + fact, err := store.Get(ctx, summary.URI, project) if err != nil { - return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to create page token: %w", err) + if storage.IsNotFound(err) { + continue + } + + return projectStatusPage{}, fmt.Errorf( + "GetProjectStatus failed to load project fact for request %q and project %q: %w", + summary.RequestID, project, err, + ) + } + + if err := validateProjectFact(fact, summary, project); err != nil { + return projectStatusPage{}, err + } + + page.facts = append(page.facts, fact) + } + + var err error + if end < len(token.Projects) { + token.NextIndex = end + page.nextToken, err = encodeProjectStatusPageToken(token) + } + + return page, err +} + +func listProjectStatusPage( + ctx context.Context, + summary entity.RequestSummary, + store storage.ValidationFactStore, + token projectStatusPageToken, + pageSize int, +) (projectStatusPage, error) { + facts, err := store.ListProjects(ctx, summary.URI, token.AfterProject, pageSize+1) + if err != nil { + return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to list project facts for request %q: %w", summary.RequestID, err) + } + + for _, fact := range facts { + if err := validateProjectFact(fact, summary, fact.Project); err != nil { + return projectStatusPage{}, err + } + + if fact.Project == "" { + return projectStatusPage{}, &ProjectStatusConsistencyError{message: "project listing contains a repository fact"} } - page.nextToken = token } - return page, nil + + page := projectStatusPage{facts: facts} + if len(facts) > pageSize { + page.facts = facts[:pageSize] + token.AfterProject = page.facts[pageSize-1].Project + page.nextToken, err = encodeProjectStatusPageToken(token) + } + + return page, err } func encodeProjectStatusPageToken(token projectStatusPageToken) (string, error) { contents, err := json.Marshal(token) if err != nil { - return "", err + return "", fmt.Errorf("GetProjectStatus failed to create page token: %w", err) } + return base64.RawURLEncoding.EncodeToString(contents), nil } -func decodeProjectStatusPageToken(encoded string) (projectStatusPageToken, error) { - contents, err := base64.RawURLEncoding.DecodeString(encoded) +func parseProjectStatusPageToken( + req entity.GetProjectStatusRequest, + summary entity.RequestSummary, +) (projectStatusPageToken, error) { + if req.PageToken == "" { + return projectStatusPageToken{ + Queue: summary.Queue, + RequestID: summary.RequestID, + Projects: req.Projects, + }, nil + } + + invalidToken := fmt.Errorf("GetProjectStatus page_token does not match the selected request and projects: %w", ErrInvalidRequest) + + contents, err := base64.RawURLEncoding.DecodeString(req.PageToken) if err != nil { - return projectStatusPageToken{}, fmt.Errorf("GetProjectStatus invalid page_token: %w", ErrInvalidRequest) + return projectStatusPageToken{}, invalidToken } + var token projectStatusPageToken if err := json.Unmarshal(contents, &token); err != nil { - return projectStatusPageToken{}, fmt.Errorf("GetProjectStatus invalid page_token: %w", ErrInvalidRequest) + return projectStatusPageToken{}, invalidToken } + + if token.Queue != summary.Queue || + token.RequestID != summary.RequestID || + !slices.Equal(token.Projects, req.Projects) { + return projectStatusPageToken{}, invalidToken + } + + if len(req.Projects) == 0 { + if token.NextIndex != 0 || validateHistoryIdentifier("project", token.AfterProject) != nil { + return projectStatusPageToken{}, invalidToken + } + } else if token.AfterProject != "" || + token.NextIndex <= 0 || + token.NextIndex >= len(req.Projects) { + return projectStatusPageToken{}, invalidToken + } + return token, nil } diff --git a/stovepipe/controller/project_status_test.go b/stovepipe/controller/project_status_test.go index 06f644fe0..6832c7152 100644 --- a/stovepipe/controller/project_status_test.go +++ b/stovepipe/controller/project_status_test.go @@ -40,6 +40,18 @@ const ( func TestGetProjectStatus(t *testing.T) { requestSummary := entity.RequestSummary{RequestID: projectStatusID, Queue: projectStatusQueue, URI: projectStatusURI, State: entity.RequestStateProcessing, RequestVersion: 1, StateTimestampMs: 1} + projectFact := entity.ValidationFact{ + URI: projectStatusURI, + RequestID: projectStatusID, + Project: "project-a", + Degree: 0.5, + CreatedAt: 20, + } + greenFact := projectFact + greenFact.Project = "project-green" + greenFact.Degree = entity.DegreeGreen + readErr := errors.New("project listing failed") + tests := []struct { name string request entity.GetProjectStatusRequest @@ -47,6 +59,9 @@ func TestGetProjectStatus(t *testing.T) { summaryErr error fact entity.ValidationFact factErr error + projectFacts []entity.ValidationFact + wantProjects []entity.ValidationFact + wantErr error wantFact bool wantNotFound bool wantRetryable bool @@ -59,7 +74,28 @@ func TestGetProjectStatus(t *testing.T) { {name: "missing uri", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-a"}}, uriErr: storage.ErrNotFound, wantNotFound: true}, {name: "request summary visibility gap", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-a"}}, summaryErr: storage.ErrNotFound, wantRetryable: true}, {name: "empty queue", request: entity.GetProjectStatusRequest{ChangeURI: projectStatusURI}, wantInvalid: true}, - {name: "empty projects", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI}, wantInvalid: true}, + { + name: "omitted projects returns stored results", + request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI}, + factErr: storage.ErrNotFound, projectFacts: []entity.ValidationFact{projectFact, greenFact}, + wantProjects: []entity.ValidationFact{projectFact, greenFact}, + }, + { + name: "empty projects returns stored results", + request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{}}, + factErr: storage.ErrNotFound, projectFacts: []entity.ValidationFact{projectFact}, + wantProjects: []entity.ValidationFact{projectFact}, + }, + { + name: "no recorded projects", + request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI}, + factErr: storage.ErrNotFound, + }, + { + name: "project listing failure", + request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI}, + factErr: storage.ErrNotFound, wantErr: readErr, + }, {name: "empty project", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{""}}, wantInvalid: true}, {name: "duplicate projects", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, Projects: []string{"project-a", "project-a"}}, wantInvalid: true}, {name: "invalid page size", request: entity.GetProjectStatusRequest{Queue: projectStatusQueue, ChangeURI: projectStatusURI, PageSize: maxProjectStatusPageSize + 1}, wantInvalid: true}, @@ -86,6 +122,9 @@ func TestGetProjectStatus(t *testing.T) { store.EXPECT().GetValidationFactStore().Return(factStore) factStore.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(tt.fact, tt.factErr) if !tt.noProjectLookup && !(tt.factErr == nil && tt.wantConsistency) { + if len(tt.request.Projects) == 0 { + factStore.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "", defaultProjectStatusPageSize+1).Return(tt.projectFacts, tt.wantErr) + } for _, project := range tt.request.Projects { factStore.EXPECT().Get(gomock.Any(), projectStatusURI, project).Return(entity.ValidationFact{}, storage.ErrNotFound) } @@ -97,6 +136,11 @@ func TestGetProjectStatus(t *testing.T) { controller := NewProjectStatusController(zap.NewNop().Sugar(), tally.NoopScope, factory) got, err := controller.GetProjectStatus(context.Background(), tt.request) + if tt.wantErr != nil { + require.ErrorIs(t, err, tt.wantErr) + return + } + if tt.wantInvalid { assert.True(t, IsInvalidRequest(err)) } @@ -111,6 +155,7 @@ func TestGetProjectStatus(t *testing.T) { assert.Equal(t, requestSummary, got.RequestSummary) assert.Equal(t, tt.wantFact, got.HasRepositoryValidationFact) assert.False(t, got.ProjectResultsComplete) + assert.Equal(t, tt.wantProjects, got.ProjectValidationFacts) if tt.wantFact { assert.Equal(t, tt.fact, got.RepositoryValidationFact) } @@ -200,7 +245,6 @@ func TestGetProjectStatusRejectsInvalidIDSelectors(t *testing.T) { {name: "zero ID", modify: func(req *entity.GetProjectStatusRequest) { req.RequestID = "0" }}, {name: "negative ID", modify: func(req *entity.GetProjectStatusRequest) { req.RequestID = "-7" }}, {name: "overflow ID", modify: func(req *entity.GetProjectStatusRequest) { req.RequestID = "9223372036854775808" }}, - {name: "empty projects", modify: func(req *entity.GetProjectStatusRequest) { req.Projects = nil }}, {name: "empty project", modify: func(req *entity.GetProjectStatusRequest) { req.Projects = []string{""} }}, {name: "duplicate project", modify: func(req *entity.GetProjectStatusRequest) { req.Projects = []string{"p", "p"} }}, {name: "negative page size", modify: func(req *entity.GetProjectStatusRequest) { req.PageSize = -1 }}, @@ -316,12 +360,135 @@ func TestGetProjectStatusPaginatesProjectFacts(t *testing.T) { assert.Empty(t, got.NextPageToken) } -func TestSelectProjectStatusPageRejectsMismatchedToken(t *testing.T) { - token, err := encodeProjectStatusPageToken(projectStatusPageToken{RequestID: projectStatusID, Projects: []string{"project-a", "project-b"}, NextIndex: 1}) +func TestParseProjectStatusPageTokenRejectsMismatches(t *testing.T) { + modes := []struct { + name string + projects []string + }{ + {name: "listed projects"}, + { + name: "requested projects", + projects: []string{"a", "b"}, + }, + } + + tests := []struct { + name string + modify func(*projectStatusPageToken) + }{ + { + name: "other request", + modify: func(token *projectStatusPageToken) { token.RequestID = "8" }, + }, + { + name: "other queue", + modify: func(token *projectStatusPageToken) { token.Queue = "other/main" }, + }, + { + name: "missing queue", + modify: func(token *projectStatusPageToken) { token.Queue = "" }, + }, + { + name: "different selection", + modify: func(token *projectStatusPageToken) { token.Projects = []string{"b", "a"} }, + }, + { + name: "mixed cursors", + modify: func(token *projectStatusPageToken) { + token.AfterProject = "a" + token.NextIndex = 1 + }, + }, + { + name: "missing cursor", + modify: func(token *projectStatusPageToken) { + token.AfterProject = "" + token.NextIndex = 0 + }, + }, + { + name: "invalid cursor", + modify: func(token *projectStatusPageToken) { + if len(token.Projects) == 0 { + token.AfterProject = strings.Repeat("p", 256) + } else { + token.NextIndex = len(token.Projects) + } + }, + }, + { + name: "negative index", + modify: func(token *projectStatusPageToken) { token.NextIndex = -1 }, + }, + } + + for _, mode := range modes { + t.Run(mode.name, func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + token := projectStatusPageToken{Queue: projectStatusQueue, RequestID: projectStatusID, Projects: mode.projects} + if len(mode.projects) == 0 { + token.AfterProject = "a" + } else { + token.NextIndex = 1 + } + + tt.modify(&token) + encoded, err := encodeProjectStatusPageToken(token) + require.NoError(t, err) + + _, err = parseProjectStatusPageToken( + entity.GetProjectStatusRequest{Projects: mode.projects, PageToken: encoded}, + entity.RequestSummary{Queue: projectStatusQueue, RequestID: projectStatusID}, + ) + require.Error(t, err) + assert.True(t, IsInvalidRequest(err)) + }) + } + }) + } +} + +func TestGetProjectStatusPaginatesStoredProjects(t *testing.T) { + summary := entity.RequestSummary{RequestID: projectStatusID, Queue: projectStatusQueue, URI: projectStatusURI, StateTimestampMs: 10} + stored := []entity.ValidationFact{ + {URI: projectStatusURI, RequestID: projectStatusID, Project: "a", Degree: 0.5, CreatedAt: 20}, + {URI: projectStatusURI, RequestID: projectStatusID, Project: "b", Degree: 0, CreatedAt: 50}, + {URI: projectStatusURI, RequestID: projectStatusID, Project: "c", Degree: 1, CreatedAt: 60}, + } + + ctrl := gomock.NewController(t) + factory := storagemock.NewMockFactory(ctrl) + store := storagemock.NewMockStorage(ctrl) + summaries := storagemock.NewMockRequestSummaryStore(ctrl) + facts := storagemock.NewMockValidationFactStore(ctrl) + + factory.EXPECT().For(storage.Config{QueueName: projectStatusQueue}).Return(store, nil).Times(2) + store.EXPECT().GetRequestSummaryStore().Return(summaries).Times(2) + summaries.EXPECT().Get(gomock.Any(), projectStatusID).Return(summary, nil).Times(2) + store.EXPECT().GetValidationFactStore().Return(facts).Times(2) + facts.EXPECT().Get(gomock.Any(), projectStatusURI, "").Return(entity.ValidationFact{}, storage.ErrNotFound).Times(2) + facts.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "", 3).Return(stored, nil) + facts.EXPECT().ListProjects(gomock.Any(), projectStatusURI, "b", defaultProjectStatusPageSize+1).Return(stored[2:], nil) + + controller := NewProjectStatusController(zap.NewNop().Sugar(), tally.NoopScope, factory) + req := entity.GetProjectStatusRequest{Queue: projectStatusQueue, RequestID: projectStatusID, PageSize: 2} + + first, err := controller.GetProjectStatus(context.Background(), req) require.NoError(t, err) - _, err = selectProjectStatusPage(entity.GetProjectStatusRequest{Projects: []string{"project-b", "project-a"}, PageToken: token}, projectStatusID) - require.Error(t, err) - assert.True(t, IsInvalidRequest(err)) + assert.Equal(t, stored[:2], first.ProjectValidationFacts) + assert.Equal(t, int64(50), first.UpdatedAtMs) + require.NotEmpty(t, first.NextPageToken) + + req.PageToken = first.NextPageToken + req.Projects = []string{} + req.PageSize = 0 + + second, err := controller.GetProjectStatus(context.Background(), req) + require.NoError(t, err) + assert.Equal(t, stored[2:], second.ProjectValidationFacts) + assert.Equal(t, int64(60), second.UpdatedAtMs) + assert.Empty(t, second.NextPageToken) } func TestValidateProjectFact(t *testing.T) { diff --git a/stovepipe/entity/project_status.go b/stovepipe/entity/project_status.go index 5d1c7f261..3cacf53b6 100644 --- a/stovepipe/entity/project_status.go +++ b/stovepipe/entity/project_status.go @@ -22,9 +22,9 @@ type GetProjectStatusRequest struct { RequestID string // ChangeURI identifies the exact commit under validation, empty when selecting by request ID. ChangeURI string - // Projects limits results to the supplied consumer-defined project IDs. + // Projects limits results to the supplied consumer-defined project IDs; empty selects all recorded projects. Projects []string - // PageSize is the requested maximum number of requested projects to inspect. + // PageSize is the maximum number of project results per page. PageSize int32 // PageToken is an opaque continuation token for full project-result pagination. PageToken string @@ -40,7 +40,7 @@ type GetProjectStatusResult struct { HasRepositoryValidationFact bool // ProjectResultsComplete reports whether the implementation has finished its project-result set. ProjectResultsComplete bool - // ProjectValidationFacts contains recorded results for requested projects in request order. + // ProjectValidationFacts contains recorded results in request order, or project-ID order when Projects is empty. ProjectValidationFacts []ValidationFact // NextPageToken continues requested-project result pagination when another page exists. NextPageToken string diff --git a/stovepipe/extension/storage/mock/validation_fact_store_mock.go b/stovepipe/extension/storage/mock/validation_fact_store_mock.go index 7fe036cdc..0c232a65f 100644 --- a/stovepipe/extension/storage/mock/validation_fact_store_mock.go +++ b/stovepipe/extension/storage/mock/validation_fact_store_mock.go @@ -69,3 +69,18 @@ func (mr *MockValidationFactStoreMockRecorder) Get(ctx, uri, project any) *gomoc mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockValidationFactStore)(nil).Get), ctx, uri, project) } + +// ListProjects mocks base method. +func (m *MockValidationFactStore) ListProjects(ctx context.Context, uri, afterProject string, limit int) ([]entity.ValidationFact, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListProjects", ctx, uri, afterProject, limit) + ret0, _ := ret[0].([]entity.ValidationFact) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListProjects indicates an expected call of ListProjects. +func (mr *MockValidationFactStoreMockRecorder) ListProjects(ctx, uri, afterProject, limit any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListProjects", reflect.TypeOf((*MockValidationFactStore)(nil).ListProjects), ctx, uri, afterProject, limit) +} diff --git a/stovepipe/extension/storage/mysql/validation_fact_store.go b/stovepipe/extension/storage/mysql/validation_fact_store.go index 85c3fe586..6c58a5593 100644 --- a/stovepipe/extension/storage/mysql/validation_fact_store.go +++ b/stovepipe/extension/storage/mysql/validation_fact_store.go @@ -93,3 +93,34 @@ func (v *validationFactStore) Get(ctx context.Context, uri, project string) (ret return fact, nil } + +func (v *validationFactStore) ListProjects(ctx context.Context, uri, afterProject string, limit int) (ret []entity.ValidationFact, retErr error) { + op := metrics.Begin(v.scope, "list_projects", metrics.StorageLatencyBuckets) + defer func() { op.Complete(retErr) }() + + if limit <= 0 { + return nil, fmt.Errorf("validation fact list limit must be positive") + } + rows, err := v.db.QueryContext(ctx, ` + SELECT uri, project, degree, request_id, created_at FROM validation_fact + WHERE queue = ? AND uri = ? AND project > ? + ORDER BY project ASC LIMIT ?`, + v.queue, uri, afterProject, limit, + ) + if err != nil { + return nil, fmt.Errorf("failed to list project facts queue=%s uri=%s: %w", v.queue, uri, err) + } + defer rows.Close() + + for rows.Next() { + var fact entity.ValidationFact + if err := rows.Scan(&fact.URI, &fact.Project, &fact.Degree, &fact.RequestID, &fact.CreatedAt); err != nil { + return nil, fmt.Errorf("failed to scan project fact: %w", err) + } + ret = append(ret, fact) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("failed to read project facts: %w", err) + } + return ret, nil +} diff --git a/stovepipe/extension/storage/mysql/validation_fact_store_test.go b/stovepipe/extension/storage/mysql/validation_fact_store_test.go index e5628f87d..cfb4c36e6 100644 --- a/stovepipe/extension/storage/mysql/validation_fact_store_test.go +++ b/stovepipe/extension/storage/mysql/validation_fact_store_test.go @@ -179,3 +179,56 @@ func TestValidationFactStore_Get(t *testing.T) { }) } } + +func TestValidationFactStore_ListProjects(t *testing.T) { + readErr := fmt.Errorf("connection reset") + columns := []string{"uri", "project", "degree", "request_id", "created_at"} + want := entity.ValidationFact{URI: testFactURI, Project: "project-b", Degree: 0.5, RequestID: "2", CreatedAt: 20} + tests := []struct { + name string + rows *sqlmock.Rows + queryErr error + wantErr bool + want []entity.ValidationFact + }{ + {name: "found", rows: sqlmock.NewRows(columns).AddRow(want.URI, want.Project, want.Degree, want.RequestID, want.CreatedAt), want: []entity.ValidationFact{want}}, + {name: "empty", rows: sqlmock.NewRows(columns)}, + {name: "query failure", queryErr: readErr, wantErr: true}, + {name: "scan failure", rows: sqlmock.NewRows(columns).AddRow(want.URI, want.Project, "invalid", want.RequestID, want.CreatedAt), wantErr: true}, + {name: "iteration failure", rows: sqlmock.NewRows(columns).AddRow(want.URI, want.Project, want.Degree, want.RequestID, want.CreatedAt).RowError(0, readErr), wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + db, mock, store := setupValidationFactStoreTest(t) + defer db.Close() + query := mock.ExpectQuery(`SELECT uri, project, degree, request_id, created_at FROM validation_fact\s+WHERE queue = \? AND uri = \? AND project > \?\s+ORDER BY project ASC LIMIT \?`).WithArgs(testFactQueue, testFactURI, "project-a", 3) + if tt.queryErr != nil { + query.WillReturnError(tt.queryErr) + } else { + query.WillReturnRows(tt.rows) + } + got, err := store.ListProjects(context.Background(), testFactURI, "project-a", 3) + if tt.wantErr { + require.Error(t, err) + if tt.queryErr != nil { + assert.ErrorIs(t, err, tt.queryErr) + } + } else { + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + require.NoError(t, mock.ExpectationsWereMet()) + }) + } +} + +func TestValidationFactStore_ListProjectsRejectsInvalidLimit(t *testing.T) { + for _, limit := range []int{0, -1} { + db, mock, store := setupValidationFactStoreTest(t) + _, err := store.ListProjects(context.Background(), testFactURI, "", limit) + require.Error(t, err) + require.NoError(t, mock.ExpectationsWereMet()) + mock.ExpectClose() + require.NoError(t, db.Close()) + } +} diff --git a/stovepipe/extension/storage/validation_fact_store.go b/stovepipe/extension/storage/validation_fact_store.go index fb2fe15b7..67726f6b5 100644 --- a/stovepipe/extension/storage/validation_fact_store.go +++ b/stovepipe/extension/storage/validation_fact_store.go @@ -35,4 +35,9 @@ type ValidationFactStore interface { // Get returns the bound queue's fact for uri and project. Returns ErrNotFound if no // fact exists, which means the scope was never validated rather than validated green. Get(ctx context.Context, uri, project string) (entity.ValidationFact, error) + + // ListProjects returns at most limit project facts for uri in ascending project-ID + // order, excluding the repository fact. afterProject is an exclusive cursor (empty + // for the first page); limit must be positive. No matches returns an empty slice. + ListProjects(ctx context.Context, uri, afterProject string, limit int) ([]entity.ValidationFact, error) } diff --git a/test/integration/stovepipe/extension/storage/suite.go b/test/integration/stovepipe/extension/storage/suite.go index 46ca2449e..fec3a3d8b 100644 --- a/test/integration/stovepipe/extension/storage/suite.go +++ b/test/integration/stovepipe/extension/storage/suite.go @@ -680,3 +680,37 @@ func (s *ValidationFactStoreContractSuite) TestValidationFactStore_QueueIsolatio require.NoError(t, err) assert.Equal(t, factA, gotA) } + +func (s *ValidationFactStoreContractSuite) TestValidationFactStore_ListProjects() { + t := s.T() + const queue = "contract/fact-list-projects" + const uri = "git://remote/contract/fact-list-projects/aaaa" + store := s.storeFor(queue) + failedA := entity.ValidationFact{URI: uri, Project: "a", Degree: 0.25, RequestID: "1", CreatedAt: 10} + failedC := entity.ValidationFact{URI: uri, Project: "c", Degree: 1, RequestID: "1", CreatedAt: 20} + greenB := entity.ValidationFact{URI: uri, Project: "b", Degree: 0, RequestID: "1", CreatedAt: 30} + for _, fact := range []entity.ValidationFact{ + failedC, failedA, + greenB, + {URI: uri, Degree: 1, RequestID: "1", CreatedAt: 40}, + {URI: uri + "-other", Project: "other-uri", Degree: 1, RequestID: "2", CreatedAt: 50}, + } { + require.NoError(t, store.Create(s.ctx, fact)) + } + require.NoError(t, s.storeFor(queue+"-other").Create(s.ctx, entity.ValidationFact{URI: uri, Project: "other-queue", Degree: 1, RequestID: "1", CreatedAt: 60})) + first, err := store.ListProjects(s.ctx, uri, "", 1) + require.NoError(t, err) + assert.Equal(t, []entity.ValidationFact{failedA}, first) + second, err := store.ListProjects(s.ctx, uri, "a", 10) + require.NoError(t, err) + assert.Equal(t, []entity.ValidationFact{greenB, failedC}, second) + all, err := store.ListProjects(s.ctx, uri, "", 10) + require.NoError(t, err) + assert.Equal(t, []entity.ValidationFact{failedA, greenB, failedC}, all) + empty, err := store.ListProjects(s.ctx, uri, "c", 10) + require.NoError(t, err) + assert.Empty(t, empty) + missing, err := store.ListProjects(s.ctx, "missing", "", 10) + require.NoError(t, err) + assert.Empty(t, missing) +} diff --git a/test/integration/stovepipe/suite_test.go b/test/integration/stovepipe/suite_test.go index 7afc77bd0..e6437b518 100644 --- a/test/integration/stovepipe/suite_test.go +++ b/test/integration/stovepipe/suite_test.go @@ -343,6 +343,32 @@ func (s *StovepipeIntegrationSuite) TestProjectStatusAPI() { assert.Empty(t, lastPage.Projects) assert.Empty(t, lastPage.NextPageToken) + _, err = s.db.Exec(`INSERT INTO validation_fact (queue, uri, project, degree, request_id, created_at) + VALUES (?, ?, ?, ?, ?, ?), (?, ?, ?, ?, ?, ?)`, + queue, uri, "project-green", 0.0, id, 4000, queue, uri, "project-b", 1.0, id, 5000) + require.NoError(t, err) + firstPage, err := s.client.GetProjectStatusByURI(s.ctx, &pb.GetProjectStatusByURIRequest{ + Queue: queue, ChangeUri: uri, PageSize: 1, + }) + require.NoError(t, err) + require.Len(t, firstPage.Projects, 1) + assert.Equal(t, "project-a", firstPage.Projects[0].GetProject()) + assert.Equal(t, 0.5, firstPage.Projects[0].GetBreakageDegree()) + assert.Equal(t, int64(3000), firstPage.GetUpdatedAtMs()) + require.NotEmpty(t, firstPage.NextPageToken) + remaining, err := s.client.GetProjectStatusByID(s.ctx, &pb.GetProjectStatusByIDRequest{ + Queue: queue, RequestId: id, PageToken: firstPage.NextPageToken, + }) + require.NoError(t, err) + require.Len(t, remaining.Projects, 2) + assert.Equal(t, "project-b", remaining.Projects[0].GetProject()) + assert.Equal(t, 1.0, remaining.Projects[0].GetBreakageDegree()) + assert.Equal(t, "project-green", remaining.Projects[1].GetProject()) + assert.Zero(t, remaining.Projects[1].GetBreakageDegree()) + assert.Equal(t, int64(5000), remaining.GetUpdatedAtMs()) + assert.Empty(t, remaining.NextPageToken) + assert.False(t, remaining.ProjectResultsComplete) + for _, selector := range []struct{ queue, id string }{{queue, "999"}, {"project-status-api/other", id}} { _, err := s.client.GetProjectStatusByID(s.ctx, &pb.GetProjectStatusByIDRequest{ Queue: selector.queue, RequestId: selector.id, Projects: []string{"project-a"}, diff --git a/web/api/src/gen/api/stovepipe/proto/stovepipe_pb.ts b/web/api/src/gen/api/stovepipe/proto/stovepipe_pb.ts index d67c881e7..2432f9341 100644 --- a/web/api/src/gen/api/stovepipe/proto/stovepipe_pb.ts +++ b/web/api/src/gen/api/stovepipe/proto/stovepipe_pb.ts @@ -352,7 +352,7 @@ export type GetProjectStatusByIDRequest = Message<"uber.submitqueue.stovepipe.Ge requestId: string; /** - * Required consumer-defined project IDs whose recorded results are requested. + * Optional consumer-defined project IDs. Empty returns all recorded project results. * * @generated from field: repeated string projects = 3; */ @@ -401,7 +401,7 @@ export type GetProjectStatusByURIRequest = Message<"uber.submitqueue.stovepipe.G changeUri: string; /** - * Required consumer-defined project IDs whose recorded results are requested. + * Optional consumer-defined project IDs. Empty returns all recorded project results. * * @generated from field: repeated string projects = 3; */