feat(stovepipe): Make project selection optional

api/stovepipe/proto/stovepipe.proto

@@ -116,7 +116,7 @@ message GetProjectStatusByIDRequest {
     string queue = 1;
     // Canonical decimal request identifier returned from Ingest.
     string request_id = 2;
-    // Required consumer-defined project IDs whose recorded results are requested.
+    // Optional consumer-defined project IDs. Empty returns all recorded project results.
     repeated string projects = 3;
     // Maximum requested projects to return in one page. Zero selects the server default.
     int32 page_size = 4;
@@ -130,7 +130,7 @@ message GetProjectStatusByURIRequest {
     string queue = 1;
     // Exact VCS-agnostic URI of the commit under validation.
     string change_uri = 2;
-    // Required consumer-defined project IDs whose recorded results are requested.
+    // Optional consumer-defined project IDs. Empty returns all recorded project results.
     repeated string projects = 3;
     // Maximum requested projects to return in one page. Zero selects the server default.
     int32 page_size = 4;

doc/rfc/stovepipe/project-status-api.md

@@ -17,7 +17,7 @@ The request and response fields have the following API-level meaning. The publis
 | `queue` | Yes | Identifies the queue in which to find the validation request. |
 | `request_id` | ID lookup | Canonical positive decimal request ID returned by `Ingest`. |
 | `change_uri` | URI lookup | Identifies the exact commit URI under validation. |
-| `projects` | Yes | Consumer-defined project IDs whose results are requested. At least one ID is required. |
+| `projects` | No | Consumer-defined project IDs whose results are requested. Omitted or empty returns all recorded project results. |
 | `page_size` | No | Limits one page of results for the requested projects. |
 | `page_token` | No | Continues a previous page of results for the requested projects. |
 
@@ -29,7 +29,7 @@ The request and response fields have the following API-level meaning. The publis
 | `updated_at_ms` | Records, in Unix milliseconds, the newest durable lifecycle or result record represented by the response. |
 | `repository_breakage_degree` | Returns the whole-repository result when it is durably recorded. |
 | `project_results_complete` | Indicates whether the implementation has durably finished producing results for the supplied project IDs. |
-| `projects` | Returns recorded results for requested project IDs. |
+| `projects` | Returns recorded results for requested project IDs, or all recorded project results when none are supplied. |
 | `next_page_token` | Continues requested-project result pagination when another page exists. |
 
 `request_state` is a stable public lifecycle vocabulary: `accepted`, `processing`, `succeeded`, `failed`, `cancelled`, or `superseded`. Clients must tolerate a future value. A terminal request state does not by itself mean that project results are complete.
@@ -56,25 +56,27 @@ Both lookups are queue-scoped and use the same durable request summary and valid
 
 The request-URI mapping admits one request per `(queue, change_uri)`, so the endpoint returns one authoritative request. Revalidation support must introduce an explicit authoritative-request rule; it must not silently change this lookup's meaning.
 
-Queue, URI, request ID, and project ID comparisons are byte-exact and are limited to 255 bytes. An empty queue, lookup selector, project list, or project ID is invalid. A request cannot contain duplicate project IDs. `page_size=0` selects the default of 50; the maximum is 200. Request IDs use the same canonical positive decimal format as `Ingest` and request-history lookups. Page tokens are opaque and bound to the selected request and requested project list; either endpoint can continue a page for the same queue, request, and projects.
+Queue, URI, request ID, and project ID comparisons are byte-exact and are limited to 255 bytes. An empty queue, lookup selector, or individual project ID is invalid. A request cannot contain duplicate project IDs. `page_size=0` selects the default of 50; the maximum is 200. Request IDs use the same canonical positive decimal format as `Ingest` and request-history lookups. Page tokens are opaque and bound to the selected request and requested project list; either endpoint can continue a page for the same queue, request, and projects.
 
 ## Project Results and Pagination
 
-The caller supplies the project IDs for which it wants recorded results. The API does not define project discovery, validation selection, or attribution. A project absent from the response has no universal meaning and consumers must not infer that it is green.
+The caller may supply the project IDs for which it wants recorded results. The API does not define project discovery, validation selection, or attribution. A project absent from the response has no universal meaning and consumers must not infer that it is green.
 
-The response includes only results for the supplied project IDs, in the supplied order, and paginates that list. An implementation may record results for every requested project, only requested projects with a non-green result, or another documented subset.
+When project IDs are supplied, the response includes only results for those IDs, in the supplied order, and paginates that list. An implementation may record results for every requested project, only requested projects with a non-green result, or another documented subset.
+
+With no project IDs supplied, the response lists all recorded project results in project-ID order, excluding the repository fact. Results are a live view; restart pagination to discover facts recorded behind the cursor.
 
 Each project result is an immutable validation fact keyed by queue, commit URI, and project ID. A result must identify the selected request. `project_results_complete` is false until the implementation has durably finished producing results for the supplied project IDs; a completed empty set is valid.
 
 Callers use `project_results_complete`, rather than the presence or absence of an individual project result, to determine whether the implementation has finished producing results for the requested projects. `next_page_token` is empty only on the final page. A terminal lifecycle notification is emitted only after the repository result and any applicable project completion record are durable. If notification delivery fails, consumers can recover the same result through either endpoint.
 
 ## Implementation
 
-Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and any recorded facts for requested projects in paginated request order. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page.
+Both endpoints return durable lifecycle summaries, the whole-repository fact when present, and recorded facts for requested projects in paginated request order, or all recorded project results in project-ID order when none are requested. `project_results_complete` remains false until a durable completion projection is available. Missing facts are omitted; an explicitly recorded zero degree remains present. `updated_at_ms` is the maximum timestamp of the summary and the facts represented on the page.
 
 ## Errors and Authorization
 
-- Invalid selectors, an empty project list, an invalid project ID, duplicate project IDs, or malformed page tokens are user errors.
+- Invalid selectors, an invalid project ID, duplicate project IDs, or malformed page tokens are user errors.
 - An unknown queue-scoped request ID or commit URI is not found.
 - A request-URI mapping whose summary is not visible is retryable.
 - A selector and summary that disagree, a result belonging to another request, or a completion record with a missing project result is an internal consistency error.

stovepipe/controller/project_status.go

@@ -23,7 +23,7 @@ import (
 	"errors"
 	"fmt"
 	"math"
-	"reflect"
+	"slices"
 
 	"github.com/uber-go/tally"
 	"github.com/uber/submitqueue/platform/errs"
@@ -39,9 +39,11 @@ const (
 )
 
 type projectStatusPageToken struct {
-	RequestID string   `json:"request_id"`
-	Projects  []string `json:"projects"`
-	NextIndex int      `json:"next_index"`
+	Queue        string   `json:"queue"`
+	RequestID    string   `json:"request_id"`
+	Projects     []string `json:"projects"`
+	NextIndex    int      `json:"next_index"`
+	AfterProject string   `json:"after_project,omitempty"`
 }
 
 // ProjectStatusNotFoundError indicates that no validation request matches a lookup selector.
@@ -102,7 +104,7 @@ func NewProjectStatusController(logger *zap.SugaredLogger, scope tally.Scope, st
 	}
 }
 
-// GetProjectStatus returns the selected request summary and its repository validation fact, if recorded.
+// GetProjectStatus returns the selected request summary and recorded validation facts.
 func (c *projectStatusController) GetProjectStatus(ctx context.Context, req entity.GetProjectStatusRequest) (result entity.GetProjectStatusResult, retErr error) {
 	op := metrics.Begin(c.metricsScope, "get_project_status", metrics.StorageLatencyBuckets, metrics.TagsFromContext(ctx)...)
 	defer func() { op.Complete(retErr) }()
@@ -158,26 +160,19 @@ func (c *projectStatusController) GetProjectStatus(ctx context.Context, req enti
 			result.UpdatedAtMs = fact.CreatedAt
 		}
 	}
-	page, err := selectProjectStatusPage(req, requestSummary.RequestID)
+
+	page, err := loadProjectStatusPage(ctx, req, requestSummary, factStore)
 	if err != nil {
 		return entity.GetProjectStatusResult{}, err
 	}
-	for _, project := range page.projects {
-		projectFact, err := factStore.Get(ctx, requestSummary.URI, project)
-		if err != nil {
-			if storage.IsNotFound(err) {
-				continue
-			}
-			return entity.GetProjectStatusResult{}, fmt.Errorf("GetProjectStatus failed to load project fact for request %q and project %q: %w", requestSummary.RequestID, project, err)
-		}
-		if err := validateProjectFact(projectFact, requestSummary, project); err != nil {
-			return entity.GetProjectStatusResult{}, err
-		}
-		result.ProjectValidationFacts = append(result.ProjectValidationFacts, projectFact)
+
+	result.ProjectValidationFacts = page.facts
+	for _, projectFact := range page.facts {
 		if projectFact.CreatedAt > result.UpdatedAtMs {
 			result.UpdatedAtMs = projectFact.CreatedAt
 		}
 	}
+
 	result.NextPageToken = page.nextToken
 
 	c.logger.Debugw("project status retrieved", "request_id", requestSummary.RequestID, "queue", requestSummary.Queue, "change_uri", requestSummary.URI, "has_repository_result", result.HasRepositoryValidationFact, "project_results", len(result.ProjectValidationFacts))
@@ -200,9 +195,6 @@ func validateProjectStatusRequest(req entity.GetProjectStatusRequest) error {
 			return fmt.Errorf("GetProjectStatus invalid change_uri=%q: %w", req.ChangeURI, err)
 		}
 	}
-	if len(req.Projects) == 0 {
-		return fmt.Errorf("GetProjectStatus projects must be non-empty: %w", ErrInvalidRequest)
-	}
 	seen := make(map[string]struct{}, len(req.Projects))
 	for _, project := range req.Projects {
 		if err := validateHistoryIdentifier("project", project); err != nil {
@@ -220,55 +212,153 @@ func validateProjectStatusRequest(req entity.GetProjectStatusRequest) error {
 }
 
 type projectStatusPage struct {
-	projects  []string
+	facts     []entity.ValidationFact
 	nextToken string
 }
 
-func selectProjectStatusPage(req entity.GetProjectStatusRequest, requestID string) (projectStatusPage, error) {
-	start := 0
-	if req.PageToken != "" {
-		token, err := decodeProjectStatusPageToken(req.PageToken)
-		if err != nil {
-			return projectStatusPage{}, err
-		}
-		if token.RequestID != requestID || !reflect.DeepEqual(token.Projects, req.Projects) || token.NextIndex <= 0 || token.NextIndex >= len(req.Projects) {
-			return projectStatusPage{}, fmt.Errorf("GetProjectStatus page_token does not match the selected request and projects: %w", ErrInvalidRequest)
-		}
-		start = token.NextIndex
+func loadProjectStatusPage(
+	ctx context.Context,
+	req entity.GetProjectStatusRequest,
+	summary entity.RequestSummary,
+	store storage.ValidationFactStore,
+) (projectStatusPage, error) {
+	token, err := parseProjectStatusPageToken(req, summary)
+	if err != nil {
+		return projectStatusPage{}, err
 	}
+
 	pageSize := int(req.PageSize)
 	if pageSize == 0 {
 		pageSize = defaultProjectStatusPageSize
 	}
-	end := min(start+pageSize, len(req.Projects))
-	page := projectStatusPage{projects: req.Projects[start:end]}
-	if end < len(req.Projects) {
-		token, err := encodeProjectStatusPageToken(projectStatusPageToken{RequestID: requestID, Projects: req.Projects, NextIndex: end})
+
+	if len(req.Projects) == 0 {
+		return listProjectStatusPage(ctx, summary, store, token, pageSize)
+	}
+
+	return getRequestedProjectStatusPage(ctx, summary, store, token, pageSize)
+}
+
+func getRequestedProjectStatusPage(
+	ctx context.Context,
+	summary entity.RequestSummary,
+	store storage.ValidationFactStore,
+	token projectStatusPageToken,
+	pageSize int,
+) (projectStatusPage, error) {
+	end := min(token.NextIndex+pageSize, len(token.Projects))
+	var page projectStatusPage
+
+	for _, project := range token.Projects[token.NextIndex:end] {
+		fact, err := store.Get(ctx, summary.URI, project)
 		if err != nil {
-			return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to create page token: %w", err)
+			if storage.IsNotFound(err) {
+				continue
+			}
+
+			return projectStatusPage{}, fmt.Errorf(
+				"GetProjectStatus failed to load project fact for request %q and project %q: %w",
+				summary.RequestID, project, err,
+			)
+		}
+
+		if err := validateProjectFact(fact, summary, project); err != nil {
+			return projectStatusPage{}, err
+		}
+
+		page.facts = append(page.facts, fact)
+	}
+
+	var err error
+	if end < len(token.Projects) {
+		token.NextIndex = end
+		page.nextToken, err = encodeProjectStatusPageToken(token)
+	}
+
+	return page, err
+}
+
+func listProjectStatusPage(
+	ctx context.Context,
+	summary entity.RequestSummary,
+	store storage.ValidationFactStore,
+	token projectStatusPageToken,
+	pageSize int,
+) (projectStatusPage, error) {
+	facts, err := store.ListProjects(ctx, summary.URI, token.AfterProject, pageSize+1)
+	if err != nil {
+		return projectStatusPage{}, fmt.Errorf("GetProjectStatus failed to list project facts for request %q: %w", summary.RequestID, err)
+	}
+
+	for _, fact := range facts {
+		if err := validateProjectFact(fact, summary, fact.Project); err != nil {
+			return projectStatusPage{}, err
+		}
+
+		if fact.Project == "" {
+			return projectStatusPage{}, &ProjectStatusConsistencyError{message: "project listing contains a repository fact"}
 		}
-		page.nextToken = token
 	}
-	return page, nil
+
+	page := projectStatusPage{facts: facts}
+	if len(facts) > pageSize {
+		page.facts = facts[:pageSize]
+		token.AfterProject = page.facts[pageSize-1].Project
+		page.nextToken, err = encodeProjectStatusPageToken(token)
+	}
+
+	return page, err
 }
 
 func encodeProjectStatusPageToken(token projectStatusPageToken) (string, error) {
 	contents, err := json.Marshal(token)
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("GetProjectStatus failed to create page token: %w", err)
 	}
+
 	return base64.RawURLEncoding.EncodeToString(contents), nil
 }
 
-func decodeProjectStatusPageToken(encoded string) (projectStatusPageToken, error) {
-	contents, err := base64.RawURLEncoding.DecodeString(encoded)
+func parseProjectStatusPageToken(
+	req entity.GetProjectStatusRequest,
+	summary entity.RequestSummary,
+) (projectStatusPageToken, error) {
+	if req.PageToken == "" {
+		return projectStatusPageToken{
+			Queue:     summary.Queue,
+			RequestID: summary.RequestID,
+			Projects:  req.Projects,
+		}, nil
+	}
+
+	invalidToken := fmt.Errorf("GetProjectStatus page_token does not match the selected request and projects: %w", ErrInvalidRequest)
+
+	contents, err := base64.RawURLEncoding.DecodeString(req.PageToken)
 	if err != nil {
-		return projectStatusPageToken{}, fmt.Errorf("GetProjectStatus invalid page_token: %w", ErrInvalidRequest)
+		return projectStatusPageToken{}, invalidToken
 	}
+
 	var token projectStatusPageToken
 	if err := json.Unmarshal(contents, &token); err != nil {
-		return projectStatusPageToken{}, fmt.Errorf("GetProjectStatus invalid page_token: %w", ErrInvalidRequest)
+		return projectStatusPageToken{}, invalidToken
 	}
+
+	if token.Queue != summary.Queue ||
+		token.RequestID != summary.RequestID ||
+		!slices.Equal(token.Projects, req.Projects) {
+		return projectStatusPageToken{}, invalidToken
+	}
+
+	if len(req.Projects) == 0 {
+		if token.NextIndex != 0 || validateHistoryIdentifier("project", token.AfterProject) != nil {
+			return projectStatusPageToken{}, invalidToken
+		}
+	} else if token.AfterProject != "" ||
+		token.NextIndex <= 0 ||
+		token.NextIndex >= len(req.Projects) {
+		return projectStatusPageToken{}, invalidToken
+	}
+
 	return token, nil
 }