Skip to content

Commit e1c50cb

Browse files
committed
feat(stovepipe): Persist build issue observations
1 parent 3c505ea commit e1c50cb

16 files changed

Lines changed: 577 additions & 1 deletion

File tree

‎stovepipe/extension/storage/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_issue_store.go",
67
"build_store.go",
78
"queue_log_store.go",
89
"queue_store.go",

‎stovepipe/extension/storage/README.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Storage
22

3-
Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`). Implementations live under `extension/storage/<impl>/`.
3+
Pluggable persistence interfaces for Stovepipe entities (`RequestStore`, `RequestURIStore`, `QueueStore`, `BuildStore`, `BuildIssueStore`). Implementations live under `extension/storage/<impl>/`.
44

55
The aggregate is resolved per queue through a factory keyed by queue name, mirroring the extension contract and the [submitqueue storage seam](../../../submitqueue/extension/storage/README.md#queue-scoped-resolution): a resolved instance is bound to its queue, entity arguments whose queue disagrees with the binding are rejected, and reads never surface another queue's records. Stovepipe has no cross-queue read paths, so every store — including `QueueStore`, whose row key is the queue name itself — lives inside the queue-scoped aggregate; there is no global remainder.
66

@@ -23,3 +23,11 @@ Same design space as [`submitqueue/extension/storage`](../../../submitqueue/exte
2323
`RequestAcceptanceStore` is the ordered lookup mapping for List: immutable `(queue, accepted_at_ms, request_id)` keys with bounded primary-key range scans, not a secondary index or a second status projection. The request-log materializer ensures each known acceptance has a mapping after summary persistence, including retries where the summary is unchanged. Unknown acceptance times are not mapped; historical backfill is separate.
2424

2525
`QueueStore` owns internal enablement alongside pipeline coordination. `QueueSummaryStore` persists the current read model projected from immutable `QueueLogStore` occurrences. Enablement sequences, operational row versions and projection versions are distinct. These stores use primary-key operations; version arithmetic, duplicate reconciliation and projection repair belong to business logic. See [queue state](../../../doc/rfc/stovepipe/queue-state.md).
26+
27+
## Build issue observations
28+
29+
`BuildIssueStore` accumulates immutable observations identified by build, task, and unit within the bound queue. The build ID is supplied separately from the detected issue. Repeated observations return `ErrAlreadyExists`, which callers may treat as successful persistence. Empty unit IDs represent task-wide observations.
30+
31+
Each observation is stored separately, not in a serialized collection. Observations are historical evidence; they do not track whether an issue is currently open or resolved. No update, deletion, or absence-based recovery is implied by this store.
32+
33+
The MySQL primary key contains the queue, build ID, task ID, and unit ID directly as binary strings, preserving byte-exact identity. Their limits are 255, 1020, 255, and 1024 bytes respectively, for a combined maximum of 2554 bytes. Creates and reads reject identifiers exceeding these limits before querying the database; identifiers are never truncated.
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package storage
16+
17+
//go:generate mockgen -source=build_issue_store.go -destination=mock/build_issue_store_mock.go -package=mock
18+
19+
import (
20+
"context"
21+
22+
"github.com/uber/submitqueue/stovepipe/entity"
23+
)
24+
25+
// BuildIssueStore retains immutable observations keyed by build, task, and unit
26+
// within the bound queue. Presence records an observation, not a currently open issue.
27+
type BuildIssueStore interface {
28+
// Create returns ErrAlreadyExists for a previously recorded build/task/unit.
29+
// Callers may treat that duplicate as success when accumulating observations.
30+
Create(ctx context.Context, buildID string, issue entity.BuildIssue) error
31+
32+
// Get returns ErrNotFound when the build/task/unit has not been recorded.
33+
Get(ctx context.Context, buildID, taskID, unitID string) (entity.BuildIssue, error)
34+
}

‎stovepipe/extension/storage/mock/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_issue_store_mock.go",
67
"build_store_mock.go",
78
"queue_log_store_mock.go",
89
"queue_store_mock.go",

‎stovepipe/extension/storage/mock/build_issue_store_mock.go‎

Lines changed: 71 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎stovepipe/extension/storage/mock/storage_mock.go‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎stovepipe/extension/storage/mysql/BUILD.bazel‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "go_default_library",
55
srcs = [
6+
"build_issue_store.go",
67
"build_store.go",
78
"queue_log_store.go",
89
"queue_store.go",
@@ -29,6 +30,7 @@ go_library(
2930
go_test(
3031
name = "go_default_test",
3132
srcs = [
33+
"build_issue_store_test.go",
3234
"build_store_test.go",
3335
"queue_store_test.go",
3436
"queue_summary_store_test.go",
Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
// Copyright (c) 2026 Uber Technologies, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package mysql
16+
17+
import (
18+
"context"
19+
"database/sql"
20+
"errors"
21+
"fmt"
22+
23+
"github.com/uber-go/tally"
24+
25+
"github.com/uber/submitqueue/platform/metrics"
26+
"github.com/uber/submitqueue/stovepipe/entity"
27+
"github.com/uber/submitqueue/stovepipe/extension/storage"
28+
)
29+
30+
type buildIssueStore struct {
31+
db *sql.DB
32+
scope tally.Scope
33+
queue string
34+
}
35+
36+
// NewBuildIssueStore binds observations to queue over the shared database.
37+
func NewBuildIssueStore(db *sql.DB, scope tally.Scope, queue string) storage.BuildIssueStore {
38+
return &buildIssueStore{db: db, scope: scope, queue: queue}
39+
}
40+
41+
func (s *buildIssueStore) Create(ctx context.Context, buildID string, issue entity.BuildIssue) (retErr error) {
42+
op := metrics.Begin(s.scope, "create", metrics.StorageLatencyBuckets)
43+
defer func() { op.Complete(retErr) }()
44+
45+
if err := validateBuildIssueKey(s.queue, buildID, issue.TaskID, issue.UnitID); err != nil {
46+
return err
47+
}
48+
_, err := s.db.ExecContext(ctx,
49+
`INSERT INTO build_issue (queue, build_id, task_id, unit_id)
50+
VALUES (?, ?, ?, ?)`,
51+
s.queue, buildID, issue.TaskID, issue.UnitID,
52+
)
53+
if err != nil {
54+
if isDuplicateEntry(err) {
55+
return fmt.Errorf("build issue build=%q task=%q unit=%q: %w", buildID, issue.TaskID, issue.UnitID, storage.ErrAlreadyExists)
56+
}
57+
return fmt.Errorf("failed to create build issue build=%q task=%q unit=%q: %w", buildID, issue.TaskID, issue.UnitID, err)
58+
}
59+
return nil
60+
}
61+
62+
func (s *buildIssueStore) Get(ctx context.Context, buildID, taskID, unitID string) (ret entity.BuildIssue, retErr error) {
63+
op := metrics.Begin(s.scope, "get", metrics.StorageLatencyBuckets)
64+
defer func() { op.Complete(retErr) }()
65+
66+
if err := validateBuildIssueKey(s.queue, buildID, taskID, unitID); err != nil {
67+
return entity.BuildIssue{}, err
68+
}
69+
var issue entity.BuildIssue
70+
err := s.db.QueryRowContext(ctx,
71+
`SELECT task_id, unit_id FROM build_issue
72+
WHERE queue = ? AND build_id = ? AND task_id = ? AND unit_id = ?`,
73+
s.queue, buildID, taskID, unitID,
74+
).Scan(&issue.TaskID, &issue.UnitID)
75+
if errors.Is(err, sql.ErrNoRows) {
76+
return entity.BuildIssue{}, storage.WrapNotFound(err)
77+
}
78+
if err != nil {
79+
return entity.BuildIssue{}, fmt.Errorf("failed to get build issue build=%q task=%q unit=%q: %w", buildID, taskID, unitID, err)
80+
}
81+
return issue, nil
82+
}
83+
84+
func validateBuildIssueKey(queue, buildID, taskID, unitID string) error {
85+
for _, field := range []struct {
86+
name string
87+
value string
88+
maxBytes int
89+
}{
90+
{"queue", queue, 255},
91+
{"build ID", buildID, 1020},
92+
{"task ID", taskID, 255},
93+
{"unit ID", unitID, 1024},
94+
} {
95+
if len(field.value) > field.maxBytes {
96+
return fmt.Errorf("build issue %s exceeds %d-byte limit", field.name, field.maxBytes)
97+
}
98+
}
99+
return nil
100+
}

0 commit comments

Comments
 (0)