diff --git a/Makefile b/Makefile index 24597829b..de514a8fd 100644 --- a/Makefile +++ b/Makefile @@ -629,6 +629,7 @@ GO_GENERATE_PACKAGES := \ ./stovepipe/core/requestlog/... \ ./stovepipe/extension/buildissue/... \ ./stovepipe/extension/buildrunner/... \ + ./stovepipe/extension/fixer/... \ ./stovepipe/extension/projectstatus/... \ ./stovepipe/extension/queueconfig/... \ ./stovepipe/extension/sourcecontrol/... \ diff --git a/doc/rfc/stovepipe/fixer.md b/doc/rfc/stovepipe/fixer.md new file mode 100644 index 000000000..00f039d6e --- /dev/null +++ b/doc/rfc/stovepipe/fixer.md @@ -0,0 +1,88 @@ +# Stovepipe repair coordination + +Build analysis discovers confirmed validation issues. The `fix` queue stage selects a repair, persists assignment, and submits it through a Go extension. The `fixsignal` stage monitors execution; build analysis establishes verified recovery. The [workflow](workflow.md#workflow) shows these stages alongside normal validation. + +## Initial policy: one repair per build + +Request a repair as soon as the detector confirms the first failure, while the build may still be active. Persist the selected repair in `Build.FirstFixID`. Subsequent polls, duplicate observations, and later failures reuse that selection. Keep the field after the attempt finishes so the same build cannot automatically launch another fix. + +Continue collecting every confirmed failure. Additional repairs require a later decision policy that can distinguish unrelated causes. A reader never supplies a mandatory cause key. Existing assignment of an issue to a repair takes precedence over creating more work. + +## 1. Persist the observation and issue opening + +[`buildanalysis.Controller.Process`](../../../stovepipe/controller/buildanalysis/buildanalysis.go) calls [`buildissue.Detector.Detect`](../../../stovepipe/extension/buildissue/buildissue.go). The detector owns confirmation and provider retry rules. It returns `BuildIssue{TaskID, UnitID}`: a stable logical task and optional failed unit. Empty unit identifies a task-wide failure. + +Persist each positive observation through [`BuildIssueStore`](../../../stovepipe/extension/storage/build_issue_store.go). Reconcile its [`OpenBuildIssue`](../../../stovepipe/entity/open_build_issue.go), keyed by queue/task/unit. `FirstObservedBuildID` identifies this continuous opening; `LastObservedBuildID` records subsequent evidence. `FixID` is the authoritative assignment. Updates guard both the opening and expected version, preserving fields owned by the other stage. + +Use [`BuildIssueRef`](../../../stovepipe/entity/fix.go) containing task, unit, and first-observed build in each repair. Build IDs are opaque identity, not ordering tokens. Reopening uses a new first-observed build. Delayed writes must never update a different opening. + +After persistence, publish [`FixRequest`](../../../stovepipe/core/messagequeue/proto/fix.proto) containing the detecting BuildID and issue-opening reference. Partition by detecting BuildID for the initial build-scoped decision. Duplicate observation writes still reach reconciliation/publication: a crash after the first write must not suppress repair forever. Publication retries reuse a deterministic identity; later evidence/recovery notices identify their own occasion. + +For provenance, use `Fix.BuildID -> Build.RequestID -> Request.URI`. Issue-opening references retain the original failure build even when the initiating repair uses a later observation. Build analysis owns historical observation and resolution retention when current issue rows are removed. + +## 2. Select the build's first repair + +`fix.Controller.Process` reloads the Build, Request, and current issue opening. It checks queue identity and the persisted observation before considering dispatch. + +1. If `Build.FirstFixID` is set, resume that selection and complete any interrupted submission or monitoring publication. Later failures do not start additional repairs. +2. If the exact issue opening already has an active assignment, conditionally select that repair for the build and resume its monitoring. +3. Otherwise, call [`RepairEvaluator.Evaluate`](../../../stovepipe/extension/fixer/fixer.go) with the detecting build, issue reference, and candidate repair IDs from `Queue.ActiveFixIDs`. The implementation resolves diagnostics through injected dependencies. + +| Decision | Action | +| --- | --- | +| Start | Persist a new queued repair, then conditionally set `Build.FirstFixID`. | +| Attach | Validate the chosen repair, then conditionally select it and assign the issue. | +| Defer | Hold the delivery under a bounded policy and evaluate again later. | + +The initial evaluator starts a repair for the first unassigned confirmed issue. It does not claim to recognize common root causes. A later evaluator can inspect active repairs and recommend attachment, deferral, or additional independent work. + +Build selection uses `Build.Version`, preserving status updates from buildsignal. A competing selection wins permanently for this build; unused unsubmitted candidates can be skipped. This coordinates automatic admission without a separate reservation store. + +## 3. Persist membership and assignment + +Persist the repair before selecting it on the Build. After selection, add it to `Queue.ActiveFixIDs`, persist the issue reference on the repair, and conditionally assign the exact open issue's `FixID`. Every update reloads on conflict and preserves unrelated snapshot fields. Never mutate caller-owned slices or maps. + +An existing issue assignment wins over a stale evaluation. If recovery or reassignment makes a queued candidate unnecessary before dispatch, mark it skipped, preserve the build selection, and remove its active membership. `Fix.Issues` supports navigation; it does not override the open issue's authoritative assignment. + +Queue membership shares `Queue.Version`; all queue writers preserve active IDs. It enumerates repairs for observation and candidate discovery through primary-key reads. A crash before membership can leave an unsubmitted candidate; replay follows `Build.FirstFixID` to finish selection. A losing candidate must never dispatch. + +## 4. Trigger the repair + +Each [`Fix`](../../../stovepipe/entity/fix.go) is one durable attempt covering one or more issue openings. Its ID is stable across delivery retries. Deliberately trying another repair creates a new Fix rather than resetting a terminal attempt. + +`fix.Controller.Process` rechecks assignment, persists `submitting`, and calls [`RepairExecutor.Submit`](../../../stovepipe/extension/fixer/fixer.go). The executor receives the persisted queue/fix identity. Publish [`FixSignal`](../../../stovepipe/core/messagequeue/proto/fixsignal.proto) after persisting `submitting` and before the outbound call, so monitoring survives an ambiguous reply or exhausted submission delivery. Success returns a nonempty opaque provider execution reference; conditionally persist it in `Fix.ExecutionRef`. + +An error may follow acceptance. Preserve the same fix identity and allow `Observe` even when the submission response was lost and `ExecutionRef` is empty. The provider adapter correlates by that identity and uses a known execution reference as additional evidence. A missing observation means execution is unresolved; it does not prove nonacceptance. Apply bounded startup observation and retry policies with durable timing. Occasional duplicate provider jobs are accepted. + +Use conditional snapshot writes for returned results. On a conflict, reload before deciding what remains to persist. A change to membership or metadata does not create a new attempt. Reconcile failed and dead-lettered publication so an accepted repair remains discoverable. + +## 5. Observe execution and recovery + +`fixsignal.Controller.Process` reloads the repair, calls `RepairExecutor.Observe`, and persists execution progress. Hold while execution or verification is pending. A successful submission, finished worker, or merged change advances progress but does not establish issue recovery. + +Build analysis owns authoritative resolution for the exact opening, including the validating BuildID. Its detector only reports positive failures: neither an empty detection result nor the accumulated historical observation table establishes recovery. A successful final analysis must establish comparable configured coverage. The runner makes final artifacts available at completion, so no artifact-availability wait is needed. Cancelled/skipped coverage, failed analysis, and no-op detection leave recovery unknown. + +Resolution/history must survive deletion of the current open row and notify its assigned repair through replayable publication. A missing current row alone does not supply the validating build or explain its removal. Source-order reconciliation and enumeration of relevant open issue keys remain build-analysis responsibilities. + +Keep the repair awaiting verification until that resolution integration is available. Complete it once its assigned occurrences have recovered and all known external executions have settled. Preserve known duplicate executions in monitoring; one failed result must not hide other outstanding work. + +## 6. Extend decision making when coverage is available + +A worker may discover a common root cause while investigating. Use structured coverage evidence before associating another issue with that repair. Persist expected assignment and repair versions, update membership and assignment through replay, and notify the affected repairs. + +Assigning another issue does not itself update or stop an external worker. Define the worker's context-refresh and supersession acknowledgment protocol before relying on cooperative overlap cancellation. Human-readable summaries can inform evaluation but are not authoritative coverage or recovery. + +## Storage and integration + +| Record | Key | Repair data | +| --- | --- | --- | +| Fix | queue/fix ID | Initiating BuildID, issue references, attempt lifecycle, optional execution reference and metadata, version, timestamps. | +| Existing Build | queue/build ID | Write-once `FirstFixID` under the existing snapshot version. | +| Existing Queue | queue | `ActiveFixIDs` under the existing snapshot version. | +| OpenBuildIssue | queue/task/unit | Current `FixID` alongside opening identity and observation state. | + +The fixer adds one table and uses primary-key operations. Build-analysis history remains separate from repair records. Nil/empty metadata maps persist as `{}`; nil/empty active-ID lists persist as `[]`. Existing databases need the new columns added/backfilled before deployment. Schemas initialize fresh databases. + +Both queue controllers follow [`consumer.Controller`](../../../platform/consumer/controller.go): `Process(context.Context, consumer.Delivery) error`, `Name`, `TopicKey`, and `ConsumerGroup`. Service wiring resolves per-queue evaluators/executors, registers fix/fixsignal and their recovery behavior, and supplies host integrations. + +The API layer defines the contracts; the persistence layer supplies MySQL support. Runtime controller wiring and provider adapters follow as separate changes. Initial dispatch can run before complete recovery reconciliation, while leaving completion awaiting verification. diff --git a/doc/rfc/stovepipe/workflow.md b/doc/rfc/stovepipe/workflow.md index 77b80a175..639808b06 100644 --- a/doc/rfc/stovepipe/workflow.md +++ b/doc/rfc/stovepipe/workflow.md @@ -188,6 +188,8 @@ Per-stage design detail lives under `steps/` so this doc stays a pipeline overvi - [buildsignal.md](steps/buildsignal.md) — the poll loop: hold-based re-poll cadence, per-build partitioning, and the fail-closed handoff to record - [record.md](steps/record.md) — turning a terminal build outcome into an immutable validation fact, monotonic last-green advancement and ref promotion, and the hook event announcing the outcome. Project analysis already runs inline through `projectstatus.Resolver` and may later use results calculated by build analysis +For repair decisions, execution, and recovery, see the [repair RFC](fixer.md). + ## Dedup, idempotency, and history rewrites Ingestion is idempotent on `(Queue, head URI)`, so duplicate poller reports — and any future webhook producer racing the poller — converge on one Request. The pipeline persists the Queue's last-green URI and per-URI greenness, so it must tolerate a **history rewrite**: when `SourceControl` reports that the last-green URI is no longer an ancestor of the current head, `process` falls back to a full-monorepo build rather than trusting a baseline that no longer exists on the branch. The system converges to a correct greenness rather than wedging on a stale pointer. diff --git a/stovepipe/core/messagequeue/README.md b/stovepipe/core/messagequeue/README.md index 39d8df7a8..09fe1757d 100644 --- a/stovepipe/core/messagequeue/README.md +++ b/stovepipe/core/messagequeue/README.md @@ -11,3 +11,12 @@ Payloads are defined in proto3 (`proto/`, generated into `protopb/`) and seriali - **buildsignal** (`TopicKeyBuildSignal`, `BuildSignal`) — build publishes the build id here after triggering; buildsignal holds the delivery between polls until the build reaches a terminal status. Partitioned by build id, so each build's poll loop is an independent partition. - **buildanalysis** (`TopicKeyBuildAnalysis`, `BuildAnalysis`) — identifies a persisted build for issue detection, partitioned by build id. Build publishes a start notice and buildsignal publishes a completion notice with a distinct message id. See [build analysis](../../controller/buildanalysis/README.md) for polling and persistence behavior. - **record** (`TopicKeyRecord`, `Record`) — buildsignal publishes the request id after persisting the terminal outcome; record reloads the request and records its validation fact. Partitioned by request id. + +## Repair stage contracts + +The repair messages and topic keys support the [repair design](../../../doc/rfc/stovepipe/fixer.md). Both stages implement `consumer.Controller`. + +- **fix** (`TopicKeyFix`, [`FixRequest`](proto/fix.proto)) — identifies a persisted observation and issue opening using queue, detecting build, task, unit, and first-observed build. Partition by detecting build ID. The initial policy selects one repair in `Build.FirstFixID` and retains that selection after completion. Later failures continue to be recorded; additional repairs require future decision logic. +- **fixsignal** (`TopicKeyFixSignal`, [`FixSignal`](proto/fixsignal.proto)) — identifies a persisted repair by queue and fix ID. Partition by fix ID. Publish monitoring before outbound submission so an accepted repair remains observable if the trigger reply is lost. Hold between execution observations and while awaiting verified recovery. + +Persist observations and their current opening before publishing a request. Replay publication after duplicate observation writes so a crash between persistence and handoff can recover. Provider success advances execution progress; build analysis supplies authoritative recovery for each issue opening. diff --git a/stovepipe/core/messagequeue/messagequeue.go b/stovepipe/core/messagequeue/messagequeue.go index e9fb2c944..c71ff6284 100644 --- a/stovepipe/core/messagequeue/messagequeue.go +++ b/stovepipe/core/messagequeue/messagequeue.go @@ -56,6 +56,12 @@ type ( // Record is the payload buildsignal publishes to the record stage once a // build reaches a terminal status: the build id to record. Record = protopb.Record + + // FixRequest identifies a confirmed issue opening and the detecting build. + FixRequest = protopb.FixRequest + + // FixSignal identifies a persisted repair whose execution and recovery need processing. + FixSignal = protopb.FixSignal ) // marshalOpts keeps the JSON field names identical to the proto field names diff --git a/stovepipe/core/messagequeue/messagequeue_test.go b/stovepipe/core/messagequeue/messagequeue_test.go index cbcc1c884..2ccefe789 100644 --- a/stovepipe/core/messagequeue/messagequeue_test.go +++ b/stovepipe/core/messagequeue/messagequeue_test.go @@ -66,6 +66,44 @@ func TestRecordRoundTrip(t *testing.T) { assert.True(t, proto.Equal(rec, got), "round-tripped Record should equal the original") } +func TestFixMessagesRoundTrip(t *testing.T) { + for _, tt := range []struct { + name string + message proto.Message + }{ + { + name: "issue request", + message: &FixRequest{ + QueueName: "repo/main", + BuildId: "build-42", TaskId: "compile", UnitId: "package", FirstObservedBuildId: "build-41", + }, + }, + { + name: "repair signal", + message: &FixSignal{QueueName: "repo/main", Id: "repair-42"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + data, err := Marshal(tt.message) + require.NoError(t, err) + got := tt.message.ProtoReflect().New().Interface() + require.NoError(t, Unmarshal(data, got)) + assert.True(t, proto.Equal(tt.message, got)) + }) + } +} + +func TestFixRequestWireFormat(t *testing.T) { + message := &FixRequest{QueueName: "repo/main", BuildId: "build-42", TaskId: "check-a", FirstObservedBuildId: "build-41"} + data, err := Marshal(message) + require.NoError(t, err) + assert.JSONEq(t, `{"queue_name":"repo/main","build_id":"build-42","task_id":"check-a","first_observed_build_id":"build-41"}`, string(data)) + + var got FixRequest + require.NoError(t, Unmarshal([]byte(`{"queue_name":"repo/main","build_id":"build-42","task_id":"check-a","first_observed_build_id":"build-41","future_field":true}`), &got)) + assert.True(t, proto.Equal(message, &got)) +} + // TestWireFormat locks the protojson encoding decision the contract relies on: // snake_case field names (UseProtoNames). func TestWireFormat(t *testing.T) { @@ -80,7 +118,7 @@ func TestWireFormat(t *testing.T) { // no topic_keys option names an unknown key. func TestTopicKeysBindEveryTopicKey(t *testing.T) { bound := map[string]int{} - for _, m := range []proto.Message{&ProcessRequest{}, &BuildRequest{}, &BuildSignal{}, &BuildAnalysis{}, &Record{}} { + for _, m := range []proto.Message{&ProcessRequest{}, &BuildRequest{}, &BuildSignal{}, &BuildAnalysis{}, &Record{}, &FixRequest{}, &FixSignal{}} { keys := TopicKeys(m) require.NotEmpty(t, keys, "message must declare a non-empty topic_keys option") for _, key := range keys { @@ -94,6 +132,8 @@ func TestTopicKeysBindEveryTopicKey(t *testing.T) { TopicKeyBuildSignal, TopicKeyBuildAnalysis, TopicKeyRecord, + TopicKeyFix, + TopicKeyFixSignal, } valid := map[string]bool{} diff --git a/stovepipe/core/messagequeue/proto/BUILD.bazel b/stovepipe/core/messagequeue/proto/BUILD.bazel index 382e14f5b..b3c1d7814 100644 --- a/stovepipe/core/messagequeue/proto/BUILD.bazel +++ b/stovepipe/core/messagequeue/proto/BUILD.bazel @@ -3,6 +3,8 @@ exports_files( "build.proto", "buildanalysis.proto", "buildsignal.proto", + "fix.proto", + "fixsignal.proto", "process.proto", "record.proto", ], diff --git a/stovepipe/core/messagequeue/proto/fix.proto b/stovepipe/core/messagequeue/proto/fix.proto new file mode 100644 index 000000000..c132fec63 --- /dev/null +++ b/stovepipe/core/messagequeue/proto/fix.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.stovepipe.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/stovepipe/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "FixRequestProto"; +option java_package = "com.uber.submitqueue.stovepipe.messagequeue"; + +// FixRequest identifies a confirmed issue opening and the build requesting repair. +// The observation and open issue must be persisted before publication. +message FixRequest { + option (uber.base.messagequeue.topic_keys) = "fix"; + + string queue_name = 1; + string build_id = 2; + string task_id = 3; + string unit_id = 4; + string first_observed_build_id = 5; +} diff --git a/stovepipe/core/messagequeue/proto/fixsignal.proto b/stovepipe/core/messagequeue/proto/fixsignal.proto new file mode 100644 index 000000000..6cb808524 --- /dev/null +++ b/stovepipe/core/messagequeue/proto/fixsignal.proto @@ -0,0 +1,35 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.stovepipe.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/stovepipe/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "FixSignalProto"; +option java_package = "com.uber.submitqueue.stovepipe.messagequeue"; + +// FixSignal references a durable repair; execution and issue recovery are loaded from storage. +// Publishing requires the repair and its issue associations to have been persisted first. +message FixSignal { + option (uber.base.messagequeue.topic_keys) = "fixsignal"; + + // id is the queue-scoped repair identifier. + string id = 1; + // queue_name identifies the validation queue containing the repair. + string queue_name = 2; +} diff --git a/stovepipe/core/messagequeue/protopb/BUILD.bazel b/stovepipe/core/messagequeue/protopb/BUILD.bazel index f1d2656fe..be9153a22 100644 --- a/stovepipe/core/messagequeue/protopb/BUILD.bazel +++ b/stovepipe/core/messagequeue/protopb/BUILD.bazel @@ -6,6 +6,8 @@ go_library( "build.pb.go", "buildanalysis.pb.go", "buildsignal.pb.go", + "fix.pb.go", + "fixsignal.pb.go", "process.pb.go", "record.pb.go", ], diff --git a/stovepipe/core/messagequeue/protopb/fix.pb.go b/stovepipe/core/messagequeue/protopb/fix.pb.go new file mode 100644 index 000000000..6b4c4ac3e --- /dev/null +++ b/stovepipe/core/messagequeue/protopb/fix.pb.go @@ -0,0 +1,179 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v5.29.3 +// source: fix.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// FixRequest identifies a confirmed issue opening and the build requesting repair. +// The observation and open issue must be persisted before publication. +type FixRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + QueueName string `protobuf:"bytes,1,opt,name=queue_name,json=queueName,proto3" json:"queue_name,omitempty"` + BuildId string `protobuf:"bytes,2,opt,name=build_id,json=buildId,proto3" json:"build_id,omitempty"` + TaskId string `protobuf:"bytes,3,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + UnitId string `protobuf:"bytes,4,opt,name=unit_id,json=unitId,proto3" json:"unit_id,omitempty"` + FirstObservedBuildId string `protobuf:"bytes,5,opt,name=first_observed_build_id,json=firstObservedBuildId,proto3" json:"first_observed_build_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FixRequest) Reset() { + *x = FixRequest{} + mi := &file_fix_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FixRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FixRequest) ProtoMessage() {} + +func (x *FixRequest) ProtoReflect() protoreflect.Message { + mi := &file_fix_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FixRequest.ProtoReflect.Descriptor instead. +func (*FixRequest) Descriptor() ([]byte, []int) { + return file_fix_proto_rawDescGZIP(), []int{0} +} + +func (x *FixRequest) GetQueueName() string { + if x != nil { + return x.QueueName + } + return "" +} + +func (x *FixRequest) GetBuildId() string { + if x != nil { + return x.BuildId + } + return "" +} + +func (x *FixRequest) GetTaskId() string { + if x != nil { + return x.TaskId + } + return "" +} + +func (x *FixRequest) GetUnitId() string { + if x != nil { + return x.UnitId + } + return "" +} + +func (x *FixRequest) GetFirstObservedBuildId() string { + if x != nil { + return x.FirstObservedBuildId + } + return "" +} + +var File_fix_proto protoreflect.FileDescriptor + +const file_fix_proto_rawDesc = "" + + "\n" + + "\tfix.proto\x12\x1buber.stovepipe.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"\xb8\x01\n" + + "\n" + + "FixRequest\x12\x1d\n" + + "\n" + + "queue_name\x18\x01 \x01(\tR\tqueueName\x12\x19\n" + + "\bbuild_id\x18\x02 \x01(\tR\abuildId\x12\x17\n" + + "\atask_id\x18\x03 \x01(\tR\x06taskId\x12\x17\n" + + "\aunit_id\x18\x04 \x01(\tR\x06unitId\x125\n" + + "\x17first_observed_build_id\x18\x05 \x01(\tR\x14firstObservedBuildId:\a\x8a\xb5\x18\x03fixB\x81\x01\n" + + "+com.uber.submitqueue.stovepipe.messagequeueB\x0fFixRequestProtoP\x01Z?github.com/uber/submitqueue/stovepipe/core/messagequeue/protopbb\x06proto3" + +var ( + file_fix_proto_rawDescOnce sync.Once + file_fix_proto_rawDescData []byte +) + +func file_fix_proto_rawDescGZIP() []byte { + file_fix_proto_rawDescOnce.Do(func() { + file_fix_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_fix_proto_rawDesc), len(file_fix_proto_rawDesc))) + }) + return file_fix_proto_rawDescData +} + +var file_fix_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_fix_proto_goTypes = []any{ + (*FixRequest)(nil), // 0: uber.stovepipe.messagequeue.FixRequest +} +var file_fix_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_fix_proto_init() } +func file_fix_proto_init() { + if File_fix_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_fix_proto_rawDesc), len(file_fix_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_fix_proto_goTypes, + DependencyIndexes: file_fix_proto_depIdxs, + MessageInfos: file_fix_proto_msgTypes, + }.Build() + File_fix_proto = out.File + file_fix_proto_goTypes = nil + file_fix_proto_depIdxs = nil +} diff --git a/stovepipe/core/messagequeue/protopb/fixsignal.pb.go b/stovepipe/core/messagequeue/protopb/fixsignal.pb.go new file mode 100644 index 000000000..f06d3b46f --- /dev/null +++ b/stovepipe/core/messagequeue/protopb/fixsignal.pb.go @@ -0,0 +1,153 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v5.29.3 +// source: fixsignal.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// FixSignal references a durable repair; execution and issue recovery are loaded from storage. +// Publishing requires the repair and its issue associations to have been persisted first. +type FixSignal struct { + state protoimpl.MessageState `protogen:"open.v1"` + // id is the queue-scoped repair identifier. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // queue_name identifies the validation queue containing the repair. + QueueName string `protobuf:"bytes,2,opt,name=queue_name,json=queueName,proto3" json:"queue_name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FixSignal) Reset() { + *x = FixSignal{} + mi := &file_fixsignal_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FixSignal) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FixSignal) ProtoMessage() {} + +func (x *FixSignal) ProtoReflect() protoreflect.Message { + mi := &file_fixsignal_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FixSignal.ProtoReflect.Descriptor instead. +func (*FixSignal) Descriptor() ([]byte, []int) { + return file_fixsignal_proto_rawDescGZIP(), []int{0} +} + +func (x *FixSignal) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *FixSignal) GetQueueName() string { + if x != nil { + return x.QueueName + } + return "" +} + +var File_fixsignal_proto protoreflect.FileDescriptor + +const file_fixsignal_proto_rawDesc = "" + + "\n" + + "\x0ffixsignal.proto\x12\x1buber.stovepipe.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"I\n" + + "\tFixSignal\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x1d\n" + + "\n" + + "queue_name\x18\x02 \x01(\tR\tqueueName:\r\x8a\xb5\x18\tfixsignalB\x80\x01\n" + + "+com.uber.submitqueue.stovepipe.messagequeueB\x0eFixSignalProtoP\x01Z?github.com/uber/submitqueue/stovepipe/core/messagequeue/protopbb\x06proto3" + +var ( + file_fixsignal_proto_rawDescOnce sync.Once + file_fixsignal_proto_rawDescData []byte +) + +func file_fixsignal_proto_rawDescGZIP() []byte { + file_fixsignal_proto_rawDescOnce.Do(func() { + file_fixsignal_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_fixsignal_proto_rawDesc), len(file_fixsignal_proto_rawDesc))) + }) + return file_fixsignal_proto_rawDescData +} + +var file_fixsignal_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_fixsignal_proto_goTypes = []any{ + (*FixSignal)(nil), // 0: uber.stovepipe.messagequeue.FixSignal +} +var file_fixsignal_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_fixsignal_proto_init() } +func file_fixsignal_proto_init() { + if File_fixsignal_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_fixsignal_proto_rawDesc), len(file_fixsignal_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_fixsignal_proto_goTypes, + DependencyIndexes: file_fixsignal_proto_depIdxs, + MessageInfos: file_fixsignal_proto_msgTypes, + }.Build() + File_fixsignal_proto = out.File + file_fixsignal_proto_goTypes = nil + file_fixsignal_proto_depIdxs = nil +} diff --git a/stovepipe/core/messagequeue/topics.go b/stovepipe/core/messagequeue/topics.go index 09da535a6..d4487b08d 100644 --- a/stovepipe/core/messagequeue/topics.go +++ b/stovepipe/core/messagequeue/topics.go @@ -51,4 +51,12 @@ const ( // terminal status; non-terminal polls never publish here. Partitioned by // request id. TopicKeyRecord TopicKey = "record" + + // TopicKeyFix carries confirmed issue references for repair selection. + // Deliveries are partitioned by detecting build ID. + TopicKeyFix TopicKey = "fix" + + // TopicKeyFixSignal carries repair IDs for execution and recovery reconciliation. + // Deliveries are partitioned by fix ID and held between observations while work is active. + TopicKeyFixSignal TopicKey = "fixsignal" ) diff --git a/stovepipe/entity/BUILD.bazel b/stovepipe/entity/BUILD.bazel index a8ea6a659..ebb10478a 100644 --- a/stovepipe/entity/BUILD.bazel +++ b/stovepipe/entity/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "build.go", "build_issue.go", + "fix.go", "ingest.go", "list.go", "open_build_issue.go", diff --git a/stovepipe/entity/build.go b/stovepipe/entity/build.go index e278bf904..09079e3b9 100644 --- a/stovepipe/entity/build.go +++ b/stovepipe/entity/build.go @@ -53,10 +53,8 @@ func (s BuildStatus) IsTerminal() bool { return s == BuildStatusSucceeded || s == BuildStatusFailed || s == BuildStatusCancelled } -// Build represents a single build triggered for a Request's commit. All -// fields except Status and Version are immutable after creation — build is -// the sole creator (via BuildStore.Create), and buildsignal is the sole -// writer of Status/Version afterward. +// Build represents one validation execution. ID and RequestID are immutable; +// Status, FirstFixID, and Version form its mutable snapshot. type Build struct { // ID is the build's own key: the runner-assigned id returned by // Trigger (e.g. a Buildkite build number). Opaque; never parsed or @@ -67,6 +65,9 @@ type Build struct { RequestID string `json:"request_id"` // Status is the build's lifecycle state. Status BuildStatus `json:"status"` + // FirstFixID is the first selected repair attempt; empty before selection. + // Selection is write-once and survives repair completion. + FirstFixID string `json:"first_fix_id"` // Version is used for optimistic locking. Versioning starts at 1 and // is incremented for each change to the object. Version int32 `json:"version"` diff --git a/stovepipe/entity/fix.go b/stovepipe/entity/fix.go new file mode 100644 index 000000000..c148db230 --- /dev/null +++ b/stovepipe/entity/fix.go @@ -0,0 +1,65 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package entity + +// FixState is a repair's durable lifecycle, independent of check health. +type FixState string + +const ( + FixStateUnknown FixState = "" + FixQueued FixState = "queued" + FixSubmitting FixState = "submitting" + FixRunning FixState = "running" + FixAwaitingVerification FixState = "awaiting_verification" + FixCompleted FixState = "completed" + FixFailed FixState = "failed" + FixSkipped FixState = "skipped" +) + +func (s FixState) Terminal() bool { return s == FixCompleted || s == FixFailed || s == FixSkipped } + +// BuildIssueRef identifies one opening of a validation issue within a queue. +type BuildIssueRef struct { + // TaskID is the logical validation task, stable across builds and retries. + TaskID string `json:"task_id"` + // UnitID is the failed unit; empty for a task-wide issue. + UnitID string `json:"unit_id"` + // FirstObservedBuildID is the immutable identity of this opening. + FirstObservedBuildID string `json:"first_observed_build_id"` +} + +// Fix is a versioned repair attempt covering one or more issue openings. +type Fix struct { + // ID is the queue-scoped attempt identifier, stable across delivery retries. + ID string `json:"id"` + // Queue is the owning validation queue. + Queue string `json:"queue"` + // BuildID is the build whose confirmed failure initiated this attempt. + BuildID string `json:"build_id"` + // State is the repair lifecycle. + State FixState `json:"state"` + // Issues contains issue-opening references; current assignment is stored on the open issue. + Issues []BuildIssueRef `json:"issues"` + // ExecutionRef is the optional provider reference returned by submission, empty until known. + ExecutionRef string `json:"execution_ref"` + // Metadata contains optional change references and provider context; nil and empty maps are equivalent. + Metadata map[string]string `json:"metadata"` + // Version is the optimistic storage version. + Version int32 `json:"version"` + // CreatedAt is the UTC creation time in milliseconds since the epoch. + CreatedAt int64 `json:"created_at"` + // UpdatedAt is the UTC update time in milliseconds since the epoch. + UpdatedAt int64 `json:"updated_at"` +} diff --git a/stovepipe/entity/open_build_issue.go b/stovepipe/entity/open_build_issue.go index b046eef87..56e1b08a2 100644 --- a/stovepipe/entity/open_build_issue.go +++ b/stovepipe/entity/open_build_issue.go @@ -26,6 +26,8 @@ type OpenBuildIssue struct { FirstObservedBuildID string `json:"first_observed_build_id"` // LastObservedBuildID identifies the build of the latest persisted observation. LastObservedBuildID string `json:"last_observed_build_id"` + // FixID is the currently assigned repair attempt within this queue; empty when unassigned. + FixID string `json:"fix_id"` // Version is the positive optimistic-lock version of this opening. Version int32 `json:"version"` } diff --git a/stovepipe/entity/queue.go b/stovepipe/entity/queue.go index 9ca1c8fcc..2c19e070a 100644 --- a/stovepipe/entity/queue.go +++ b/stovepipe/entity/queue.go @@ -43,6 +43,9 @@ type Queue struct { // Empty until the first request is created. Coalescing compares IDs via CompareRequestID. LatestRequestID string `json:"latest_request_id"` + // ActiveFixIDs contains unique IDs of active repairs within the queue; nil and empty are equivalent. + ActiveFixIDs []string `json:"active_fix_ids"` + // Enablement is the applied enabled/disabled state; empty means unregistered. Enablement QueueEnablement `json:"enablement"` diff --git a/stovepipe/extension/fixer/BUILD.bazel b/stovepipe/extension/fixer/BUILD.bazel new file mode 100644 index 000000000..a6af87006 --- /dev/null +++ b/stovepipe/extension/fixer/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["fixer.go"], + importpath = "github.com/uber/submitqueue/stovepipe/extension/fixer", + visibility = ["//visibility:public"], + deps = ["//stovepipe/entity:go_default_library"], +) diff --git a/stovepipe/extension/fixer/fixer.go b/stovepipe/extension/fixer/fixer.go new file mode 100644 index 000000000..78aee5f59 --- /dev/null +++ b/stovepipe/extension/fixer/fixer.go @@ -0,0 +1,99 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fixer defines repair evaluation and asynchronous execution. +package fixer + +//go:generate mockgen -source=fixer.go -destination=mock/fixer_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/stovepipe/entity" +) + +// Action is an evaluator's recommendation for an issue opening. +type Action string + +const ( + ActionUnknown Action = "" + Start Action = "start" + Attach Action = "attach" + Defer Action = "defer" +) + +// RepairEvaluator recommends how to handle an unassigned issue opening. +// Implementations resolve diagnostics and investigation progress through injected dependencies. +type RepairEvaluator interface { + Evaluate(context.Context, EvaluationRequest) (Decision, error) +} + +// EvaluationRequest identifies an issue opening and the active repairs considered for attachment. +type EvaluationRequest struct { + // Queue is the validation queue containing the episode and candidate repairs. + Queue string + // BuildID is the detecting build for the repair decision. + BuildID string + // Issue identifies the persisted issue opening. + Issue entity.BuildIssueRef + // CandidateFixIDs contains active repair IDs from the queue snapshot. + CandidateFixIDs []string +} + +// Decision is a recommendation; durable assignment requires a conditional state update. +type Decision struct { + // Action is Start, Attach, or Defer. + Action Action + // FixID is a candidate repair ID required for Attach and empty for other actions. + FixID string + // Detail is an optional explanation of the recommendation. + Detail string +} + +// EvaluatorFactory resolves an evaluator for a validation queue. +type EvaluatorFactory interface { + For(Config) (RepairEvaluator, error) +} + +// RepairExecutor submits repairs and observes their execution and change lifecycle. +// Submit returns a nonempty execution ID on success; its ID is ignored on error. +// An error may follow acceptance, so retrying can start duplicate work. +// Observe also correlates by Queue and ID when ExecutionRef is empty after a lost reply. +// Every delivery retry uses the same fix identity; a deliberate new attempt gets a new ID. +type RepairExecutor interface { + Submit(context.Context, entity.Fix) (string, error) + Observe(context.Context, entity.Fix) (Snapshot, error) +} + +// Config identifies the queue whose repair extension is being resolved. +type Config struct { + // QueueName is the validation queue. + QueueName string +} + +// Factory resolves configured repair executors. +type Factory interface { + For(Config) (RepairExecutor, error) +} + +// Snapshot is provider evidence about a repair. +type Snapshot struct { + // Found reports whether execution is visible for this fix identity; false leaves execution unresolved. + Found bool + // State is running, awaiting_verification, or failed. + // Completion is established separately by trusted recovery evidence. + State entity.FixState + // Metadata contains optional change references, provider details, and diagnostics. + Metadata map[string]string +} diff --git a/stovepipe/extension/fixer/mock/BUILD.bazel b/stovepipe/extension/fixer/mock/BUILD.bazel new file mode 100644 index 000000000..5727675f3 --- /dev/null +++ b/stovepipe/extension/fixer/mock/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["fixer_mock.go"], + importpath = "github.com/uber/submitqueue/stovepipe/extension/fixer/mock", + visibility = ["//visibility:public"], + deps = [ + "//stovepipe/entity:go_default_library", + "//stovepipe/extension/fixer:go_default_library", + "@org_uber_go_mock//gomock:go_default_library", + ], +) diff --git a/stovepipe/extension/fixer/mock/fixer_mock.go b/stovepipe/extension/fixer/mock/fixer_mock.go new file mode 100644 index 000000000..42feeaea2 --- /dev/null +++ b/stovepipe/extension/fixer/mock/fixer_mock.go @@ -0,0 +1,190 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: fixer.go +// +// Generated by this command: +// +// mockgen -source=fixer.go -destination=mock/fixer_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/stovepipe/entity" + fixer "github.com/uber/submitqueue/stovepipe/extension/fixer" + gomock "go.uber.org/mock/gomock" +) + +// MockRepairEvaluator is a mock of RepairEvaluator interface. +type MockRepairEvaluator struct { + ctrl *gomock.Controller + recorder *MockRepairEvaluatorMockRecorder + isgomock struct{} +} + +// MockRepairEvaluatorMockRecorder is the mock recorder for MockRepairEvaluator. +type MockRepairEvaluatorMockRecorder struct { + mock *MockRepairEvaluator +} + +// NewMockRepairEvaluator creates a new mock instance. +func NewMockRepairEvaluator(ctrl *gomock.Controller) *MockRepairEvaluator { + mock := &MockRepairEvaluator{ctrl: ctrl} + mock.recorder = &MockRepairEvaluatorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepairEvaluator) EXPECT() *MockRepairEvaluatorMockRecorder { + return m.recorder +} + +// Evaluate mocks base method. +func (m *MockRepairEvaluator) Evaluate(arg0 context.Context, arg1 fixer.EvaluationRequest) (fixer.Decision, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Evaluate", arg0, arg1) + ret0, _ := ret[0].(fixer.Decision) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Evaluate indicates an expected call of Evaluate. +func (mr *MockRepairEvaluatorMockRecorder) Evaluate(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Evaluate", reflect.TypeOf((*MockRepairEvaluator)(nil).Evaluate), arg0, arg1) +} + +// MockEvaluatorFactory is a mock of EvaluatorFactory interface. +type MockEvaluatorFactory struct { + ctrl *gomock.Controller + recorder *MockEvaluatorFactoryMockRecorder + isgomock struct{} +} + +// MockEvaluatorFactoryMockRecorder is the mock recorder for MockEvaluatorFactory. +type MockEvaluatorFactoryMockRecorder struct { + mock *MockEvaluatorFactory +} + +// NewMockEvaluatorFactory creates a new mock instance. +func NewMockEvaluatorFactory(ctrl *gomock.Controller) *MockEvaluatorFactory { + mock := &MockEvaluatorFactory{ctrl: ctrl} + mock.recorder = &MockEvaluatorFactoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockEvaluatorFactory) EXPECT() *MockEvaluatorFactoryMockRecorder { + return m.recorder +} + +// For mocks base method. +func (m *MockEvaluatorFactory) For(arg0 fixer.Config) (fixer.RepairEvaluator, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "For", arg0) + ret0, _ := ret[0].(fixer.RepairEvaluator) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// For indicates an expected call of For. +func (mr *MockEvaluatorFactoryMockRecorder) For(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "For", reflect.TypeOf((*MockEvaluatorFactory)(nil).For), arg0) +} + +// MockRepairExecutor is a mock of RepairExecutor interface. +type MockRepairExecutor struct { + ctrl *gomock.Controller + recorder *MockRepairExecutorMockRecorder + isgomock struct{} +} + +// MockRepairExecutorMockRecorder is the mock recorder for MockRepairExecutor. +type MockRepairExecutorMockRecorder struct { + mock *MockRepairExecutor +} + +// NewMockRepairExecutor creates a new mock instance. +func NewMockRepairExecutor(ctrl *gomock.Controller) *MockRepairExecutor { + mock := &MockRepairExecutor{ctrl: ctrl} + mock.recorder = &MockRepairExecutorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepairExecutor) EXPECT() *MockRepairExecutorMockRecorder { + return m.recorder +} + +// Observe mocks base method. +func (m *MockRepairExecutor) Observe(arg0 context.Context, arg1 entity.Fix) (fixer.Snapshot, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Observe", arg0, arg1) + ret0, _ := ret[0].(fixer.Snapshot) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Observe indicates an expected call of Observe. +func (mr *MockRepairExecutorMockRecorder) Observe(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Observe", reflect.TypeOf((*MockRepairExecutor)(nil).Observe), arg0, arg1) +} + +// Submit mocks base method. +func (m *MockRepairExecutor) Submit(arg0 context.Context, arg1 entity.Fix) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Submit", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Submit indicates an expected call of Submit. +func (mr *MockRepairExecutorMockRecorder) Submit(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockRepairExecutor)(nil).Submit), arg0, arg1) +} + +// MockFactory is a mock of Factory interface. +type MockFactory struct { + ctrl *gomock.Controller + recorder *MockFactoryMockRecorder + isgomock struct{} +} + +// MockFactoryMockRecorder is the mock recorder for MockFactory. +type MockFactoryMockRecorder struct { + mock *MockFactory +} + +// NewMockFactory creates a new mock instance. +func NewMockFactory(ctrl *gomock.Controller) *MockFactory { + mock := &MockFactory{ctrl: ctrl} + mock.recorder = &MockFactoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFactory) EXPECT() *MockFactoryMockRecorder { + return m.recorder +} + +// For mocks base method. +func (m *MockFactory) For(arg0 fixer.Config) (fixer.RepairExecutor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "For", arg0) + ret0, _ := ret[0].(fixer.RepairExecutor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// For indicates an expected call of For. +func (mr *MockFactoryMockRecorder) For(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "For", reflect.TypeOf((*MockFactory)(nil).For), arg0) +} diff --git a/stovepipe/extension/storage/BUILD.bazel b/stovepipe/extension/storage/BUILD.bazel index 616d40a81..c7cfcb456 100644 --- a/stovepipe/extension/storage/BUILD.bazel +++ b/stovepipe/extension/storage/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "build_issue_store.go", "build_store.go", + "fix_store.go", "open_build_issue_store.go", "queue_log_store.go", "queue_store.go", diff --git a/stovepipe/extension/storage/README.md b/stovepipe/extension/storage/README.md index 4bb191468..5f0e6dc85 100644 --- a/stovepipe/extension/storage/README.md +++ b/stovepipe/extension/storage/README.md @@ -43,3 +43,12 @@ Updates and deletion check both the first-observed build and the caller's expect MySQL identifiers use the same binary byte limits as per-build observations. Build IDs are opaque context, not ordering tokens. Optimistic locking protects row changes but does not decide which build is newer. The store does not choose resolution, enumerate issues, write lifecycle history, or reconcile analysis responses. Resolution belongs to a later controller: a successful, authoritative final analysis can establish that a previously failing condition is absent. Build terminality alone is not evidence that analysis succeeded; an analysis failure or an unfinished poll must not close issues. The reference host defaults to one admitted validation per queue, but final analysis can lag behind a later build because polling and analysis run independently. Source-order reconciliation is separate from these storage primitives. +## Repair contracts + +`FixStore` defines primary-key operations on queue-scoped repair attempts. A fix retains its initiating BuildID, issue-opening references, lifecycle, provider execution reference, optional metadata, timestamps, and snapshot version. Delivery retries reuse the same fix ID; deliberate new repair attempts get new IDs. + +`Queue.ActiveFixIDs` enumerates active repairs under `Queue.Version`. `Build.FirstFixID` records the first selected repair for that build under `Build.Version`, and remains set after completion. This initial policy permits one automatic repair per build. A future evaluator can admit additional repairs. + +`OpenBuildIssue.FixID` is the authoritative assignment for its opening. Updates replace last-observed build and assignment together under the opening/version guards. Analysis preserves assignments; fixer preserves observation fields. Resolution history and recovery evidence remain owned by build analysis; deleting a current row alone supplies no validating build reference. + +The [repair RFC](../../../doc/rfc/stovepipe/fixer.md) defines persistence, publication, and observation ordering. Persistence implementations follow in the storage layer. diff --git a/stovepipe/extension/storage/build_store.go b/stovepipe/extension/storage/build_store.go index a6d188d0e..86ee8e2bc 100644 --- a/stovepipe/extension/storage/build_store.go +++ b/stovepipe/extension/storage/build_store.go @@ -23,9 +23,9 @@ import ( ) // BuildStore persists builds, keyed by build ID (the runner-assigned id minted at Trigger). -// build is the sole creator of a row; buildsignal is the sole writer of Status/Version -// afterward. No reverse index from Request to its builds is needed — buildsignal and record -// reach a build by the id carried in their messages. +// build creates rows, buildsignal updates Status, and fix selects FirstFixID. +// Writers preserve the other fields under Version. buildsignal and record +// reach a build by the ID carried in their messages. type BuildStore interface { // Create persists a new build. The build must have a unique ID already assigned. // Returns ErrAlreadyExists if a build with the same ID already exists. diff --git a/stovepipe/extension/storage/fix_store.go b/stovepipe/extension/storage/fix_store.go new file mode 100644 index 000000000..be25421e0 --- /dev/null +++ b/stovepipe/extension/storage/fix_store.go @@ -0,0 +1,32 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +//go:generate mockgen -source=fix_store.go -destination=mock/fix_store_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/stovepipe/entity" +) + +// FixStore persists queue-scoped versioned logical repairs by ID. +// Create/Get use ErrAlreadyExists/ErrNotFound; Update uses ErrVersionMismatch. +// Callers supply both the guard version and replacement version. +type FixStore interface { + Create(context.Context, entity.Fix) error + Get(context.Context, string) (entity.Fix, error) + Update(context.Context, entity.Fix, int32, int32) error +} diff --git a/stovepipe/extension/storage/mock/BUILD.bazel b/stovepipe/extension/storage/mock/BUILD.bazel index 72d23595d..da142ccb6 100644 --- a/stovepipe/extension/storage/mock/BUILD.bazel +++ b/stovepipe/extension/storage/mock/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "build_issue_store_mock.go", "build_store_mock.go", + "fix_store_mock.go", "open_build_issue_store_mock.go", "queue_log_store_mock.go", "queue_store_mock.go", diff --git a/stovepipe/extension/storage/mock/fix_store_mock.go b/stovepipe/extension/storage/mock/fix_store_mock.go new file mode 100644 index 000000000..6b6919c4f --- /dev/null +++ b/stovepipe/extension/storage/mock/fix_store_mock.go @@ -0,0 +1,85 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: fix_store.go +// +// Generated by this command: +// +// mockgen -source=fix_store.go -destination=mock/fix_store_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/stovepipe/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockFixStore is a mock of FixStore interface. +type MockFixStore struct { + ctrl *gomock.Controller + recorder *MockFixStoreMockRecorder + isgomock struct{} +} + +// MockFixStoreMockRecorder is the mock recorder for MockFixStore. +type MockFixStoreMockRecorder struct { + mock *MockFixStore +} + +// NewMockFixStore creates a new mock instance. +func NewMockFixStore(ctrl *gomock.Controller) *MockFixStore { + mock := &MockFixStore{ctrl: ctrl} + mock.recorder = &MockFixStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFixStore) EXPECT() *MockFixStoreMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockFixStore) Create(arg0 context.Context, arg1 entity.Fix) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockFixStoreMockRecorder) Create(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockFixStore)(nil).Create), arg0, arg1) +} + +// Get mocks base method. +func (m *MockFixStore) Get(arg0 context.Context, arg1 string) (entity.Fix, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].(entity.Fix) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockFixStoreMockRecorder) Get(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockFixStore)(nil).Get), arg0, arg1) +} + +// Update mocks base method. +func (m *MockFixStore) Update(arg0 context.Context, arg1 entity.Fix, arg2, arg3 int32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockFixStoreMockRecorder) Update(arg0, arg1, arg2, arg3 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockFixStore)(nil).Update), arg0, arg1, arg2, arg3) +} diff --git a/stovepipe/extension/storage/open_build_issue_store.go b/stovepipe/extension/storage/open_build_issue_store.go index bf4893f70..dbe20d91a 100644 --- a/stovepipe/extension/storage/open_build_issue_store.go +++ b/stovepipe/extension/storage/open_build_issue_store.go @@ -32,7 +32,7 @@ type OpenBuildIssueStore interface { // Get returns ErrNotFound when the task/unit is not currently open. Get(ctx context.Context, taskID, unitID string) (entity.OpenBuildIssue, error) - // Update changes the last-observed build and writes newVersion when both the + // Update replaces the last-observed build and FixID, writing newVersion when the // first-observed build and oldVersion match. Otherwise it returns ErrVersionMismatch. // Callers own version arithmetic; issue.Version is not used as a write value. Update(ctx context.Context, issue entity.OpenBuildIssue, oldVersion, newVersion int32) error diff --git a/tool/proto/BUILD.bazel b/tool/proto/BUILD.bazel index 62be9e389..b48585bad 100644 --- a/tool/proto/BUILD.bazel +++ b/tool/proto/BUILD.bazel @@ -105,6 +105,8 @@ go_proto_generated_files( "//stovepipe/core/messagequeue/proto:build.proto", "//stovepipe/core/messagequeue/proto:buildanalysis.proto", "//stovepipe/core/messagequeue/proto:buildsignal.proto", + "//stovepipe/core/messagequeue/proto:fix.proto", + "//stovepipe/core/messagequeue/proto:fixsignal.proto", "//stovepipe/core/messagequeue/proto:process.proto", "//stovepipe/core/messagequeue/proto:record.proto", ],