diff --git a/Makefile b/Makefile index 24597829b..de514a8fd 100644 --- a/Makefile +++ b/Makefile @@ -629,6 +629,7 @@ GO_GENERATE_PACKAGES := \ ./stovepipe/core/requestlog/... \ ./stovepipe/extension/buildissue/... \ ./stovepipe/extension/buildrunner/... \ + ./stovepipe/extension/fixer/... \ ./stovepipe/extension/projectstatus/... \ ./stovepipe/extension/queueconfig/... \ ./stovepipe/extension/sourcecontrol/... \ diff --git a/doc/rfc/stovepipe/fixer.md b/doc/rfc/stovepipe/fixer.md new file mode 100644 index 000000000..0cbb115fa --- /dev/null +++ b/doc/rfc/stovepipe/fixer.md @@ -0,0 +1,45 @@ +# First-failure repair submission + +Build analysis records confirmed failures. The `fix` queue step selects the first eligible failed unit for each build and submits one repair through a Go extension. Submission can happen while the build is running. See the [workflow](workflow.md#workflow). + +## 1. Publish a confirmed failure + +[`buildanalysis.Controller.Process`](../../../stovepipe/controller/buildanalysis/buildanalysis.go) obtains confirmed [`BuildIssue`](../../../stovepipe/entity/build_issue.go) observations from [`buildissue.Detector`](../../../stovepipe/extension/buildissue/buildissue.go). The detector owns provider confirmation and retry rules. + +The build-analysis integration persists the per-build observation through [`BuildIssueStore`](../../../stovepipe/extension/storage/build_issue_store.go) and reconciles its [`OpenIssue`](../../../stovepipe/entity/open_issue.go). Each continuous opening retains its immutable `FirstSeenBuildID`. A recurrence after recovery has a new opening build. Detecting builds and opening builds are separate identities. + +Publish [`FixRequest`](../../../stovepipe/core/messagequeue/proto/fix.proto) with the queue, detecting BuildID, TaskID, UnitID, and FirstSeenBuildID. Partition by detecting BuildID. Replay publication after duplicate observation writes so interrupted handoffs can finish. + +## 2. Select one issue per build + +[`fix.Controller.Process`](../../../stovepipe/controller/fix/fix.go) reloads the Build and Request and checks queue identity. For a new attempt, it verifies the persisted failure and current issue opening. + +1. Create a queued [`Fix`](../../../stovepipe/entity/fix.go) using the detecting BuildID as its queue-scoped ID. Create-if-absent selects the first eligible request processed, including when deliveries race. +2. If that key already exists, reload its saved selection. Preparation retries use the saved issue, even when another target caused the delivery. +3. Before dispatch, recheck the selected opening. If it recovered or reopened, mark the queued attempt skipped and retain the build's selection. + +Build analysis continues collecting all failures. Later deliveries reconcile the saved submission and its history. Each build has its own initial attempt; cross-build deduplication and decisions about additional root causes are follow-up work. + +## 3. Submit once + +[`RepairExecutor.Submit`](../../../stovepipe/extension/fixer/fixer.go) accepts `SubmitRequest{BuildID, Issue}` and returns a nonempty opaque provider reference or an error. The executor is bound to its queue by its factory. Resolve it before claiming dispatch. + +Conditionally persist `FixSubmitting` before calling Submit. Only the delivery whose conditional write succeeds calls the executor. Redelivery after that marker reconciles history and finishes without another trigger. + +On acceptance, persist `FixSubmitted` and `ExecutionRef`. On an error or empty reference, persist `FixSubmissionUnknown` and the diagnostic in metadata. A trigger error can follow acceptance, so it leaves execution unknown. A failed result write leaves the submitting marker. + +This is at-most-once dispatch. A crash between the durable marker and the outbound call can leave the repair unlaunched. A lost response leaves acceptance unknown. These cases require explicit follow-up. Execution monitoring can be added separately. + +## 4. Record request history and provenance + +Record `fix_submission_started` for the durable dispatch claim and `fix_submission_recorded` for a retained accepted or uncertain trigger result. Stable occurrence IDs derive from the fix ID. Replay reconstructs missing history from the saved Fix, including when a later target supplies the delivery. History failures never repeat submission. An attempt left submitting has only a started event. + +[`FixStore`](../../../stovepipe/extension/storage/fix_store.go) adds one table with primary-key create, get, and conditional update. It retains the detecting BuildID, issue-opening references, submission state, optional provider reference and metadata, timestamps, and version. Build and OpenIssue keep their existing schemas. + +Navigate `Fix.BuildID → Build.RequestID → Request.URI` for the initiating request and change. `Fix.Issues` retains opening identities after recovery. Build analysis owns failure and recovery evidence. + +## Integration + +[`core/fixrequest.Publish`](../../../stovepipe/core/fixrequest/publish.go) is the handoff after observation and positive open-issue persistence. The build-analysis owner supplies reconciliation and calls the helper, including on replay. The reference host retains its no-op detector until that integration lands. + +The initial flow ends after submission and request-history recording. Execution monitoring, recovery correlation, and decisions about related failures can build on these records later. diff --git a/doc/rfc/stovepipe/workflow.md b/doc/rfc/stovepipe/workflow.md index ecc3e9ae4..b8b53f4f9 100644 --- a/doc/rfc/stovepipe/workflow.md +++ b/doc/rfc/stovepipe/workflow.md @@ -169,6 +169,8 @@ Per-stage design detail lives under `steps/` so this doc stays a pipeline overvi - [buildsignal.md](steps/buildsignal.md) — the poll loop: hold-based re-poll cadence, per-build partitioning, and the fail-closed handoff to record - [record.md](steps/record.md) — turning a terminal build outcome into an immutable validation fact, monotonic last-green advancement and ref promotion, and the hook event announcing the outcome. Project analysis already runs inline through `projectstatus.Resolver` and may later use results calculated by build analysis +For repair decisions, execution, and recovery, see the [repair RFC](fixer.md). + ## Dedup, idempotency, and history rewrites Ingestion is idempotent on `(Queue, head URI)`, so duplicate poller reports — and any future webhook producer racing the poller — converge on one Request. The pipeline persists the Queue's last-green URI and per-URI greenness, so it must tolerate a **history rewrite**: when `SourceControl` reports that the last-green URI is no longer an ancestor of the current head, `process` falls back to a full-monorepo build rather than trusting a baseline that no longer exists on the branch. The system converges to a correct greenness rather than wedging on a stale pointer. diff --git a/stovepipe/core/messagequeue/README.md b/stovepipe/core/messagequeue/README.md index 39d8df7a8..fe6223ef5 100644 --- a/stovepipe/core/messagequeue/README.md +++ b/stovepipe/core/messagequeue/README.md @@ -11,3 +11,9 @@ Payloads are defined in proto3 (`proto/`, generated into `protopb/`) and seriali - **buildsignal** (`TopicKeyBuildSignal`, `BuildSignal`) — build publishes the build id here after triggering; buildsignal holds the delivery between polls until the build reaches a terminal status. Partitioned by build id, so each build's poll loop is an independent partition. - **buildanalysis** (`TopicKeyBuildAnalysis`, `BuildAnalysis`) — identifies a persisted build for issue detection, partitioned by build id. Build publishes a start notice and buildsignal publishes a completion notice with a distinct message id. See [build analysis](../../controller/buildanalysis/README.md) for polling and persistence behavior. - **record** (`TopicKeyRecord`, `Record`) — buildsignal publishes the request id after persisting the terminal outcome; record reloads the request and records its validation fact. Partitioned by request id. + +## Repair stage contract + +**fix** (`TopicKeyFix`, [`FixRequest`](proto/fix.proto)) identifies a persisted observation and issue opening using queue, detecting build, task, unit, and first-seen build. Partition by detecting build ID. The `consumer.Controller` selects the first eligible failed unit by creating a Fix keyed by the detecting BuildID, persists a submitting marker, and calls `RepairExecutor.Submit` once. Later deliveries reconcile the selected submission and its request history. + +Persist observations and their current opening before publishing a request. Replay publication after duplicate observation writes so a crash between persistence and handoff can recover. Redelivery can complete preparation, but a durable submitting marker suppresses any further outbound call. Execution monitoring and verified recovery are follow-up work. diff --git a/stovepipe/core/messagequeue/messagequeue.go b/stovepipe/core/messagequeue/messagequeue.go index e9fb2c944..6d0c57235 100644 --- a/stovepipe/core/messagequeue/messagequeue.go +++ b/stovepipe/core/messagequeue/messagequeue.go @@ -56,6 +56,9 @@ type ( // Record is the payload buildsignal publishes to the record stage once a // build reaches a terminal status: the build id to record. Record = protopb.Record + + // FixRequest identifies a confirmed issue opening and the detecting build. + FixRequest = protopb.FixRequest ) // marshalOpts keeps the JSON field names identical to the proto field names diff --git a/stovepipe/core/messagequeue/messagequeue_test.go b/stovepipe/core/messagequeue/messagequeue_test.go index cbcc1c884..02a1232b0 100644 --- a/stovepipe/core/messagequeue/messagequeue_test.go +++ b/stovepipe/core/messagequeue/messagequeue_test.go @@ -66,6 +66,40 @@ func TestRecordRoundTrip(t *testing.T) { assert.True(t, proto.Equal(rec, got), "round-tripped Record should equal the original") } +func TestFixMessagesRoundTrip(t *testing.T) { + for _, tt := range []struct { + name string + message proto.Message + }{ + { + name: "issue request", + message: &FixRequest{ + QueueName: "repo/main", + BuildId: "build-42", TaskId: "compile", UnitId: "package", FirstSeenBuildId: "build-41", + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + data, err := Marshal(tt.message) + require.NoError(t, err) + got := tt.message.ProtoReflect().New().Interface() + require.NoError(t, Unmarshal(data, got)) + assert.True(t, proto.Equal(tt.message, got)) + }) + } +} + +func TestFixRequestWireFormat(t *testing.T) { + message := &FixRequest{QueueName: "repo/main", BuildId: "build-42", TaskId: "check-a", FirstSeenBuildId: "build-41"} + data, err := Marshal(message) + require.NoError(t, err) + assert.JSONEq(t, `{"queue_name":"repo/main","build_id":"build-42","task_id":"check-a","first_seen_build_id":"build-41"}`, string(data)) + + var got FixRequest + require.NoError(t, Unmarshal([]byte(`{"queue_name":"repo/main","build_id":"build-42","task_id":"check-a","first_seen_build_id":"build-41","future_field":true}`), &got)) + assert.True(t, proto.Equal(message, &got)) +} + // TestWireFormat locks the protojson encoding decision the contract relies on: // snake_case field names (UseProtoNames). func TestWireFormat(t *testing.T) { @@ -80,7 +114,7 @@ func TestWireFormat(t *testing.T) { // no topic_keys option names an unknown key. func TestTopicKeysBindEveryTopicKey(t *testing.T) { bound := map[string]int{} - for _, m := range []proto.Message{&ProcessRequest{}, &BuildRequest{}, &BuildSignal{}, &BuildAnalysis{}, &Record{}} { + for _, m := range []proto.Message{&ProcessRequest{}, &BuildRequest{}, &BuildSignal{}, &BuildAnalysis{}, &Record{}, &FixRequest{}} { keys := TopicKeys(m) require.NotEmpty(t, keys, "message must declare a non-empty topic_keys option") for _, key := range keys { @@ -94,6 +128,7 @@ func TestTopicKeysBindEveryTopicKey(t *testing.T) { TopicKeyBuildSignal, TopicKeyBuildAnalysis, TopicKeyRecord, + TopicKeyFix, } valid := map[string]bool{} diff --git a/stovepipe/core/messagequeue/proto/BUILD.bazel b/stovepipe/core/messagequeue/proto/BUILD.bazel index 382e14f5b..3a6efba1e 100644 --- a/stovepipe/core/messagequeue/proto/BUILD.bazel +++ b/stovepipe/core/messagequeue/proto/BUILD.bazel @@ -3,6 +3,7 @@ exports_files( "build.proto", "buildanalysis.proto", "buildsignal.proto", + "fix.proto", "process.proto", "record.proto", ], diff --git a/stovepipe/core/messagequeue/proto/fix.proto b/stovepipe/core/messagequeue/proto/fix.proto new file mode 100644 index 000000000..8dc155d80 --- /dev/null +++ b/stovepipe/core/messagequeue/proto/fix.proto @@ -0,0 +1,36 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package uber.stovepipe.messagequeue; + +import "api/base/messagequeue/proto/messagequeue.proto"; + +option go_package = "github.com/uber/submitqueue/stovepipe/core/messagequeue/protopb"; +option java_multiple_files = true; +option java_outer_classname = "FixRequestProto"; +option java_package = "com.uber.submitqueue.stovepipe.messagequeue"; + +// FixRequest identifies a confirmed issue opening and the build requesting repair. +// The observation and open issue must be persisted before publication. +message FixRequest { + option (uber.base.messagequeue.topic_keys) = "fix"; + + string queue_name = 1; + string build_id = 2; + string task_id = 3; + string unit_id = 4; + string first_seen_build_id = 5; +} diff --git a/stovepipe/core/messagequeue/protopb/BUILD.bazel b/stovepipe/core/messagequeue/protopb/BUILD.bazel index f1d2656fe..3d666b85c 100644 --- a/stovepipe/core/messagequeue/protopb/BUILD.bazel +++ b/stovepipe/core/messagequeue/protopb/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "build.pb.go", "buildanalysis.pb.go", "buildsignal.pb.go", + "fix.pb.go", "process.pb.go", "record.pb.go", ], diff --git a/stovepipe/core/messagequeue/protopb/fix.pb.go b/stovepipe/core/messagequeue/protopb/fix.pb.go new file mode 100644 index 000000000..a1a83c165 --- /dev/null +++ b/stovepipe/core/messagequeue/protopb/fix.pb.go @@ -0,0 +1,179 @@ +// Copyright (c) 2026 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v5.29.3 +// source: fix.proto + +package protopb + +import ( + reflect "reflect" + sync "sync" + unsafe "unsafe" + + _ "github.com/uber/submitqueue/api/base/messagequeue/protopb" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// FixRequest identifies a confirmed issue opening and the build requesting repair. +// The observation and open issue must be persisted before publication. +type FixRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + QueueName string `protobuf:"bytes,1,opt,name=queue_name,json=queueName,proto3" json:"queue_name,omitempty"` + BuildId string `protobuf:"bytes,2,opt,name=build_id,json=buildId,proto3" json:"build_id,omitempty"` + TaskId string `protobuf:"bytes,3,opt,name=task_id,json=taskId,proto3" json:"task_id,omitempty"` + UnitId string `protobuf:"bytes,4,opt,name=unit_id,json=unitId,proto3" json:"unit_id,omitempty"` + FirstSeenBuildId string `protobuf:"bytes,5,opt,name=first_seen_build_id,json=firstSeenBuildId,proto3" json:"first_seen_build_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FixRequest) Reset() { + *x = FixRequest{} + mi := &file_fix_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FixRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FixRequest) ProtoMessage() {} + +func (x *FixRequest) ProtoReflect() protoreflect.Message { + mi := &file_fix_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FixRequest.ProtoReflect.Descriptor instead. +func (*FixRequest) Descriptor() ([]byte, []int) { + return file_fix_proto_rawDescGZIP(), []int{0} +} + +func (x *FixRequest) GetQueueName() string { + if x != nil { + return x.QueueName + } + return "" +} + +func (x *FixRequest) GetBuildId() string { + if x != nil { + return x.BuildId + } + return "" +} + +func (x *FixRequest) GetTaskId() string { + if x != nil { + return x.TaskId + } + return "" +} + +func (x *FixRequest) GetUnitId() string { + if x != nil { + return x.UnitId + } + return "" +} + +func (x *FixRequest) GetFirstSeenBuildId() string { + if x != nil { + return x.FirstSeenBuildId + } + return "" +} + +var File_fix_proto protoreflect.FileDescriptor + +const file_fix_proto_rawDesc = "" + + "\n" + + "\tfix.proto\x12\x1buber.stovepipe.messagequeue\x1a.api/base/messagequeue/proto/messagequeue.proto\"\xb0\x01\n" + + "\n" + + "FixRequest\x12\x1d\n" + + "\n" + + "queue_name\x18\x01 \x01(\tR\tqueueName\x12\x19\n" + + "\bbuild_id\x18\x02 \x01(\tR\abuildId\x12\x17\n" + + "\atask_id\x18\x03 \x01(\tR\x06taskId\x12\x17\n" + + "\aunit_id\x18\x04 \x01(\tR\x06unitId\x12-\n" + + "\x13first_seen_build_id\x18\x05 \x01(\tR\x10firstSeenBuildId:\a\x8a\xb5\x18\x03fixB\x81\x01\n" + + "+com.uber.submitqueue.stovepipe.messagequeueB\x0fFixRequestProtoP\x01Z?github.com/uber/submitqueue/stovepipe/core/messagequeue/protopbb\x06proto3" + +var ( + file_fix_proto_rawDescOnce sync.Once + file_fix_proto_rawDescData []byte +) + +func file_fix_proto_rawDescGZIP() []byte { + file_fix_proto_rawDescOnce.Do(func() { + file_fix_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_fix_proto_rawDesc), len(file_fix_proto_rawDesc))) + }) + return file_fix_proto_rawDescData +} + +var file_fix_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_fix_proto_goTypes = []any{ + (*FixRequest)(nil), // 0: uber.stovepipe.messagequeue.FixRequest +} +var file_fix_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_fix_proto_init() } +func file_fix_proto_init() { + if File_fix_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_fix_proto_rawDesc), len(file_fix_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_fix_proto_goTypes, + DependencyIndexes: file_fix_proto_depIdxs, + MessageInfos: file_fix_proto_msgTypes, + }.Build() + File_fix_proto = out.File + file_fix_proto_goTypes = nil + file_fix_proto_depIdxs = nil +} diff --git a/stovepipe/core/messagequeue/topics.go b/stovepipe/core/messagequeue/topics.go index 09da535a6..289598f61 100644 --- a/stovepipe/core/messagequeue/topics.go +++ b/stovepipe/core/messagequeue/topics.go @@ -51,4 +51,8 @@ const ( // terminal status; non-terminal polls never publish here. Partitioned by // request id. TopicKeyRecord TopicKey = "record" + + // TopicKeyFix carries confirmed issue references for repair selection. + // Deliveries are partitioned by detecting build ID. + TopicKeyFix TopicKey = "fix" ) diff --git a/stovepipe/entity/BUILD.bazel b/stovepipe/entity/BUILD.bazel index 16d1bbfd1..c539aac8d 100644 --- a/stovepipe/entity/BUILD.bazel +++ b/stovepipe/entity/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "build.go", "build_issue.go", + "fix.go", "ingest.go", "list.go", "open_issue.go", diff --git a/stovepipe/entity/fix.go b/stovepipe/entity/fix.go new file mode 100644 index 000000000..7c5a23b53 --- /dev/null +++ b/stovepipe/entity/fix.go @@ -0,0 +1,54 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package entity + +// FixState is a repair's submission state, independent of execution or check health. +type FixState string + +const ( + FixStateUnknown FixState = "" + FixQueued FixState = "queued" + // FixSubmitting means a dispatch claim is durable; acceptance is unknown. + FixSubmitting FixState = "submitting" + // FixSubmitted means a provider returned an execution reference. + FixSubmitted FixState = "submitted" + // FixSubmissionUnknown means the trigger returned an error or no reference. + FixSubmissionUnknown FixState = "submission_unknown" + FixSkipped FixState = "skipped" +) + +// Fix is a versioned repair submission associated with issue openings. +type Fix struct { + // ID is the queue-scoped attempt identifier, stable across message redelivery. + ID string `json:"id"` + // Queue is the owning validation queue. + Queue string `json:"queue"` + // BuildID is the build whose confirmed failure initiated this attempt. + BuildID string `json:"build_id"` + // State is the submission state; submitted does not imply execution or recovery. + State FixState `json:"state"` + // Issues contains issue-opening references; the initial submission contains one issue. + Issues []OpenIssue `json:"issues"` + // ExecutionRef is the optional provider reference returned by submission, empty until known. + ExecutionRef string `json:"execution_ref"` + // Metadata contains optional change references and provider context; nil and empty maps are equivalent. + Metadata map[string]string `json:"metadata"` + // Version is the optimistic storage version. + Version int32 `json:"version"` + // CreatedAt is the UTC creation time in milliseconds since the epoch. + CreatedAt int64 `json:"created_at"` + // UpdatedAt is the UTC update time in milliseconds since the epoch. + UpdatedAt int64 `json:"updated_at"` +} diff --git a/stovepipe/entity/request_log.go b/stovepipe/entity/request_log.go index 7c53434c4..6b876fc3f 100644 --- a/stovepipe/entity/request_log.go +++ b/stovepipe/entity/request_log.go @@ -32,6 +32,10 @@ const ( RequestEventProjectFactsRecorded RequestEvent = "project_facts_recorded" // RequestEventRecordAbandoned records that record-stage work stopped after exhausting retries. RequestEventRecordAbandoned RequestEvent = "record_abandoned" + // RequestEventFixSubmissionStarted records a durable repair dispatch claim. + RequestEventFixSubmissionStarted RequestEvent = "fix_submission_started" + // RequestEventFixSubmissionRecorded records an accepted or uncertain trigger result. + RequestEventFixSubmissionRecorded RequestEvent = "fix_submission_recorded" // RequestEventBuildAnalysisAbandoned records incomplete build analysis, not a validation outcome. RequestEventBuildAnalysisAbandoned RequestEvent = "build_analysis_abandoned" ) @@ -139,7 +143,7 @@ func (e RequestLog) validateEvent() error { return fmt.Errorf("event log must not contain request-state context") } switch e.Event { - case RequestEventBuildTriggered, RequestEventBuildFinished, RequestEventValidationFactRecorded, RequestEventProjectFactsRecorded, RequestEventRecordAbandoned, RequestEventBuildAnalysisAbandoned: + case RequestEventBuildTriggered, RequestEventBuildFinished, RequestEventValidationFactRecorded, RequestEventProjectFactsRecorded, RequestEventRecordAbandoned, RequestEventBuildAnalysisAbandoned, RequestEventFixSubmissionStarted, RequestEventFixSubmissionRecorded: default: return fmt.Errorf("unknown request event %q", e.Event) } diff --git a/stovepipe/extension/fixer/BUILD.bazel b/stovepipe/extension/fixer/BUILD.bazel new file mode 100644 index 000000000..a6af87006 --- /dev/null +++ b/stovepipe/extension/fixer/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["fixer.go"], + importpath = "github.com/uber/submitqueue/stovepipe/extension/fixer", + visibility = ["//visibility:public"], + deps = ["//stovepipe/entity:go_default_library"], +) diff --git a/stovepipe/extension/fixer/fixer.go b/stovepipe/extension/fixer/fixer.go new file mode 100644 index 000000000..98c441969 --- /dev/null +++ b/stovepipe/extension/fixer/fixer.go @@ -0,0 +1,50 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package fixer defines one-shot repair submission. +package fixer + +//go:generate mockgen -source=fixer.go -destination=mock/fixer_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/stovepipe/entity" +) + +// RepairExecutor submits one repair attempt and returns its provider reference. +// A nonempty reference means submission was accepted, not that the issue recovered. +// An error can follow acceptance. Callers must not automatically repeat Submit. +type RepairExecutor interface { + Submit(context.Context, SubmitRequest) (string, error) +} + +// SubmitRequest identifies the validation failure to repair within the configured queue. +type SubmitRequest struct { + // BuildID identifies the build whose failure initiated this submission. + BuildID string + // Issue identifies the selected condition and its continuous opening. + Issue entity.OpenIssue +} + +// Config identifies the queue whose repair extension is being resolved. +type Config struct { + // QueueName is the validation queue. + QueueName string +} + +// Factory resolves configured repair executors. +type Factory interface { + For(Config) (RepairExecutor, error) +} diff --git a/stovepipe/extension/fixer/mock/BUILD.bazel b/stovepipe/extension/fixer/mock/BUILD.bazel new file mode 100644 index 000000000..896f52725 --- /dev/null +++ b/stovepipe/extension/fixer/mock/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "go_default_library", + srcs = ["fixer_mock.go"], + importpath = "github.com/uber/submitqueue/stovepipe/extension/fixer/mock", + visibility = ["//visibility:public"], + deps = [ + "//stovepipe/extension/fixer:go_default_library", + "@org_uber_go_mock//gomock:go_default_library", + ], +) diff --git a/stovepipe/extension/fixer/mock/fixer_mock.go b/stovepipe/extension/fixer/mock/fixer_mock.go new file mode 100644 index 000000000..e94293bad --- /dev/null +++ b/stovepipe/extension/fixer/mock/fixer_mock.go @@ -0,0 +1,96 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: fixer.go +// +// Generated by this command: +// +// mockgen -source=fixer.go -destination=mock/fixer_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + fixer "github.com/uber/submitqueue/stovepipe/extension/fixer" + gomock "go.uber.org/mock/gomock" +) + +// MockRepairExecutor is a mock of RepairExecutor interface. +type MockRepairExecutor struct { + ctrl *gomock.Controller + recorder *MockRepairExecutorMockRecorder + isgomock struct{} +} + +// MockRepairExecutorMockRecorder is the mock recorder for MockRepairExecutor. +type MockRepairExecutorMockRecorder struct { + mock *MockRepairExecutor +} + +// NewMockRepairExecutor creates a new mock instance. +func NewMockRepairExecutor(ctrl *gomock.Controller) *MockRepairExecutor { + mock := &MockRepairExecutor{ctrl: ctrl} + mock.recorder = &MockRepairExecutorMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRepairExecutor) EXPECT() *MockRepairExecutorMockRecorder { + return m.recorder +} + +// Submit mocks base method. +func (m *MockRepairExecutor) Submit(arg0 context.Context, arg1 fixer.SubmitRequest) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Submit", arg0, arg1) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Submit indicates an expected call of Submit. +func (mr *MockRepairExecutorMockRecorder) Submit(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockRepairExecutor)(nil).Submit), arg0, arg1) +} + +// MockFactory is a mock of Factory interface. +type MockFactory struct { + ctrl *gomock.Controller + recorder *MockFactoryMockRecorder + isgomock struct{} +} + +// MockFactoryMockRecorder is the mock recorder for MockFactory. +type MockFactoryMockRecorder struct { + mock *MockFactory +} + +// NewMockFactory creates a new mock instance. +func NewMockFactory(ctrl *gomock.Controller) *MockFactory { + mock := &MockFactory{ctrl: ctrl} + mock.recorder = &MockFactoryMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFactory) EXPECT() *MockFactoryMockRecorder { + return m.recorder +} + +// For mocks base method. +func (m *MockFactory) For(arg0 fixer.Config) (fixer.RepairExecutor, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "For", arg0) + ret0, _ := ret[0].(fixer.RepairExecutor) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// For indicates an expected call of For. +func (mr *MockFactoryMockRecorder) For(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "For", reflect.TypeOf((*MockFactory)(nil).For), arg0) +} diff --git a/stovepipe/extension/storage/BUILD.bazel b/stovepipe/extension/storage/BUILD.bazel index a90b46c32..f24c85b2f 100644 --- a/stovepipe/extension/storage/BUILD.bazel +++ b/stovepipe/extension/storage/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "build_issue_store.go", "build_store.go", + "fix_store.go", "open_issue_store.go", "queue_log_store.go", "queue_store.go", diff --git a/stovepipe/extension/storage/README.md b/stovepipe/extension/storage/README.md index 5e9072477..10557bd1a 100644 --- a/stovepipe/extension/storage/README.md +++ b/stovepipe/extension/storage/README.md @@ -41,3 +41,9 @@ An open row is immutable. Its first-seen build identifies the current opening, n Deletion matches the task, unit, and first-seen build within the bound queue. It is safe to repeat: an absent opening is a no-op, and a stale deletion never removes a different opening of the same condition. Database failures are returned for retry. MySQL identifiers use the same binary byte limits as per-build observations. Build IDs are opaque context, not ordering tokens. The store does not choose resolution, enumerate issues, write lifecycle history, or reconcile analysis responses. History and source-order reconciliation belong to later controllers, not this membership store. + +## Repair submissions + +`FixStore` retains versioned repair submissions by queue and fix ID. The initial submission uses the detecting BuildID as its ID, so create-if-absent selects one issue per build without modifying the Build or OpenIssue records. Its issue references use `OpenIssue`, including the immutable `FirstSeenBuildID` of each opening. + +The [repair RFC](../../../doc/rfc/stovepipe/fixer.md) defines one-shot dispatch and request-history recording. Failure and recovery evidence remain owned by build analysis. diff --git a/stovepipe/extension/storage/fix_store.go b/stovepipe/extension/storage/fix_store.go new file mode 100644 index 000000000..be25421e0 --- /dev/null +++ b/stovepipe/extension/storage/fix_store.go @@ -0,0 +1,32 @@ +// Copyright (c) 2025 Uber Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +//go:generate mockgen -source=fix_store.go -destination=mock/fix_store_mock.go -package=mock + +import ( + "context" + + "github.com/uber/submitqueue/stovepipe/entity" +) + +// FixStore persists queue-scoped versioned logical repairs by ID. +// Create/Get use ErrAlreadyExists/ErrNotFound; Update uses ErrVersionMismatch. +// Callers supply both the guard version and replacement version. +type FixStore interface { + Create(context.Context, entity.Fix) error + Get(context.Context, string) (entity.Fix, error) + Update(context.Context, entity.Fix, int32, int32) error +} diff --git a/stovepipe/extension/storage/mock/BUILD.bazel b/stovepipe/extension/storage/mock/BUILD.bazel index e43236134..f14811109 100644 --- a/stovepipe/extension/storage/mock/BUILD.bazel +++ b/stovepipe/extension/storage/mock/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "build_issue_store_mock.go", "build_store_mock.go", + "fix_store_mock.go", "open_issue_store_mock.go", "queue_log_store_mock.go", "queue_store_mock.go", diff --git a/stovepipe/extension/storage/mock/fix_store_mock.go b/stovepipe/extension/storage/mock/fix_store_mock.go new file mode 100644 index 000000000..6b6919c4f --- /dev/null +++ b/stovepipe/extension/storage/mock/fix_store_mock.go @@ -0,0 +1,85 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: fix_store.go +// +// Generated by this command: +// +// mockgen -source=fix_store.go -destination=mock/fix_store_mock.go -package=mock +// + +// Package mock is a generated GoMock package. +package mock + +import ( + context "context" + reflect "reflect" + + entity "github.com/uber/submitqueue/stovepipe/entity" + gomock "go.uber.org/mock/gomock" +) + +// MockFixStore is a mock of FixStore interface. +type MockFixStore struct { + ctrl *gomock.Controller + recorder *MockFixStoreMockRecorder + isgomock struct{} +} + +// MockFixStoreMockRecorder is the mock recorder for MockFixStore. +type MockFixStoreMockRecorder struct { + mock *MockFixStore +} + +// NewMockFixStore creates a new mock instance. +func NewMockFixStore(ctrl *gomock.Controller) *MockFixStore { + mock := &MockFixStore{ctrl: ctrl} + mock.recorder = &MockFixStoreMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFixStore) EXPECT() *MockFixStoreMockRecorder { + return m.recorder +} + +// Create mocks base method. +func (m *MockFixStore) Create(arg0 context.Context, arg1 entity.Fix) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockFixStoreMockRecorder) Create(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockFixStore)(nil).Create), arg0, arg1) +} + +// Get mocks base method. +func (m *MockFixStore) Get(arg0 context.Context, arg1 string) (entity.Fix, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1) + ret0, _ := ret[0].(entity.Fix) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockFixStoreMockRecorder) Get(arg0, arg1 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockFixStore)(nil).Get), arg0, arg1) +} + +// Update mocks base method. +func (m *MockFixStore) Update(arg0 context.Context, arg1 entity.Fix, arg2, arg3 int32) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Update", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// Update indicates an expected call of Update. +func (mr *MockFixStoreMockRecorder) Update(arg0, arg1, arg2, arg3 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockFixStore)(nil).Update), arg0, arg1, arg2, arg3) +} diff --git a/tool/proto/BUILD.bazel b/tool/proto/BUILD.bazel index 62be9e389..c0394e9c8 100644 --- a/tool/proto/BUILD.bazel +++ b/tool/proto/BUILD.bazel @@ -105,6 +105,7 @@ go_proto_generated_files( "//stovepipe/core/messagequeue/proto:build.proto", "//stovepipe/core/messagequeue/proto:buildanalysis.proto", "//stovepipe/core/messagequeue/proto:buildsignal.proto", + "//stovepipe/core/messagequeue/proto:fix.proto", "//stovepipe/core/messagequeue/proto:process.proto", "//stovepipe/core/messagequeue/proto:record.proto", ],