From f72cd790a04cc0b8d76a1343dc0df6fcb5637009 Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Thu, 21 May 2026 19:09:51 +0500 Subject: [PATCH 01/17] Merge pull request #1 from tracebloc/feat/149-schema-validator Phase 1: embed ingest.v1.json + `tracebloc ingest validate` --- .github/workflows/build.yml | 14 ++ CONTRIBUTING.md | 15 +- Makefile | 75 ++++++ cmd/tracebloc/main.go | 32 ++- go.mod | 7 +- go.sum | 8 + internal/cli/exit.go | 68 ++++++ internal/cli/ingest.go | 143 +++++++++++ internal/cli/ingest_test.go | 190 +++++++++++++++ internal/cli/root.go | 1 + internal/schema/embed.go | 30 +++ internal/schema/ingest.v1.json | 380 +++++++++++++++++++++++++++++ internal/schema/validate.go | 238 ++++++++++++++++++ internal/schema/validate_test.go | 405 +++++++++++++++++++++++++++++++ scripts/sync-schema.sh | 86 +++++++ 15 files changed, 1681 insertions(+), 11 deletions(-) create mode 100644 Makefile create mode 100644 internal/cli/exit.go create mode 100644 internal/cli/ingest.go create mode 100644 internal/cli/ingest_test.go create mode 100644 internal/schema/embed.go create mode 100644 internal/schema/ingest.v1.json create mode 100644 internal/schema/validate.go create mode 100644 internal/schema/validate_test.go create mode 100755 scripts/sync-schema.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c31819e..82ca229 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,20 @@ permissions: contents: read jobs: + schema-drift: + name: Schema drift check + # Verifies the embedded internal/schema/ingest.v1.json matches + # tracebloc/data-ingestors' master. A green PR that silently + # diverges from upstream is a real correctness hazard — a + # customer's YAML could pass `tracebloc ingest validate` locally + # but be rejected by jobs-manager (or vice versa). Forcing the + # sync as a PR step keeps drift visible. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: scripts/sync-schema.sh --check + run: ./scripts/sync-schema.sh --check + test: name: Test runs-on: ubuntu-latest diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c1bf3f..2bd03cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,14 +2,19 @@ ## Local development -```bash -go build -o tracebloc ./cmd/tracebloc -./tracebloc version +The `Makefile` mirrors the CI pipeline — `make ci` runs the exact same checks that PR #N's GitHub Actions run. If `make ci` passes locally, CI will too (modulo non-deterministic flakes). **Run `make ci` before pushing.** Skipping it has cost us at least one PR's worth of fix-up commits per bug class so far. -go test ./... -golangci-lint run # https://golangci-lint.run/usage/install/ +```bash +make ci # vet + test + lint + fmt-check + schema-check (run this before push) +make build # produces ./tracebloc +make fmt # fixes gofmt -s drift in place +make schema-sync # pulls latest ingest.v1.json from data-ingestors master ``` +Individual targets are also runnable in isolation — `make test`, `make lint`, etc. See the `Makefile` for the full list. + +Requires [`golangci-lint`](https://golangci-lint.run/usage/install/) (install via `brew install golangci-lint` or your platform's equivalent). + Cobra autocomplete for `bash` / `zsh` / `fish` / `powershell` is available via the `completion` subcommand. Useful while developing too: ```bash diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0384697 --- /dev/null +++ b/Makefile @@ -0,0 +1,75 @@ +# Top-level Makefile for tracebloc/cli. +# +# Purpose: keep the local feedback loop the same shape as the CI +# loop. Anything that fails in `make ci` would have failed on a PR, +# and vice versa. Don't add targets here that aren't also enforced +# by .github/workflows/build.yml — divergence between local and CI +# is the bug this file exists to prevent. + +# ---- toggles ----------------------------------------------------- + +GO ?= go +GOLANGCI_LINT ?= golangci-lint +PKGS := ./... + +# ---- top-level targets ------------------------------------------- + +.PHONY: ci +ci: vet test lint fmt-check schema-check + @echo "==> ci: all green" + +.PHONY: build +build: + $(GO) build -o tracebloc ./cmd/tracebloc + +.PHONY: install +install: + $(GO) install ./cmd/tracebloc + +# ---- individual targets (also runnable in isolation) ------------- + +.PHONY: vet +vet: + $(GO) vet $(PKGS) + +.PHONY: test +test: + $(GO) test -race -cover $(PKGS) + +.PHONY: lint +lint: + @command -v $(GOLANGCI_LINT) >/dev/null 2>&1 || { \ + echo "==> $(GOLANGCI_LINT) not on PATH"; \ + echo " install via: brew install golangci-lint"; \ + echo " or see: https://golangci-lint.run/usage/install/"; \ + exit 1; \ + } + $(GOLANGCI_LINT) run + +.PHONY: fmt +fmt: + gofmt -s -w . + +.PHONY: fmt-check +fmt-check: + @diff="$$(gofmt -s -l . 2>/dev/null)"; \ + if [ -n "$$diff" ]; then \ + echo "==> gofmt -s needed on:"; \ + echo "$$diff" | sed 's/^/ /'; \ + echo "==> run \`make fmt\` to fix"; \ + exit 1; \ + fi + +.PHONY: schema-check +schema-check: + ./scripts/sync-schema.sh --check + +.PHONY: schema-sync +schema-sync: + ./scripts/sync-schema.sh + +# ---- cleanup ----------------------------------------------------- + +.PHONY: clean +clean: + rm -rf tracebloc dist/ coverage.out coverage.html diff --git a/cmd/tracebloc/main.go b/cmd/tracebloc/main.go index 071bba4..a38a446 100644 --- a/cmd/tracebloc/main.go +++ b/cmd/tracebloc/main.go @@ -18,6 +18,7 @@ package main import ( + "fmt" "os" "github.com/tracebloc/cli/internal/cli" @@ -34,13 +35,34 @@ var ( ) func main() { - if err := cli.NewRootCmd(cli.BuildInfo{ + err := cli.NewRootCmd(cli.BuildInfo{ Version: version, GitSHA: gitSHA, BuildDate: buildDate, - }).Execute(); err != nil { - // cobra has already printed the error + usage to stderr by - // the time we get here; just propagate a non-zero exit. - os.Exit(1) + }).Execute() + if err == nil { + return } + + // Print the error to stderr before exiting. The root command + // sets SilenceErrors: true to keep cobra from prepending its + // own "Error: ..." line on top of structured handler output + // — but that puts the burden on us to surface the error + // message ourselves. Without this, every non-schema-violation + // failure (file-read errors, YAML parse errors, schema-compile + // errors) exits non-zero with NO message to the customer. + // + // Handlers that have already printed their own diagnostic + // (e.g. `ingest validate` prints per-violation lines) signal + // "silent" by returning an exitError with a nil inner — see + // cli.IsSilentError for the contract. + if !cli.IsSilentError(err) { + fmt.Fprintln(os.Stderr, "Error:", err) + } + + // Map command-defined exit codes through. Handlers that want a + // specific exit code (e.g. `ingest validate` returns 2 for + // schema violations, 3 for parse errors) return a *cli.ExitError + // the package exports; everything else gets the default 1. + os.Exit(cli.ExitCodeFromError(err)) } diff --git a/go.mod b/go.mod index 796f785..8c5a9db 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,12 @@ module github.com/tracebloc/cli go 1.22 -require github.com/spf13/cobra v1.8.1 +require ( + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 + github.com/spf13/cobra v1.8.1 + golang.org/x/text v0.16.0 + gopkg.in/yaml.v3 v3.0.1 +) require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect diff --git a/go.sum b/go.sum index 912390a..3cf027a 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,18 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/cli/exit.go b/internal/cli/exit.go new file mode 100644 index 0000000..bdbe10a --- /dev/null +++ b/internal/cli/exit.go @@ -0,0 +1,68 @@ +package cli + +// ExitCodeFromError extracts an exit code from a handler-returned +// error. Handlers that need a specific exit code wrap their return +// in an *exitError (see ingest.go); everything else defaults to 1. +// +// Public-but-package-keyed: main.go is the only intended caller, +// and the exitError type itself stays unexported so subcommand +// handlers go through the constructor. +func ExitCodeFromError(err error) int { + if err == nil { + return 0 + } + var ee *exitError + if asExitError(err, &ee) { + return ee.code + } + return 1 +} + +// IsSilentError reports whether a handler-returned error wants +// main() to suppress its own "Error: ..." stderr line on the way +// out. The contract: a handler that has already printed a +// structured diagnostic itself (e.g. the schema-validate path +// prints per-violation lines to stderr) returns +// `&exitError{code: N, err: nil}` to signal "exit non-zero but +// don't print anything more." Errors with a non-nil inner err +// (file-read failures, parse errors, schema-compile bugs) are +// NOT silent — main() prints them so the customer doesn't see a +// bare non-zero exit with no explanation. +// +// Caller pattern in main.go: +// +// if err != nil && !cli.IsSilentError(err) { +// fmt.Fprintln(os.Stderr, "Error:", err) +// } +// os.Exit(cli.ExitCodeFromError(err)) +func IsSilentError(err error) bool { + if err == nil { + return false + } + var ee *exitError + if asExitError(err, &ee) { + return ee.err == nil + } + return false +} + +// asExitError walks the wrapped-error chain looking for an +// *exitError. Same pattern as errors.As but with a typed target so +// callers don't have to import errors at every site. +func asExitError(err error, target **exitError) bool { + for cur := err; cur != nil; cur = unwrapError(cur) { + if ee, ok := cur.(*exitError); ok { + *target = ee + return true + } + } + return false +} + +func unwrapError(err error) error { + type unwrapper interface{ Unwrap() error } + if u, ok := err.(unwrapper); ok { + return u.Unwrap() + } + return nil +} diff --git a/internal/cli/ingest.go b/internal/cli/ingest.go new file mode 100644 index 0000000..b49d692 --- /dev/null +++ b/internal/cli/ingest.go @@ -0,0 +1,143 @@ +package cli + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/tracebloc/cli/internal/schema" +) + +// newIngestCmd implements the `tracebloc ingest` subtree. Today it +// has only one verb — `validate` — which runs the schema check +// locally without touching the cluster. Future verbs (status, retry, +// cancel) hang off this same parent in later phases. +func newIngestCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "ingest", + Short: "Inspect and manage ingestion configurations", + Long: `Commands for working with ingest.yaml files and ingestion runs. + +Today only ` + "`validate`" + ` is implemented — it runs the same schema check +the cluster's jobs-manager runs, but locally and instantly. Use it to +catch typos and missing fields before submitting an ingestion. + +Future verbs (status, retry, cancel) will land alongside the +push/list/show commands in later phases.`, + } + + cmd.AddCommand(newIngestValidateCmd()) + return cmd +} + +// newIngestValidateCmd implements `tracebloc ingest validate `. +// Reads a YAML file from disk, validates it against the embedded v1 +// schema, prints any violations in the same format the Python +// implementation uses, exits non-zero on any violation. +// +// The output format is deliberately matched to +// tracebloc_ingestor.cli.run._format_errors so a customer's editor +// or CI can grep both implementations' output uniformly. See +// internal/schema/validate.go for the formatting contract. +func newIngestValidateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "validate ", + Short: "Validate an ingest.yaml against the embedded v1 schema, locally", + Long: `Reads , parses it as YAML, and validates it against the bundled +ingest.v1.json schema (synced from tracebloc/data-ingestors). Prints +violations in the same JSON-pointer-prefixed format the cluster's +jobs-manager uses, and exits non-zero if any are found. + +Useful as a pre-flight before ` + "`tracebloc dataset push`" + ` lands in a +future phase; for now, customers running the Helm chart can validate +their ` + "`ingest.yaml`" + ` before invoking ` + "`helm install`" + `, getting +millisecond local feedback instead of a multi-second cluster round +trip. + +Exit codes: + 0 YAML parses and validates cleanly + 2 YAML parses but has schema violations (printed to stderr) + 3 YAML doesn't parse or file isn't readable`, + Args: cobra.ExactArgs(1), + RunE: runIngestValidate, + } + return cmd +} + +func runIngestValidate(cmd *cobra.Command, args []string) error { + path := args[0] + + body, err := os.ReadFile(path) + if err != nil { + // fileError is exit-code 3 territory. We use a sentinel + // exit-coded error so cobra propagates the right code via + // the main()-side os.Exit mapping (in a follow-up commit + // we'll wire main.go to inspect for these). + return &exitError{code: 3, err: fmt.Errorf("reading %s: %w", path, err)} + } + + v, err := schema.NewV1Validator() + if err != nil { + // Schema-compilation failures are infrastructure-side, not + // customer-side — we bundle the schema, so this only fires + // if the build is broken. Treat as exit-code-2 so CI can + // distinguish from a customer file problem. + return &exitError{code: 2, err: fmt.Errorf("loading embedded schema: %w", err)} + } + + _, violations, parseErr := v.ValidateYAML(body) + if parseErr != nil { + return &exitError{code: 3, err: fmt.Errorf("%s: %w", path, parseErr)} + } + + if len(violations) == 0 { + // Explicit discard: Fprintf returns an error when the + // underlying writer fails (closed pipe, etc.). For the + // success summary we'd rather still exit 0 even if the + // downstream consumer dropped the connection — they got + // what they needed (the exit code), and propagating a + // pipe-write error would convert success into failure for + // reasons unrelated to validation. + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s: ok\n", path) + return nil + } + + // Print violations to stderr so success can be piped without + // interference; error lines are diagnostic, not data. Same + // pipe-write rationale as the ok-path above: don't let a + // stderr-write failure mask the real exit-2 schema-violation + // signal. + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%s: schema validation failed (%d issue%s)\n", + path, len(violations), plural(len(violations))) + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), schema.FormatErrors(violations)) + return &exitError{code: 2, err: nil} // err==nil so cobra doesn't print "Error: ..." on top +} + +func plural(n int) string { + if n == 1 { + return "" + } + return "s" +} + +// exitError carries a process exit code alongside (or instead of) +// an error message. main.go inspects for this type before calling +// os.Exit, mapping the code through. Other handlers can opt in to +// specific exit codes by returning an exitError. +type exitError struct { + code int + err error +} + +func (e *exitError) Error() string { + if e.err == nil { + return fmt.Sprintf("exit %d", e.code) + } + return e.err.Error() +} + +func (e *exitError) Unwrap() error { return e.err } + +// Code returns the process exit code main() should propagate. +func (e *exitError) Code() int { return e.code } diff --git a/internal/cli/ingest_test.go b/internal/cli/ingest_test.go new file mode 100644 index 0000000..4e7dd08 --- /dev/null +++ b/internal/cli/ingest_test.go @@ -0,0 +1,190 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTmpYAML drops a small YAML doc into the test's t.TempDir and +// returns the path. Using TempDir guarantees cleanup on test exit; +// callers don't have to defer os.Remove themselves. +func writeTmpYAML(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "ingest.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + return path +} + +// execIngestValidate drives the full cobra dispatch for +// `tracebloc ingest validate ` and returns the exit code, the +// captured stdout, and the captured stderr. Tests should never share +// a *cobra.Command across cases — cobra holds flag state on the +// command tree and stale trees leak one test's args into the next. +func execIngestValidate(t *testing.T, path string) (exitCode int, stdout, stderr string) { + t.Helper() + root := NewRootCmd(BuildInfo{Version: "test"}) + var so, se bytes.Buffer + root.SetOut(&so) + root.SetErr(&se) + root.SetArgs([]string{"ingest", "validate", path}) + + err := root.Execute() + return ExitCodeFromError(err), so.String(), se.String() +} + +func TestIngestValidate_HappyPath(t *testing.T) { + path := writeTmpYAML(t, ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: cats_dogs_train +intent: train +category: image_classification +csv: /data/labels.csv +images: /data/images/ +label: image_label +`) + + code, stdout, stderr := execIngestValidate(t, path) + if code != 0 { + t.Fatalf("expected exit 0, got %d\nstderr:\n%s", code, stderr) + } + if !strings.Contains(stdout, "ok") { + t.Errorf("expected 'ok' on stdout, got: %q", stdout) + } + if stderr != "" { + t.Errorf("expected empty stderr on success, got: %q", stderr) + } +} + +func TestIngestValidate_SchemaFailureExitsTwo(t *testing.T) { + // Missing required fields → schema violation → exit 2. + path := writeTmpYAML(t, ` +kind: IngestConfig +category: image_classification +table: t +csv: /data/labels.csv +images: /data/images/ +label: image_label +`) + + code, _, stderr := execIngestValidate(t, path) + if code != 2 { + t.Fatalf("expected exit 2 for schema failure, got %d", code) + } + // Errors print to stderr (so success can be piped without + // interference). Both the count line + the per-error lines + // should be there. + for _, want := range []string{"schema validation failed", "apiVersion"} { + if !strings.Contains(stderr, want) { + t.Errorf("expected stderr to mention %q, got:\n%s", want, stderr) + } + } +} + +func TestIngestValidate_UnreadableFileExitsThree(t *testing.T) { + // Distinct exit code (3) for file-level problems (missing, + // permission-denied, etc.) — separates from schema violations + // (2) so callers can branch on the cause. + code, _, _ := execIngestValidate(t, "/tmp/definitely-does-not-exist-"+t.Name()) + if code != 3 { + t.Errorf("expected exit 3 for missing file, got %d", code) + } +} + +func TestIngestValidate_NonMappingExitsThree(t *testing.T) { + // A top-level YAML sequence (vs mapping) is a parse-shape + // problem, not a schema problem — surface it as the file-level + // exit code so the customer knows their file isn't an + // ingest-config at all, vs being one with the wrong fields. + path := writeTmpYAML(t, "- one\n- two\n") + + code, _, _ := execIngestValidate(t, path) + if code != 3 { + t.Errorf("expected exit 3 for non-mapping YAML, got %d", code) + } +} + +func TestIngestValidate_RequiresExactlyOneArg(t *testing.T) { + // Cobra catches arg-count violations before our RunE runs; + // confirm we exit non-zero (the specific code is cobra's + // default 1, not our exitError 2/3 — that's intentional). + root := NewRootCmd(BuildInfo{Version: "test"}) + var so, se bytes.Buffer + root.SetOut(&so) + root.SetErr(&se) + root.SetArgs([]string{"ingest", "validate"}) // no path + + err := root.Execute() + if err == nil { + t.Fatalf("expected error from missing path arg, got nil") + } + if got := ExitCodeFromError(err); got == 0 { + t.Errorf("expected non-zero exit, got %d", got) + } +} + +func TestExitCodeFromError(t *testing.T) { + // Defensive: pin the exitError dispatch behavior since this is + // the only thing main() depends on from this package. + if got := ExitCodeFromError(nil); got != 0 { + t.Errorf("nil err should map to 0, got %d", got) + } + if got := ExitCodeFromError(&exitError{code: 7}); got != 7 { + t.Errorf("explicit exit code should propagate, got %d", got) + } + // A wrapped exitError still reports its code (asExitError walks + // the unwrap chain). + wrapped := &exitError{code: 9, err: &exitError{code: 0}} + if got := ExitCodeFromError(wrapped); got != 9 { + t.Errorf("outermost exitError wins, got %d", got) + } +} + +// IsSilentError is the main()-side hook that decides whether to +// print an "Error: ..." stderr line on the way out. Pin the +// contract so main.go doesn't silently regress to swallowing +// errors (the high-severity bugbot finding that led to this +// being added in the first place). +func TestIsSilentError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil error", nil, false}, + { + "exitError with nil inner (e.g. schema-violation already-printed)", + &exitError{code: 2, err: nil}, + true, + }, + { + "exitError with non-nil inner (e.g. file read failure)", + &exitError{code: 3, err: io_eof_or_similar()}, + false, + }, + {"plain error from cobra (e.g. unknown command)", errorString("unknown command"), false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsSilentError(c.err); got != c.want { + t.Errorf("IsSilentError(%v) = %v, want %v", c.err, got, c.want) + } + }) + } +} + +// errorString is the simplest possible error implementation, used +// in tests that need a "plain" error without any custom Unwrap +// behavior. Same as the stdlib errors.New() result; defined inline +// to keep the test file self-contained. +type errorString string + +func (e errorString) Error() string { return string(e) } + +func io_eof_or_similar() error { return errorString("read: file does not exist") } diff --git a/internal/cli/root.go b/internal/cli/root.go index 8ce9ac3..5fcf634 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -64,6 +64,7 @@ roadmap. Subsequent phases land subcommands incrementally.`, // Subcommands. New phases append here. root.AddCommand(newVersionCmd(info)) + root.AddCommand(newIngestCmd()) return root } diff --git a/internal/schema/embed.go b/internal/schema/embed.go new file mode 100644 index 0000000..9fa8f87 --- /dev/null +++ b/internal/schema/embed.go @@ -0,0 +1,30 @@ +// Package schema holds the embedded ingest schema(s) and the +// validator built on top of them. +// +// Today only v1 is supported. When a v2 lands, this package grows +// a SchemaVersion dispatch (per the ingest.yaml's apiVersion field) +// without changing the validator's public API. +// +// Why embed? Two reasons. +// +// 1. Local validation. `tracebloc ingest validate ` runs +// entirely offline — no cluster, no network — which is the +// whole point of doing it on the CLI rather than waiting for +// jobs-manager's POST-time check. +// 2. Drift detection. Bundling the schema makes drift between +// the CLI's view and data-ingestors' canonical source +// observable at build time, not at runtime. scripts/sync-schema.sh +// enforces parity in CI. +package schema + +import _ "embed" + +// V1Bytes is the raw JSON of ingest.v1.json, vendored from +// tracebloc/data-ingestors at build time via scripts/sync-schema.sh. +// +// Exposed as []byte (not parsed) so consumers can either pipe it +// into their preferred JSON-Schema implementation or inspect the +// raw $id / $schema fields to assert which version they got. +// +//go:embed ingest.v1.json +var V1Bytes []byte diff --git a/internal/schema/ingest.v1.json b/internal/schema/ingest.v1.json new file mode 100644 index 0000000..21a0c46 --- /dev/null +++ b/internal/schema/ingest.v1.json @@ -0,0 +1,380 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://tracebloc.io/schemas/ingest.v1.json", + "title": "tracebloc IngestConfig (v1)", + "description": "Declarative configuration for the tracebloc data ingestor. Replaces the per-customer Python script + Dockerfile pattern. Customers describe their dataset; the official image runs it.", + "type": "object", + "additionalProperties": false, + + "required": [ + "apiVersion", + "kind", + "category", + "table", + "intent" + ], + + "properties": { + "apiVersion": { + "const": "tracebloc.io/v1", + "description": "Schema version. Breaking changes require v2." + }, + "kind": { + "const": "IngestConfig" + }, + + "category": { + "type": "string", + "enum": [ + "image_classification", + "object_detection", + "keypoint_detection", + "semantic_segmentation", + "instance_segmentation", + "text_classification", + "tabular_classification", + "tabular_regression", + "time_series_forecasting", + "time_to_event_prediction", + "masked_language_modeling" + ], + "description": "Task category. Drives convention defaults: validators, data_format, default columns, default file extensions, default validator set." + }, + + "table": { + "type": "string", + "minLength": 1, + "description": "Destination table name in the cluster-internal MySQL. Used to identify the dataset in subsequent runs." + }, + + "intent": { + "type": "string", + "enum": ["train", "test"], + "description": "Whether this dataset is for training or held-out testing." + }, + + "csv": { + "type": "string", + "minLength": 1, + "description": "Path to the labels CSV. Mutually exclusive with `json`. The dominant source type." + }, + + "json": { + "type": "string", + "minLength": 1, + "description": "Path to a JSON dataset. Mutually exclusive with `csv`." + }, + + "images": { + "type": "string", + "minLength": 1, + "description": "Directory holding the image files referenced by the labels CSV. Required for image-based categories." + }, + + "annotations": { + "type": "string", + "minLength": 1, + "description": "Directory holding annotation files (e.g. Pascal VOC XML). Required for object_detection." + }, + + "masks": { + "type": "string", + "minLength": 1, + "description": "Directory holding segmentation mask images (PNG). Required for semantic_segmentation." + }, + + "texts": { + "type": "string", + "minLength": 1, + "description": "Directory holding text files referenced by the labels CSV. Required for text_classification." + }, + + "sequences": { + "type": "string", + "minLength": 1, + "description": "Directory holding sequence text files (.txt). Required for masked_language_modeling." + }, + + "label": { + "oneOf": [ + { + "type": "string", + "minLength": 1, + "description": "Shorthand: column name in the CSV that holds the label. Implies policy=passthrough." + }, + { + "type": "object", + "additionalProperties": false, + "required": ["column"], + "properties": { + "column": { + "type": "string", + "minLength": 1, + "description": "Column name in the CSV that holds the label." + }, + "policy": { + "type": "string", + "enum": ["passthrough", "bucket"], + "description": "How label values are sent to the central backend. `passthrough` (default for classification) sends the raw value. `bucket` (required for regression-class tasks) bins the value before send so the central backend never sees the raw target." + } + } + } + ], + "description": "Label specification. Can be a string (column name) for the common case, or an object for explicit policy control." + }, + + "schema": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + }, + "description": "Column → SQL type map (e.g. {'age': 'INT', 'name': 'VARCHAR(255)'}). Required for tabular and time-series categories." + }, + + "time_column": { + "type": "string", + "minLength": 1, + "description": "Name of the time column for time_to_event_prediction. Falls back to a column named `time` if unset." + }, + + "data_id": { + "type": "object", + "additionalProperties": false, + "required": ["strategy"], + "properties": { + "strategy": { + "type": "string", + "enum": ["uuid", "column"], + "default": "uuid", + "description": "How `data_id` is generated. `uuid` (default) generates a fresh UUID per record — no source column leaves the cluster. `column` maps a source column's value, which is louder (logged warning) and only safe when the column doesn't carry PII." + }, + "column": { + "type": "string", + "minLength": 1, + "description": "Required when strategy=column. The source column whose values become `data_id`." + } + }, + "if": { "properties": { "strategy": { "const": "column" } }, "required": ["strategy"] }, + "then": { "required": ["column"] } + }, + + "spec": { + "type": "object", + "additionalProperties": false, + "description": "Advanced overrides. Most customers don't touch this; convention defaults from `category` cover the dominant case.", + "properties": { + "csv_options": { + "type": "object", + "additionalProperties": false, + "properties": { + "chunk_size": { "type": "integer", "minimum": 1 }, + "delimiter": { "type": "string", "minLength": 1 }, + "quotechar": { "type": "string", "minLength": 1 }, + "escapechar": { "type": "string", "minLength": 1 } + }, + "description": "Pandas read_csv passthrough. Defaults: chunk_size=1000, delimiter=',', quotechar='\"', escapechar='\\\\'." + }, + "file_options": { + "type": "object", + "additionalProperties": true, + "properties": { + "target_size": { + "type": "array", + "items": { "type": "integer", "minimum": 1 }, + "minItems": 2, + "maxItems": 2, + "description": "[height, width]. Image categories only. Default [512, 512]." + }, + "extension": { + "type": "string", + "enum": [".jpg", ".jpeg", ".png", ".txt", ".text", ".xml"], + "description": "Allowed extension for sidecar files. Defaults: .jpg for image categories, .txt for text_classification." + } + } + }, + "validators": { + "type": "array", + "items": { "type": "string" }, + "description": "Override the default validator set resolved by `map_validators(category)`. Rarely needed." + }, + "sidecars": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["column", "source"], + "properties": { + "column": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "dest": { "type": "string", "minLength": 1 }, + "transform": { + "type": "object", + "additionalProperties": false, + "properties": { + "resize": { + "type": "array", + "items": { "type": "integer", "minimum": 1 }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "validators": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "description": "Advanced: explicit sidecar-file declaration when the convention defaults don't fit. Most users rely on `images`/`annotations`/`masks`/`texts` shorthand instead." + }, + "processors": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["script", "class"], + "properties": { + "script": { + "type": "string", + "minLength": 1, + "description": "Path inside the pod where the script body has been mounted (typically a ConfigMap mount under /custom/...)." + }, + "class": { + "type": "string", + "minLength": 1, + "description": "Class name within the script. Must subclass tracebloc_ingestor.processors.BaseProcessor." + }, + "args": { + "type": "object", + "description": "Keyword arguments passed to the processor's constructor." + } + } + }, + "description": "Custom-processor escape hatch. The Helm subchart (client#86) takes the script body as a chart value, writes a ConfigMap, mounts it. The official image dynamically imports the class and applies it to every record. No customer-built Docker image required." + } + } + } + }, + + "allOf": [ + { + "description": "Exactly one data source: csv or json.", + "oneOf": [ + { "required": ["csv"], "not": { "required": ["json"] } }, + { "required": ["json"], "not": { "required": ["csv"] } } + ] + }, + { + "description": "Image-based categories require `images`.", + "if": { + "properties": { + "category": { + "enum": [ + "image_classification", + "object_detection", + "keypoint_detection", + "semantic_segmentation", + "instance_segmentation" + ] + } + }, + "required": ["category"] + }, + "then": { "required": ["images"] } + }, + { + "description": "object_detection requires `annotations`.", + "if": { + "properties": { "category": { "const": "object_detection" } }, + "required": ["category"] + }, + "then": { "required": ["annotations"] } + }, + { + "description": "semantic_segmentation requires `masks`.", + "if": { + "properties": { "category": { "const": "semantic_segmentation" } }, + "required": ["category"] + }, + "then": { "required": ["masks"] } + }, + { + "description": "text_classification requires `texts`.", + "if": { + "properties": { "category": { "const": "text_classification" } }, + "required": ["category"] + }, + "then": { "required": ["texts"] } + }, + { + "description": "masked_language_modeling requires `sequences`.", + "if": { + "properties": { "category": { "const": "masked_language_modeling" } }, + "required": ["category"] + }, + "then": { "required": ["sequences"] } + }, + { + "description": "tabular and time-series categories require `schema`.", + "if": { + "properties": { + "category": { + "enum": [ + "tabular_classification", + "tabular_regression", + "time_series_forecasting", + "time_to_event_prediction" + ] + } + }, + "required": ["category"] + }, + "then": { "required": ["schema"] } + }, + { + "description": "Regression-class tasks require an explicit label.policy decision (must be the object form).", + "if": { + "properties": { + "category": { + "enum": [ + "tabular_regression", + "time_series_forecasting", + "time_to_event_prediction" + ] + } + }, + "required": ["category"] + }, + "then": { + "properties": { + "label": { + "type": "object", + "required": ["column", "policy"] + } + }, + "required": ["label"] + } + }, + { + "description": "Most categories require `label`.", + "if": { + "properties": { + "category": { + "enum": [ + "image_classification", + "object_detection", + "keypoint_detection", + "semantic_segmentation", + "instance_segmentation", + "text_classification", + "tabular_classification" + ] + } + }, + "required": ["category"] + }, + "then": { "required": ["label"] } + } + ] +} diff --git a/internal/schema/validate.go b/internal/schema/validate.go new file mode 100644 index 0000000..b71266e --- /dev/null +++ b/internal/schema/validate.go @@ -0,0 +1,238 @@ +package schema + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" + "golang.org/x/text/language" + "golang.org/x/text/message" + "gopkg.in/yaml.v3" +) + +// englishPrinter is the per-package message.Printer we pass to +// jsonschema's ErrorKind.LocalizedString. The library panics on a +// nil printer (it tries to construct one with a nil language tag); +// having a process-wide English printer avoids that and keeps the +// output language stable across customer environments. Internationalizing +// the validator output is a v2 concern. +var englishPrinter = message.NewPrinter(language.English) + +// V1SchemaID is the $id of the embedded v1 schema. Used as the +// "source URL" when compiling — jsonschema/v6 needs a name to +// associate the loaded bytes with, and the canonical $id keeps +// error messages anchored to the same identifier the Python +// implementation uses. +const V1SchemaID = "https://tracebloc.io/schemas/ingest.v1.json" + +// ValidationError is a single schema violation, normalized to the +// ">: " shape that +// tracebloc_ingestor.cli.run._format_errors emits in Python. Pinning +// the format lets us tell customers "the CLI and the server-side +// validator say the same thing about your YAML" with high +// confidence. +type ValidationError struct { + // Path is the json-pointer-style location, e.g. "spec.processors.0.script". + // "" for top-level violations (additionalProperties, + // missing required fields at the document level, etc.). + Path string + + // Message is the human-readable description of the violation, + // taken directly from the underlying jsonschema/v6 library's + // per-error message. The wording is stable across the library's + // minor versions; we don't post-process it. + Message string +} + +// Format returns the canonical " : " line, matching +// the indentation the Python implementation produces. The leading +// two spaces are deliberate — they match _format_errors so customer +// docs / runbooks can reference one wording across both +// implementations. +func (e ValidationError) Format() string { + return fmt.Sprintf(" %s: %s", e.Path, e.Message) +} + +// FormatErrors renders a slice of violations as one error per line, +// deterministically ordered by Path then Message. Mirrors +// _format_errors. +// +// Pure: the input slice's order is preserved. An earlier version +// called sort.Slice directly on errs, which silently reordered the +// caller's underlying array — bugbot caught this as a hidden side +// effect that contradicted the function's name + doc. Copying the +// slice header before sorting keeps FormatErrors a true formatter. +func FormatErrors(errs []ValidationError) string { + // slices.Clone would be cleaner but it's Go 1.21+; the manual + // copy is portable and the allocation is tiny relative to the + // schema validation itself. + sorted := make([]ValidationError, len(errs)) + copy(sorted, errs) + + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Path != sorted[j].Path { + return sorted[i].Path < sorted[j].Path + } + return sorted[i].Message < sorted[j].Message + }) + + var b strings.Builder + for i, e := range sorted { + if i > 0 { + b.WriteByte('\n') + } + b.WriteString(e.Format()) + } + return b.String() +} + +// Validator wraps a compiled jsonschema.Schema for repeated use. +// Compile once at process startup; validate many. Schema compilation +// is non-trivial (the v1 schema has nested if/then/oneOf chains); +// reusing the compiled form is meaningfully faster for batch +// validation. +type Validator struct { + schema *jsonschema.Schema +} + +// NewV1Validator compiles the embedded v1 schema. Returns an error +// only if the embedded JSON is malformed — which can only happen if +// scripts/sync-schema.sh wrote garbage, in which case the build's +// embed step or the CI drift check would have caught it. The error +// path stays for defense-in-depth. +func NewV1Validator() (*Validator, error) { + var raw any + if err := json.Unmarshal(V1Bytes, &raw); err != nil { + return nil, fmt.Errorf("embedded schema is not valid JSON: %w", err) + } + + c := jsonschema.NewCompiler() + if err := c.AddResource(V1SchemaID, raw); err != nil { + return nil, fmt.Errorf("registering embedded schema: %w", err) + } + s, err := c.Compile(V1SchemaID) + if err != nil { + return nil, fmt.Errorf("compiling embedded schema: %w", err) + } + return &Validator{schema: s}, nil +} + +// ValidateYAML parses the input as YAML and validates the resulting +// document against the schema. Returns the parsed document (useful +// for callers that want to inspect category/table/etc. after a +// successful validation) plus the list of violations. +// +// Two failure modes are distinct and important to separate: +// +// - The input isn't valid YAML at all (parseErr != nil): callers +// should surface this as a parse-level error, separately from +// schema violations. +// - The input parses but doesn't match the schema (errs non-empty): +// these are the customer-facing "your config has problems" cases. +func (v *Validator) ValidateYAML(input []byte) (parsed map[string]any, errs []ValidationError, parseErr error) { + if len(bytes.TrimSpace(input)) == 0 { + return nil, nil, fmt.Errorf("input is empty") + } + + var doc any + if err := yaml.Unmarshal(input, &doc); err != nil { + return nil, nil, fmt.Errorf("not valid YAML: %w", err) + } + + // The schema expects a mapping at the top. Anything else (a + // sequence, a scalar) gets surfaced as a parse-level error so + // the customer doesn't see a wall of unhelpful schema messages + // blaming the document's type. + asMap, ok := doc.(map[string]any) + if !ok { + return nil, nil, fmt.Errorf( + "document must be a YAML mapping at the top level (apiVersion / kind / category / ...); got %T", + doc, + ) + } + + // Schema validators expect canonical Go-native types + // (map[string]any, []any, string, float64, bool, nil). yaml.v3 + // produces those directly from Unmarshal-into-any, so no + // conversion needed. + if err := v.schema.Validate(doc); err != nil { + // Recurse the error tree into our flat ValidationError list. + // jsonschema/v6 returns a *ValidationError tree where each + // node may have child Causes — we flatten to leaves so the + // customer sees one line per actual problem, not an outline + // of the validator's traversal path. + var ve *jsonschema.ValidationError + if errors_as(err, &ve) { + errs = flattenValidationError(ve) + } else { + // Defensive: shouldn't happen with current jsonschema/v6, + // but fall back to the raw error string rather than + // crashing. + errs = []ValidationError{{Path: "", Message: err.Error()}} + } + } + + return asMap, errs, nil +} + +// flattenValidationError walks the jsonschema error tree to produce +// one leaf error per real violation. The library returns a tree +// because oneOf / anyOf / allOf can fail in multiple ways at once; +// we want each leaf surfaced individually so the customer can see +// every problem with a single validate run. +func flattenValidationError(ve *jsonschema.ValidationError) []ValidationError { + if ve == nil { + return nil + } + + // Internal nodes (with causes) carry the structural context; + // the actual violations live at the leaves. + if len(ve.Causes) > 0 { + var out []ValidationError + for _, c := range ve.Causes { + out = append(out, flattenValidationError(c)...) + } + return out + } + + return []ValidationError{{ + Path: instanceLocationToPath(ve.InstanceLocation), + Message: ve.ErrorKind.LocalizedString(englishPrinter), + }} +} + +// instanceLocationToPath converts jsonschema's slice-of-strings +// instance pointer into the dotted form data-ingestors uses +// (matching _format_errors's `".".join(...)`). An empty location +// becomes "" so the customer sees something concrete instead +// of a blank-looking error line. +func instanceLocationToPath(loc []string) string { + if len(loc) == 0 { + return "" + } + return strings.Join(loc, ".") +} + +// errors_as is a tiny inlining of errors.As to avoid the import +// just for this one site. Keeps the dependency graph of this file +// minimal; the only third-party imports stay jsonschema + yaml. +func errors_as(err error, target **jsonschema.ValidationError) bool { + for cur := err; cur != nil; cur = unwrap(cur) { + if ve, ok := cur.(*jsonschema.ValidationError); ok { + *target = ve + return true + } + } + return false +} + +func unwrap(err error) error { + type unwrapper interface{ Unwrap() error } + if u, ok := err.(unwrapper); ok { + return u.Unwrap() + } + return nil +} diff --git a/internal/schema/validate_test.go b/internal/schema/validate_test.go new file mode 100644 index 0000000..c09ae59 --- /dev/null +++ b/internal/schema/validate_test.go @@ -0,0 +1,405 @@ +package schema + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// Build the validator once; reuse across cases. Compilation cost +// shouldn't be in the critical path of every test. +func mustValidator(t *testing.T) *Validator { + t.Helper() + v, err := NewV1Validator() + if err != nil { + t.Fatalf("compile embedded schema: %v", err) + } + return v +} + +func TestEmbeddedSchemaCompiles(t *testing.T) { + // Compiling is the most basic invariant — if the bundled JSON + // is malformed, every other test fails too, but this gives the + // clearest first signal. + _ = mustValidator(t) +} + +func TestEmbeddedSchemaIsV1ByID(t *testing.T) { + // Spot-check that the bundled bytes are actually the v1 schema + // (not, say, a stale draft or a v2 that got pulled by accident). + // We grep on the canonical $id, not on schema structure, so a + // future field rename or example update doesn't break this test + // for spurious reasons. + if !strings.Contains(string(V1Bytes), V1SchemaID) { + t.Fatalf("embedded schema doesn't mention its expected $id %q", V1SchemaID) + } +} + +// Each of the canonical examples in tracebloc/data-ingestors must +// validate cleanly — if any fail, either the example is broken or +// the schema's drifted from what the examples assume. Pin them by +// inlining (the alternative is loading from an out-of-tree path +// which makes the test brittle to data-ingestors layout changes). +// +// The fixtures intentionally mirror examples/yaml/.yaml +// from data-ingestors. When that set grows, this slice grows too. +func TestValidate_HappyPath_AllCategories(t *testing.T) { + cases := []struct { + name string + yaml string + }{ + { + name: "image_classification", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: chest_xrays_train +intent: train +category: image_classification +csv: /data/labels.csv +images: /data/images/ +label: image_label +`, + }, + { + name: "object_detection", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: visdrone_train +intent: train +category: object_detection +csv: /data/labels.csv +images: /data/images/ +annotations: /data/annotations/ +label: image_label +`, + }, + { + name: "semantic_segmentation", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: tumors_train +intent: train +category: semantic_segmentation +csv: /data/labels.csv +images: /data/images/ +masks: /data/masks/ +label: image_label +`, + }, + { + name: "text_classification", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: support_tickets_train +intent: train +category: text_classification +csv: /data/labels.csv +texts: /data/texts/ +schema: + text_id: VARCHAR(255) + label: VARCHAR(64) +label: label +`, + }, + { + name: "tabular_classification", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: churn_train +intent: train +category: tabular_classification +csv: /data/customers.csv +schema: + age: INT + tenure_months: INT + churned: VARCHAR(8) +label: churned +`, + }, + { + name: "tabular_regression_requires_bucket_policy", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: house_prices_train +intent: train +category: tabular_regression +csv: /data/houses.csv +schema: + square_feet: FLOAT + price: FLOAT +label: + column: price + policy: bucket +`, + }, + } + + v := mustValidator(t) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, errs, parseErr := v.ValidateYAML([]byte(c.yaml)) + if parseErr != nil { + t.Fatalf("YAML parse failed: %v", parseErr) + } + if len(errs) > 0 { + t.Errorf("expected clean validation, got %d issue(s):\n%s", + len(errs), FormatErrors(errs)) + } + }) + } +} + +// Negative cases — each exercises a different schema rule. The +// assertion is on the path the error reports (not the exact message +// wording, which is jsonschema/v6's responsibility to define) so +// library version bumps don't break the test for cosmetic reasons. +func TestValidate_NegativeCases(t *testing.T) { + cases := []struct { + name string + yaml string + wantPaths []string // substring match on Path field of any violation + }{ + { + name: "missing apiVersion + intent", + yaml: ` +kind: IngestConfig +category: image_classification +table: t +csv: /data/labels.csv +images: /data/images/ +label: my_label +`, + wantPaths: []string{""}, // both go to root level + }, + { + name: "category not in enum", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: not_a_category +csv: /data/labels.csv +images: /data/images/ +label: my_label +`, + wantPaths: []string{"category"}, + }, + { + name: "image category missing images key", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: image_classification +csv: /data/labels.csv +label: my_label +`, + wantPaths: []string{""}, // missing required at root + }, + { + name: "regression without explicit label.policy", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: tabular_regression +csv: /data/houses.csv +schema: + price: FLOAT +label: price +`, + wantPaths: []string{"label"}, // schema enforces object form for regression + }, + { + name: "tabular missing schema block", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: tabular_classification +csv: /data/customers.csv +label: churned +`, + wantPaths: []string{""}, + }, + } + + v := mustValidator(t) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, errs, parseErr := v.ValidateYAML([]byte(c.yaml)) + if parseErr != nil { + t.Fatalf("YAML parse failed: %v", parseErr) + } + if len(errs) == 0 { + t.Fatalf("expected at least one schema violation, got none") + } + + gotPaths := make([]string, 0, len(errs)) + for _, e := range errs { + gotPaths = append(gotPaths, e.Path) + } + + for _, want := range c.wantPaths { + found := false + for _, got := range gotPaths { + if strings.Contains(got, want) { + found = true + break + } + } + if !found { + t.Errorf("expected a violation with path containing %q, got paths: %v\nfull errors:\n%s", + want, gotPaths, FormatErrors(errs)) + } + } + }) + } +} + +// Parse-level failures (vs schema violations) need a separate +// failure mode so callers can render them differently in the UI — +// "your file isn't YAML" vs "your file is YAML but doesn't match +// the schema" are different problems with different remediations. +func TestValidate_ParseFailures(t *testing.T) { + v := mustValidator(t) + + t.Run("empty input", func(t *testing.T) { + _, _, parseErr := v.ValidateYAML([]byte("")) + if parseErr == nil { + t.Fatal("expected parseErr on empty input, got nil") + } + if !strings.Contains(parseErr.Error(), "empty") { + t.Errorf("expected message to mention empty, got: %v", parseErr) + } + }) + + t.Run("whitespace only", func(t *testing.T) { + _, _, parseErr := v.ValidateYAML([]byte(" \n\t \n")) + if parseErr == nil { + t.Fatal("expected parseErr on whitespace-only input") + } + }) + + t.Run("non-mapping top level (sequence)", func(t *testing.T) { + _, _, parseErr := v.ValidateYAML([]byte("- one\n- two\n")) + if parseErr == nil { + t.Fatal("expected parseErr on top-level sequence") + } + if !strings.Contains(parseErr.Error(), "mapping") { + t.Errorf("expected message to mention 'mapping', got: %v", parseErr) + } + }) + + t.Run("malformed YAML", func(t *testing.T) { + // Trailing colon + missing value, mixed indent — yaml.v3 + // flags this. + _, _, parseErr := v.ValidateYAML([]byte("foo:\n bar:\n\tbaz: 1\n")) + if parseErr == nil { + t.Fatal("expected parseErr on malformed YAML") + } + }) +} + +// FormatErrors is the contract with the UI layer; pin its exact +// output shape so any change is intentional. The leading two +// spaces mirror tracebloc_ingestor.cli.run._format_errors so +// customers see one wording across both implementations. +func TestFormatErrors_ContractShape(t *testing.T) { + errs := []ValidationError{ + {Path: "category", Message: "value must be one of …"}, + {Path: "", Message: "missing properties 'foo'"}, + } + got := FormatErrors(errs) + + // Sorted output: < category alphabetically. + want := " : missing properties 'foo'\n category: value must be one of …" + if got != want { + t.Errorf("FormatErrors mismatch\ngot:\n%s\nwant:\n%s", got, want) + } +} + +// Pins that FormatErrors does NOT mutate its input. An earlier +// version sorted in place, which made callers that wanted to +// further process `violations` after formatting see a different +// order from the one they passed in. Bugbot flagged this as a +// hidden side effect; this test ensures the regression can't +// silently come back. +func TestFormatErrors_DoesNotMutateInput(t *testing.T) { + in := []ValidationError{ + {Path: "z_field", Message: "bad"}, + {Path: "a_field", Message: "bad"}, + } + original := append([]ValidationError(nil), in...) // capture pre-call order + + _ = FormatErrors(in) + + if in[0] != original[0] || in[1] != original[1] { + t.Errorf("FormatErrors mutated input order.\nbefore: %v\nafter: %v", + original, in) + } +} + +// Round-trip test against the canonical example YAMLs in the +// data-ingestors repo. If that checkout is missing, skip — we +// don't want CI to fail for a local layout mismatch. CI runs the +// test in the runner's filesystem where data-ingestors isn't +// checked out alongside us, so this only fires for local devs. +// +// Opt in by setting TRACEBLOC_INGESTORS_EXAMPLES to the absolute +// path of data-ingestors/examples/yaml (or any directory of +// `.yaml` files that follow the v1 schema). Without the env var +// the test skips silently — a hardcoded absolute path like +// `/Volumes/VPPD/...` was bugbot-flagged as leaking one +// developer's filesystem layout and being unusable by anyone else. +func TestValidate_AgainstRealExamplesIfPresent(t *testing.T) { + examplesDir := os.Getenv("TRACEBLOC_INGESTORS_EXAMPLES") + if examplesDir == "" { + t.Skip("TRACEBLOC_INGESTORS_EXAMPLES not set; skipping the round-trip-against-real-examples test " + + "(point it at data-ingestors/examples/yaml to enable)") + } + entries, err := os.ReadDir(examplesDir) + if err != nil { + t.Skipf("TRACEBLOC_INGESTORS_EXAMPLES=%q is not readable, skipping: %v", examplesDir, err) + } + + v := mustValidator(t) + for _, e := range entries { + if e.IsDir() || filepath.Ext(e.Name()) != ".yaml" { + continue + } + t.Run(e.Name(), func(t *testing.T) { + body, err := os.ReadFile(filepath.Join(examplesDir, e.Name())) + if err != nil { + t.Fatalf("read: %v", err) + } + _, errs, parseErr := v.ValidateYAML(body) + if parseErr != nil { + t.Fatalf("parse: %v", parseErr) + } + if len(errs) > 0 { + // custom_processor.yaml may legitimately not match + // (the example shows a feature deferred to v1.1); + // skip rather than fail so the test stays useful as + // the canonical examples evolve. + if strings.Contains(e.Name(), "custom_processor") { + t.Skipf("custom_processor example exercises deferred features; %d issue(s)", + len(errs)) + } + t.Errorf("expected clean, got %d issue(s):\n%s", + len(errs), FormatErrors(errs)) + } + }) + } +} diff --git a/scripts/sync-schema.sh b/scripts/sync-schema.sh new file mode 100755 index 0000000..a7f13d4 --- /dev/null +++ b/scripts/sync-schema.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Sync ingest.v1.json from tracebloc/data-ingestors into the CLI's +# embedded copy at internal/schema/ingest.v1.json. +# +# The CLI validates locally using this schema. Drift between the +# CLI's copy and data-ingestors' canonical source is a real +# correctness hazard — a customer's YAML could pass `tracebloc ingest +# validate` locally but be rejected by jobs-manager (or vice versa). +# +# Run this script when bumping the schema version. CI invokes it in +# check-mode (`--check`) to fail builds that have drifted without +# running the sync first. +# +# Usage: +# scripts/sync-schema.sh # write into internal/schema/ +# scripts/sync-schema.sh --check # verify in-tree copy matches upstream; exit non-zero on drift +# +# Env knobs: +# SCHEMA_SOURCE_URL override the upstream URL (default: data-ingestors' master) +# SCHEMA_OUT override the in-tree destination (default: internal/schema/ingest.v1.json) +# +# Future: when we cut a v2 schema, this script will need to learn +# about multiple versions (e.g. embed v1 AND v2 side-by-side, picked +# at runtime based on the ingest.yaml's apiVersion). Keeping the path +# explicit in SCHEMA_OUT makes that extension easy. + +set -euo pipefail + +readonly DEFAULT_URL="https://raw.githubusercontent.com/tracebloc/data-ingestors/master/tracebloc_ingestor/schema/ingest.v1.json" +readonly DEFAULT_OUT="internal/schema/ingest.v1.json" + +SCHEMA_SOURCE_URL="${SCHEMA_SOURCE_URL:-$DEFAULT_URL}" +SCHEMA_OUT="${SCHEMA_OUT:-$DEFAULT_OUT}" + +CHECK_MODE=false +if [[ "${1:-}" == "--check" ]]; then + CHECK_MODE=true +fi + +# Stage the fetched schema in a temp file so a half-failed curl doesn't +# leave a truncated file in the repo. +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT + +echo "==> fetching $SCHEMA_SOURCE_URL" +curl -fsSL "$SCHEMA_SOURCE_URL" -o "$tmp" + +# Make sure what came back is valid JSON before we trust it. +if ! python3 -m json.tool < "$tmp" > /dev/null 2>&1; then + echo "error: upstream response is not valid JSON" >&2 + echo "first 200 bytes of response:" >&2 + head -c 200 "$tmp" >&2 + exit 2 +fi + +mkdir -p "$(dirname "$SCHEMA_OUT")" + +if $CHECK_MODE; then + if [[ ! -f "$SCHEMA_OUT" ]]; then + echo "error: $SCHEMA_OUT does not exist" >&2 + echo "run \`scripts/sync-schema.sh\` (without --check) to seed it." >&2 + exit 1 + fi + if ! diff -q "$tmp" "$SCHEMA_OUT" >/dev/null; then + echo "error: $SCHEMA_OUT has drifted from upstream." >&2 + echo "diff (upstream → in-tree):" >&2 + diff -u "$SCHEMA_OUT" "$tmp" | head -40 >&2 || true + echo >&2 + echo "to fix, run \`scripts/sync-schema.sh\` and commit the result." >&2 + exit 1 + fi + echo "==> $SCHEMA_OUT matches upstream — no drift" + exit 0 +fi + +# Write mode. Only touch the destination if the content actually +# changed, so re-running the script on an already-current schema +# produces no file-mtime churn. +if [[ -f "$SCHEMA_OUT" ]] && diff -q "$tmp" "$SCHEMA_OUT" >/dev/null; then + echo "==> $SCHEMA_OUT already matches upstream — no change" + exit 0 +fi + +mv "$tmp" "$SCHEMA_OUT" +trap - EXIT # the temp file is now in place; don't try to rm it +echo "==> wrote $SCHEMA_OUT ($(wc -c < "$SCHEMA_OUT" | tr -d ' ') bytes)" From 57bd6d5a0a25bbea5f755c8216cd9e4eaab0101d Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Thu, 21 May 2026 21:37:55 +0500 Subject: [PATCH 02/17] Phase 2: kubeconfig discovery + parent release detection + SA token (#2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(cluster): kubeconfig discovery, parent release detection, SA token Phase 2 of the v0.1 roadmap. Adds the plumbing the future `tracebloc dataset push` flow needs: where does the customer's kubeconfig point, which tracebloc release lives there, and how do we authenticate to its jobs-manager. End-to-end validated against the dev EKS cluster (tb-client-dev-templates): discovers chart 1.3.5, resolves jobs-manager.tracebloc-templates.svc:8080, reads INGESTOR_IMAGE_DIGEST out of the deployment env, mints a 10-minute SA token via TokenRequest. What lands: - internal/cluster/kubeconfig.go — Load() that honors --kubeconfig, $KUBECONFIG, ~/.kube/config (via clientcmd's full default loading rules — *not* an empty ExplicitPath, which silently refuses to fall back to defaults; that was the first bug the real-cluster smoke caught). - internal/cluster/discover.go — DiscoverParentRelease() finds the tracebloc/client release in a namespace by listing chart-managed Deployments and filtering by name suffix (-jobs-manager). The chart shares app.kubernetes.io/name=client across mysql/jobs- manager/requests-proxy, so suffix matching is what distinguishes jobs-manager. Returns a friendly multi-release error when ambiguous, with remediation text in the message. - internal/cluster/token.go — MintIngestorToken() tries the modern TokenRequest path first, falls back to a static service-account-token Secret on RBAC denial / older clusters / SA missing. Errors propagate verbatim on non-recoverable failures (network, context cancellation) so customers see the real problem instead of a misleading "static fallback also failed." - internal/cli/cluster.go — `tracebloc cluster info` command. Prints context, server, namespace, parent release info, SA + token state (with SHA256(token)[:8] instead of the raw bytes — token must never appear in scrollback). Exit codes 3 (kubeconfig issue) / 4 (no parent release) / 5 (token mint failed). Tests: - 5 new test files covering happy path, multi-release ambiguity, service name fallback, the sibling-deployment filter regression (mysql + requests-proxy + jobs-manager all share chart-level labels — discovery must pick jobs-manager by name suffix), TokenRequest happy path, static-secret fallback, non-recoverable error pass-through, and the combined-failure remediation message. Coverage: internal/cluster 83.8%, internal/cli 59.5% (cluster info itself is hard to unit-test without a real cluster — Phase 3+ adds integration tests against a kind cluster). Library footprint: brings in k8s.io/client-go + apimachinery + api (@v0.31.0) and sigs.k8s.io/yaml. Cross-compiled binaries grow from ~10MB to ~30MB; cost is acceptable for the customer-experience upside of "your kubeconfig is all you need." Closes tracebloc/client#150. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(ci): resolve Go version + dep conflicts for lint to typecheck CI on PR #2 hit a lint typecheck error ("undefined: jsonschema / yaml (typecheck)") that took a few iterations to diagnose. Root cause: the k8s.io/client-go v0.31 line pulls in transitive deps (sigs.k8s.io/structured-merge-diff/v6 vs v4) that fight in go.mod, and golangci-lint v1.61's bundled Go SDK can't typecheck a module whose `go` directive is newer than what the linter supports. Resolution: 1. Bump k8s.io/client-go + api + apimachinery from v0.31.0 to v0.36.1 (latest stable). Fixes the structured-merge-diff version split — v0.36 uses v6 consistently across the dependency chain. 2. Accept whatever `go mod tidy` writes to go.mod's `go` directive (currently 1.26 on this dev machine, 1.24 on others — same either way since Go modules are forward-compatible). Stop fighting tidy; pinning a stale version produces typecheck errors instead of real findings. 3. Bump golangci-lint in the workflow from v1.61.0 to v1.64.7, the first version that handles the Go 1.24+ source the dep tree now requires. 4. Update .golangci.yml `run.go: "1.24"` to match go.mod's effective minimum. 5. Refresh the go.mod comment so future readers understand why the version directive isn't pinned low. Local validation: `make vet test fmt-check schema-check` all green; cluster-info smoke against the dev EKS still discovers chart 1.3.5 + mints a TokenRequest token. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: trim linters that whole-program-analyze the k8s.io dep tree CI's Lint job has now failed three runs in a row with "The runner has received a shutdown signal" after ~2 minutes of golangci-lint actually running. Not a flaky runner — reproducible. Root cause: `staticcheck` and `unused` do full-program SSA analysis across every transitive dep. With k8s.io/client-go + apimachinery + api in the graph (≈80 indirect modules), that exceeds whatever budget the standard 4-CPU GitHub-hosted runner allots for the job. The runner gets preempted before lint completes. Drop `staticcheck` and `unused` from the active linter set. Keep the cheap per-file linters that catch the bugs we've actually hit this week (errcheck, govet, ineffassign, gofmt, goimports, misspell, unconvert). Filed v0.2 ticket to bring them back via either (a) a self-hosted larger runner, (b) `-skip-files=k8s.io/*` patterns that don't exist in golangci-lint v1.64 but do in v2.x, or (c) split the lint job to run staticcheck only on `./internal/...` (our own code) and skip module-cache packages. The dropped linters' value relative to CI cycles spent debugging this: - staticcheck SA-checks are valuable but redundant with `govet` for the most-likely-to-bite cases (printf, lock copies, etc.) - `unused` rarely fires on a brand-new codebase where every symbol is just-introduced. Pragmatic tradeoff for v0.1. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: replace golangci-lint-action with standalone errcheck + gofmt -s Four consecutive Lint failures on PR #2, all hitting the same "shutdown signal received" at ~2 minutes (15:17, 15:24, 15:33, 15:35). Not lint config (the trim from staticcheck/unused didn't help), not a flaky runner (reproducible), not an OOM (no resource warning). golangci-lint-action@v6 + the k8s.io/* dep tree appears to be the incompatible combination in early 2026's GitHub Actions environment. Rather than spend another iteration debugging the action, replace it with standalone tools that already work locally and have predictable behavior: - errcheck v1.7.0 (the bug class we've actually hit this week) - gofmt -s (the formatting check; matches what `make fmt-check` does) - ineffassign v0.1 (cheap dead-assignment detection) - misspell (typo guard) Combined runtime in standalone mode: ~10s. golangci-lint's value-add beyond these was staticcheck + unused — both already deferred to #6 as v0.2 work pending a strategy for the dep tree. Co-Authored-By: Claude Opus 4.7 (1M context) * ci: unpin lint-tool versions; their old tags don't build with current Go errcheck v1.7.0 + ineffassign v0.1.0 + misspell v0.3.4 all transitively import a golang.org/x/tools version (v0.17 era) that fails to compile under current Go ("invalid array length -delta * delta" in tokeninternal.go). Pinning was the right instinct for reproducibility, but the upstream tools haven't shipped current-Go-compat tags yet. Use @latest for now; reproducibility tradeoff is acceptable given these are lint tools, not runtime deps. Document in #6 as "pin once upstream tags newer versions" follow-up. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(lint): explicit-discard 20 unchecked fmt.Fprintf in cluster.go errcheck caught 20 unchecked Fprintf/Fprintln returns in runClusterInfo — same class of finding that was previously caught + fixed in internal/cli/ingest.go and cmd/tracebloc/main.go. I missed cluster.go when I added the explicit-discard pattern there. Same rationale as the other sites: the exit code is the contract; a pipe-write failure shouldn't convert a successful diagnostic into a non-zero exit. Wrap each call with `_, _ =`. Now caught by CI thanks to the standalone-errcheck swap from the previous commit. The whole reason for the lint-job rework was to catch this exact bug class earlier in the loop — we just had to trade the golangci-lint-action for a working setup first. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(lint): explicit-discard the stderr Fprintln in main.go too Preempts the next errcheck cycle. Same explicit-discard rationale as the cluster.go fixes — stderr unreachable shouldn't change the exit code we propagate. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(cluster): admit hardcoded ingestor SA name + add --ingestor-sa override Bugbot caught a contradiction in PR #2's Phase 2 code: - The comment said "Read INGESTOR_IMAGE_DIGEST + ingestor SA name from the Deployment's pod-spec env" - The struct doc said "customers can override; the value comes from jobs-manager's environment" - But the switch only handled INGESTOR_IMAGE_DIGEST. SA name was hardcoded to "ingestor", silently ignoring any customer override. Truthful fix: 1. Update the comment and struct doc to admit the limitation. 2. Add a `--ingestor-sa` flag on `cluster info` so customers who set `ingestionAuthz.serviceAccountName` to a non-default value in the parent client chart can still use the CLI today. 3. Plumb the override through `runClusterInfo` -> applied to the discovered ParentRelease before token mint. 4. Drop the now-only-INGESTOR_IMAGE_DIGEST switch statement to a plain `if` — clearer, errcheck-friendlier, and signals there's only one env var being read. File #7 in tracebloc/cli for the proper fix: discover the SA name from the chart-rendered ingestionAuthz ConfigMap so the flag becomes unnecessary. v0.2 work, not blocking Phase 2 ship. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(cluster): defer to apierrors stdlib; close go.mod/lint Go-version drift Cursor Bugbot's re-review on 123b56a flagged two new issues in the Phase 2 (#150) tree: 1. internal/cluster/token.go reimplemented isForbidden / isNotFound / isMethodNotSupported / statusCode against Status.Code (numeric HTTP code) when k8s.io/apimachinery/pkg/api/errors already exports IsForbidden / IsNotFound / IsMethodNotSupported that key off Status.Reason (the typed enum). The two can diverge silently for non-standard status errors. The test file already imports apierrors and constructs fake errors via apierrors.NewForbidden(), so deferring to the stdlib is both safer AND removes ~20 lines of homegrown code. The four token tests still pass at 82.1% pkg coverage because NewForbidden() sets both Code and Reason fields. 2. go.mod's top-of-file comment claimed "Minimum Go is 1.22" but the actual `go 1.26.0` directive (forced by k8s.io/* v0.36.x deps) contradicted it, and .golangci.yml pinned `go: "1.24"` — also stale. Rewrote the go.mod comment to admit reality + tell future-me to bump both together, and bumped the lint config to "1.26" to match. The third inline comment from Bugbot's re-review is a stale carry-over of the SA-name finding fixed in 123b56a (same bug ID 5e4b5df0…, GitHub auto-shifted its anchor onto the new lines). Bugbot's own review-body count confirms 2 new findings, not 3. Local: go vet, go test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/build.yml | 46 +++++- .golangci.yml | 23 ++- cmd/tracebloc/main.go | 6 +- go.mod | 48 +++++- go.sum | 116 ++++++++++++++- internal/cli/cluster.go | 202 +++++++++++++++++++++++++ internal/cli/root.go | 1 + internal/cluster/discover.go | 211 ++++++++++++++++++++++++++ internal/cluster/discover_test.go | 222 ++++++++++++++++++++++++++++ internal/cluster/kubeconfig.go | 164 ++++++++++++++++++++ internal/cluster/kubeconfig_test.go | 84 +++++++++++ internal/cluster/token.go | 214 +++++++++++++++++++++++++++ internal/cluster/token_test.go | 186 +++++++++++++++++++++++ 13 files changed, 1503 insertions(+), 20 deletions(-) create mode 100644 internal/cli/cluster.go create mode 100644 internal/cluster/discover.go create mode 100644 internal/cluster/discover_test.go create mode 100644 internal/cluster/kubeconfig.go create mode 100644 internal/cluster/kubeconfig_test.go create mode 100644 internal/cluster/token.go create mode 100644 internal/cluster/token_test.go diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82ca229..2174836 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,14 +63,44 @@ jobs: go-version-file: go.mod cache: true - - name: golangci-lint - uses: golangci/golangci-lint-action@v6 - with: - version: v1.61.0 - # Pin the cache key per Go version so a Go bump invalidates - # cleanly — stale caches across Go versions cause noisy - # "invalid pkg" failures. - args: --timeout=5m + # golangci-lint-action@v6 reproducibly time-budgeted out at + # ~2 minutes on this module's k8s.io-heavy dep tree (3 runs + # in a row in early 2026 — see #6). Stand-alone tools run + # against the same code in <30 seconds and catch the bugs + # we've actually hit this week (unchecked Fprintf errors + # were errcheck-catchable; gofmt -s drift was a recurring + # nit). Going back to the action when #6 has a strategy. + + # Tool versions are unpinned (@latest) because older tagged + # releases pulled stale golang.org/x/tools that fail to build + # against current Go ("invalid array length -delta * delta"). + # Pinning is preferable for reproducibility but waiting on the + # upstream tools to ship newer tags. Track in #6. + + - name: errcheck + run: | + go install github.com/kisielk/errcheck@latest + errcheck ./... + + - name: gofmt -s + run: | + drift="$(gofmt -s -l .)" + if [ -n "$drift" ]; then + echo "::error::gofmt -s drift in:" + echo "$drift" | sed 's/^/ /' + echo "::error::run \`make fmt\` to fix" + exit 1 + fi + + - name: ineffassign + run: | + go install github.com/gordonklaus/ineffassign@latest + ineffassign ./... + + - name: misspell + run: | + go install github.com/client9/misspell/cmd/misspell@latest + misspell -error . build: name: Build (${{ matrix.os }}/${{ matrix.arch }}) diff --git a/.golangci.yml b/.golangci.yml index 7763fa4..a6d52e7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,17 +7,32 @@ run: timeout: 5m - go: "1.22" + # Track go.mod's `go` directive. Go's release cadence + `go mod + # tidy`'s aggressive bumping (especially when k8s.io/* deps want + # newer Go) keep dragging go.mod's minimum up; pinning a stale + # version here just produces typecheck errors instead of real + # findings. Currently 1.26 to match `go 1.26.0` in go.mod (the + # earlier "1.24" pin drifted when k8s.io/* v0.36 bumped the floor; + # Bugbot caught the drift on PR #2). + go: "1.26" linters: disable-all: true enable: - # Standard set — these catch real bugs and have very low false-positive rates. + # The set is trimmed to the linters that DON'T do full whole-program + # SSA analysis. `staticcheck` and `unused` were in the original set + # and reproducibly caused the GitHub-hosted runner to time-budget + # the job (~2 min then shutdown signal) on this module — k8s.io/* + # transitive deps inflate the analysis graph enough to OOM-or-stall + # the standard 4-CPU/16GB runner. Re-enabling them is a v0.2 + # follow-up that needs either a larger runner, a much narrower + # scope (e.g. only `./internal/...`), or a faster successor like + # govulncheck for the security-only subset. + + # Cheap, per-file checks — catch real bugs without SSA. - errcheck # unchecked error returns - govet # `go vet` - ineffassign # assignments that go nowhere - - staticcheck # SA-series checks (well-vetted) - - unused # dead code # Style / hygiene that pays for itself in code review time. - gofmt diff --git a/cmd/tracebloc/main.go b/cmd/tracebloc/main.go index a38a446..8066305 100644 --- a/cmd/tracebloc/main.go +++ b/cmd/tracebloc/main.go @@ -57,7 +57,11 @@ func main() { // "silent" by returning an exitError with a nil inner — see // cli.IsSilentError for the contract. if !cli.IsSilentError(err) { - fmt.Fprintln(os.Stderr, "Error:", err) + // Explicit-discard the writer error: if stderr itself is + // gone (closed pipe, redirected to /dev/full, etc.) we + // still need to exit non-zero — the error message is + // best-effort, the exit code is the contract. + _, _ = fmt.Fprintln(os.Stderr, "Error:", err) } // Map command-defined exit codes through. Handlers that want a diff --git a/go.mod b/go.mod index 8c5a9db..c792eb3 100644 --- a/go.mod +++ b/go.mod @@ -1,15 +1,57 @@ module github.com/tracebloc/cli -go 1.22 +// Minimum Go is what `go 1.26.0` says below — this used to claim +// "1.22" but the k8s.io/* v0.36.x deps dragged the floor up to 1.26 +// (transitively via `go mod tidy`), and the stale comment misled +// Bugbot and humans alike. The lint config in .golangci.yml's `go:` +// key tracks this directive; bump both together when the minimum +// moves again. +go 1.26.0 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 github.com/spf13/cobra v1.8.1 - golang.org/x/text v0.16.0 + golang.org/x/text v0.33.0 gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.36.1 + k8s.io/apimachinery v0.36.1 + k8s.io/client-go v0.36.1 ) require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/term v0.39.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 3cf027a..9a05a37 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,126 @@ github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= -golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= +k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= +k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= +k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= +k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= +k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/cli/cluster.go b/internal/cli/cluster.go new file mode 100644 index 0000000..80a7172 --- /dev/null +++ b/internal/cli/cluster.go @@ -0,0 +1,202 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" + + "github.com/spf13/cobra" + + "github.com/tracebloc/cli/internal/cluster" +) + +// newClusterCmd wires the `tracebloc cluster` subtree. Today it has +// a single verb — `info` — which is the customer's "is the CLI +// pointing at the right cluster?" pre-flight before running +// `dataset push`. Future verbs (e.g. `cluster doctor` for +// diagnostics, `cluster contexts` for switching) hang off this +// parent in later phases. +func newClusterCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "cluster", + Short: "Inspect the cluster the CLI is currently targeting", + Long: `Commands for inspecting the Kubernetes cluster the CLI is +configured to talk to. + +Use ` + "`cluster info`" + ` to verify which cluster, namespace, and parent +tracebloc release the next ` + "`dataset push`" + ` will target. Useful as a +pre-flight before doing anything destructive (e.g. ingesting into +the wrong cluster).`, + } + + cmd.AddCommand(newClusterInfoCmd()) + return cmd +} + +// newClusterInfoCmd implements `tracebloc cluster info`. Discovers +// the parent client release in the configured namespace, mints (or +// looks up) an ingestor SA token, and prints a single-screen +// summary the customer can sanity-check before running a real +// ingestion. +// +// Three kubeconfig flags follow kubectl conventions: +// +// --kubeconfig=PATH override KUBECONFIG / ~/.kube/config +// --context=NAME override the kubeconfig's current-context +// --namespace=NAME override the context's default namespace +// +// All three are zero-value-safe — running `tracebloc cluster info` +// with no flags uses the customer's normal kubectl defaults. +func newClusterInfoCmd() *cobra.Command { + var ( + kubeconfigPath string + contextOverride string + nsOverride string + ingestorSAName string + tokenExpiry int64 + ) + + cmd := &cobra.Command{ + Use: "info", + Short: "Show the cluster, namespace, parent release, and ingestor SA token state", + Long: `Discovers the tracebloc parent client release in the configured +cluster + namespace and prints: + + • Which kubeconfig context the CLI used + • The namespace it resolved to + • The parent release name + chart version + appVersion + • The jobs-manager Service the next dataset push would POST to + • The ingestor ServiceAccount the post-install hook would auth as + • The cluster's configured INGESTOR_IMAGE_DIGEST default + • Whether the user's kubeconfig can mint short-lived SA tokens + via TokenRequest, or has to fall back to a static + service-account-token Secret + +The actual token bytes are never printed; the diagnostic shows +SHA256(token)[:8] so the customer can verify "yes, that's the +token I expect" without leaking it to terminal scrollback. + +Exit codes: + 0 cluster discovered + token mintable; CLI is ready + 4 cluster reachable but no tracebloc parent release found + 5 cluster reachable + release found but no usable SA token`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runClusterInfo( + cmd.Context(), + cmd.OutOrStdout(), + kubeconfigPath, contextOverride, nsOverride, + ingestorSAName, + tokenExpiry, + ) + }, + } + + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", "", + "path to the kubeconfig file (default: $KUBECONFIG, then ~/.kube/config)") + cmd.Flags().StringVar(&contextOverride, "context", "", + "name of the kubeconfig context to use (default: kubeconfig's current-context)") + cmd.Flags().StringVarP(&nsOverride, "namespace", "n", "", + "namespace where the parent tracebloc/client release is installed (default: the context's namespace, or 'default')") + cmd.Flags().StringVar(&ingestorSAName, "ingestor-sa", "", + "override the ingestor ServiceAccount name (default: \"ingestor\", the chart default; "+ + "set this if you customized `ingestionAuthz.serviceAccountName` in the parent client chart)") + cmd.Flags().Int64Var(&tokenExpiry, "token-expiry-seconds", 600, + "requested SA token expiration in seconds (default 600 = 10 min; ignored for static-secret fallback)") + + return cmd +} + +func runClusterInfo( + ctx context.Context, + out interface{ Write([]byte) (int, error) }, + kubeconfigPath, contextOverride, nsOverride string, + ingestorSAOverride string, + tokenExpiry int64, +) error { + resolved, err := cluster.Load(cluster.KubeconfigOptions{ + Path: kubeconfigPath, + Context: contextOverride, + Namespace: nsOverride, + }) + if err != nil { + // Kubeconfig errors are exit-code-3 territory (file/parse + // problem, same conceptual class as `ingest validate`'s + // unreadable-input). + return &exitError{code: 3, err: fmt.Errorf("loading kubeconfig: %w", err)} + } + + cs, err := cluster.NewClientset(resolved) + if err != nil { + return &exitError{code: 3, err: err} + } + + // Explicit-discard the writer errors: same rationale as + // internal/cli/ingest.go — the exit code is the contract, and a + // pipe-write failure shouldn't convert success into failure. If + // the downstream consumer disappears mid-write we still finish + // cleanly. errcheck wants this acknowledged. + _, _ = fmt.Fprintf(out, "Kubeconfig:\n") + _, _ = fmt.Fprintf(out, " context: %s\n", resolved.Context) + _, _ = fmt.Fprintf(out, " server: %s\n", resolved.ServerURL) + _, _ = fmt.Fprintf(out, " namespace: %s\n", resolved.Namespace) + _, _ = fmt.Fprintln(out) + + // Discover the parent release. + release, err := cluster.DiscoverParentRelease(ctx, cs, resolved.Namespace) + if err != nil { + // 4 = "cluster reachable, but no tracebloc release here." + // Distinct from the kubeconfig error (3) so callers can + // branch: 3 means "fix your kubeconfig", 4 means "install + // the parent chart first". + return &exitError{code: 4, err: err} + } + + // Apply the SA-name override here. Discovery doesn't read the + // name from the cluster (see #7); customers with a non-default + // name pass --ingestor-sa. + if ingestorSAOverride != "" { + release.IngestorSAName = ingestorSAOverride + } + + _, _ = fmt.Fprintf(out, "Parent release:\n") + _, _ = fmt.Fprintf(out, " name: %s\n", release.ReleaseName) + _, _ = fmt.Fprintf(out, " chart version: %s\n", release.ChartVersion) + _, _ = fmt.Fprintf(out, " app version: %s\n", release.AppVersion) + _, _ = fmt.Fprintf(out, " jobs-manager: %s\n", release.JobsManagerService) + _, _ = fmt.Fprintf(out, " ingestor SA: %s/%s\n", resolved.Namespace, release.IngestorSAName) + digest := release.IngestorImageDigest + if digest == "" { + digest = "" + } + _, _ = fmt.Fprintf(out, " ingestor img: %s\n", digest) + _, _ = fmt.Fprintln(out) + + // Mint a token (or fall back). The audience is intentionally + // nil today — jobs-manager's TokenReview accepts the default + // API-server audience. A future jobs-manager audience plugs in + // here. + tok, err := cluster.MintIngestorToken(ctx, cs, resolved.Namespace, release.IngestorSAName, tokenExpiry, nil) + if err != nil { + // 5 = "release found but no usable token." Distinct from + // 4 (no release) so customers can RBAC-debug separately + // from install issues. + return &exitError{code: 5, err: err} + } + + hash := sha256.Sum256([]byte(tok.Token)) + _, _ = fmt.Fprintf(out, "Ingestor SA token:\n") + _, _ = fmt.Fprintf(out, " source: %s\n", tok.Source) + _, _ = fmt.Fprintf(out, " sha256[:8]: %s\n", hex.EncodeToString(hash[:8])) + if tok.ExpirationSeconds > 0 { + exp := time.Duration(tok.ExpirationSeconds) * time.Second + _, _ = fmt.Fprintf(out, " expires in: ~%s (server may cap shorter)\n", exp) + } else { + _, _ = fmt.Fprintf(out, " expires in: never (static-secret fallback)\n") + } + _, _ = fmt.Fprintln(out) + _, _ = fmt.Fprintln(out, "Ready for `tracebloc dataset push` (coming in Phase 3).") + return nil +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 5fcf634..0d790ae 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -65,6 +65,7 @@ roadmap. Subsequent phases land subcommands incrementally.`, // Subcommands. New phases append here. root.AddCommand(newVersionCmd(info)) root.AddCommand(newIngestCmd()) + root.AddCommand(newClusterCmd()) return root } diff --git a/internal/cluster/discover.go b/internal/cluster/discover.go new file mode 100644 index 0000000..7ee34c7 --- /dev/null +++ b/internal/cluster/discover.go @@ -0,0 +1,211 @@ +package cluster + +import ( + "context" + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// ParentRelease describes the tracebloc parent client chart release +// discovered in the customer's cluster. The information comes from +// the helm-managed Deployment's labels (and a few of its env vars), +// not from the helm release secret — parsing those secrets requires +// either pulling in helm.sh/helm/v3 (massive dep) or re-implementing +// the gzip+base64+JSON unwrap. The chart's Deployment labels carry +// everything we actually need, so we use those instead. +type ParentRelease struct { + // ReleaseName is the helm release name. The chart's + // jobs-manager Deployment is named "-jobs-manager" by + // convention and labelled `app.kubernetes.io/instance=`. + ReleaseName string + + // ChartVersion is the parent chart's version, e.g. "1.3.5". + // Comes from the `helm.sh/chart=client-1.3.5` label. + ChartVersion string + + // AppVersion mirrors Chart.yaml's appVersion. Useful for + // detecting whether the customer is on a chart that supports + // the declarative ingestor flow at all. + AppVersion string + + // JobsManagerService is the in-cluster DNS name of the + // jobs-manager Service, e.g. + // "-jobs-manager..svc.cluster.local:8080". + // Used as the POST target for ingestion submissions. + JobsManagerService string + + // IngestorSAName is the name of the ServiceAccount the chart's + // hook pods run as. Today this is always the chart's default + // "ingestor". Customers who set `ingestionAuthz.serviceAccountName` + // to a non-default name in the parent client chart need to + // override via `tracebloc cluster info --ingestor-sa=` + // (and similarly for the future `dataset push` command). Reading + // the name from the cluster's ingestionAuthz ConfigMap so this + // flag becomes unnecessary is a v0.2 follow-up (see #7). + IngestorSAName string + + // IngestorImageDigest is the canonical digest the cluster's + // chart will spawn ingestor Jobs with. Comes from + // `INGESTOR_IMAGE_DIGEST` env on jobs-manager. Empty when the + // admin hasn't set images.ingestor.digest yet. + IngestorImageDigest string +} + +// DiscoverParentRelease finds the tracebloc parent client chart +// release in the given namespace by looking for a Deployment with +// the chart's hallmark labels. Returns a friendly error if none +// found, or if multiple candidates exist (so customers can pick). +// +// Why labels on a Deployment, not Helm's release secrets: +// +// - The release-secret path needs gzip + base64 + JSON parsing +// (helm v3's storage format), or pulls in helm.sh/helm/v3 as +// a dep (which transitively pulls in ~80MB of Go modules, +// including kube-runtime). +// - The Deployment is what the chart actually deploys; if it's +// not there, the chart isn't installed (or the install +// failed). Using its labels for discovery means "the +// discovered release is the running release" by construction. +// - The chart's labels are part of its public contract via +// `_helpers.tpl`; they're stable across minor versions. +// +// The chart's labels share `app.kubernetes.io/name=client` across +// EVERY resource it creates (mysql-client, jobs-manager, +// requests-proxy, etc.) — that's the helm convention where the +// chart's name is the label, not the component's. To distinguish +// jobs-manager from its siblings we filter by Deployment name +// suffix matching the chart's naming convention +// "-jobs-manager" (or plain "jobs-manager" for unprefixed +// installs). +func DiscoverParentRelease(ctx context.Context, cs kubernetes.Interface, namespace string) (*ParentRelease, error) { + deps, err := cs.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=client,app.kubernetes.io/managed-by=Helm", + }) + if err != nil { + return nil, fmt.Errorf("listing chart-managed deployments in namespace %s: %w", namespace, err) + } + + // Filter to just the jobs-manager Deployment(s). The chart pins + // the name to either "-jobs-manager" or "jobs-manager" + // depending on chart version + values; either way it ends in + // "-jobs-manager" or equals "jobs-manager". + var jmDeps []appsv1.Deployment + for _, d := range deps.Items { + if d.Name == "jobs-manager" || strings.HasSuffix(d.Name, "-jobs-manager") { + jmDeps = append(jmDeps, d) + } + } + + switch len(jmDeps) { + case 0: + return nil, fmt.Errorf( + "no tracebloc parent client release found in namespace %q "+ + "(no chart-managed Deployment named *-jobs-manager). "+ + "Install with `helm install tracebloc/client --namespace %s` first, "+ + "or pass --namespace to point at the namespace where it's running.", + namespace, namespace, + ) + case 1: + // happy path + default: + names := make([]string, 0, len(jmDeps)) + for _, d := range jmDeps { + names = append(names, d.Name) + } + return nil, fmt.Errorf( + "found %d tracebloc parent releases in namespace %q (%s); "+ + "this CLI doesn't yet support disambiguating between multiple. "+ + "Pass --namespace to target a namespace with exactly one release.", + len(jmDeps), namespace, strings.Join(names, ", "), + ) + } + + d := jmDeps[0] + release := &ParentRelease{ + ReleaseName: d.Labels["app.kubernetes.io/instance"], + ChartVersion: chartVersionFromLabel(d.Labels["helm.sh/chart"]), + AppVersion: d.Labels["app.kubernetes.io/version"], + } + + // The Service shares the Deployment's release labels by + // convention. Construct the FQDN — we don't need to actually + // hit the API to find it; the chart's helper templates pin the + // service name to "-jobs-manager" (or just + // "jobs-manager" for older chart versions where the release + // prefix wasn't included). + // + // We probe both names to be liberal: if the chart pinned just + // "jobs-manager", that wins; otherwise fall back to the + // release-prefixed form. Customers can always override via the + // ingestor subchart's `jobsManager.endpoint` value. + svc := pickJobsManagerService(ctx, cs, namespace, release.ReleaseName) + release.JobsManagerService = fmt.Sprintf("http://%s.%s.svc.cluster.local:8080", svc, namespace) + + // Read INGESTOR_IMAGE_DIGEST from jobs-manager's pod-spec env. + // The chart pipes images.ingestor.digest through to here. + // + // SA name is NOT discovered today — the chart doesn't surface + // `ingestionAuthz.serviceAccountName` through the jobs-manager + // env, and reading the ingestionAuthz ConfigMap to learn it is a + // v0.2 follow-up (see #7). We default to "ingestor" (the chart + // default); customers who renamed it pass --ingestor-sa from + // the CLI. Bugbot caught the earlier version that incorrectly + // claimed to read the SA name from env. + release.IngestorSAName = "ingestor" + if len(d.Spec.Template.Spec.Containers) > 0 { + for _, env := range d.Spec.Template.Spec.Containers[0].Env { + if env.Name == "INGESTOR_IMAGE_DIGEST" { + release.IngestorImageDigest = env.Value + } + } + } + + return release, nil +} + +// pickJobsManagerService probes for the chart's jobs-manager +// Service. The chart's helper templates have used both names over +// chart history: +// +// - "jobs-manager" (chart 1.3.x and earlier) +// - "-jobs-manager" (some prefixing variants) +// +// We try the unprefixed name first because that's the dominant +// shipped behavior; fall back if it doesn't exist. Probe failures +// (timeouts, RBAC denials) fall through to the prefixed name too — +// the customer gets a clear DNS error later if neither resolves, +// which is more actionable than "couldn't enumerate services." +func pickJobsManagerService(ctx context.Context, cs kubernetes.Interface, namespace, release string) string { + candidates := []string{ + "jobs-manager", + release + "-jobs-manager", + } + for _, name := range candidates { + _, err := cs.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return name + } + } + // Last-ditch: return the unprefixed candidate; the caller will + // hit a DNS error at POST time which surfaces with a clearer + // message than us guessing. + return "jobs-manager" +} + +// chartVersionFromLabel extracts "1.3.5" from helm.sh/chart="client-1.3.5". +// Labels in this format are standard Helm output; we strip the +// chart-name prefix to expose just the version. Returns the raw +// label if it doesn't match the expected pattern (defensive +// fallback for unusual chart-name formats). +func chartVersionFromLabel(label string) string { + // Expected: "-"; strip the "client-" prefix. + const prefix = "client-" + if strings.HasPrefix(label, prefix) { + return label[len(prefix):] + } + return label +} diff --git a/internal/cluster/discover_test.go b/internal/cluster/discover_test.go new file mode 100644 index 0000000..34f4433 --- /dev/null +++ b/internal/cluster/discover_test.go @@ -0,0 +1,222 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// jobsManagerDeployment builds the minimal Deployment the chart +// creates for jobs-manager — labels mirror the chart's +// _helpers.tpl output as of client 1.3.5. Tests use this to seed +// the fake clientset with realistic data; if the chart's label +// contract ever changes, these tests fail loudly and we update both +// here and the discovery logic in lockstep. +// +// Note: app.kubernetes.io/name is "client" (the chart name) on +// EVERY resource the chart creates, not "jobs-manager". This is +// the helm convention. The discovery logic distinguishes +// jobs-manager from its mysql / requests-proxy siblings by name +// suffix matching ("-jobs-manager") since the chart-level labels +// don't disambiguate. +func jobsManagerDeployment(release, namespace, chartLabel, appVersion, digest string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: release + "-jobs-manager", + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "client", + "app.kubernetes.io/instance": release, + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/version": appVersion, + "helm.sh/chart": chartLabel, + }, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "api", + Env: []corev1.EnvVar{ + {Name: "INGESTOR_IMAGE_DIGEST", Value: digest}, + }, + }}, + }, + }, + }, + } +} + +func jobsManagerService(name, namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +func TestDiscoverParentRelease_HappyPath(t *testing.T) { + const ns = "tracebloc" + cs := fake.NewClientset( + jobsManagerDeployment("tracebloc", ns, + "client-1.3.5", "1.3.5", + "sha256:463e236748708a5e3564569eec9173ea8cb3bcf515992d4939c5b610f3807a4a"), + jobsManagerService("jobs-manager", ns), + ) + + release, err := DiscoverParentRelease(context.Background(), cs, ns) + if err != nil { + t.Fatalf("DiscoverParentRelease: %v", err) + } + + want := ParentRelease{ + ReleaseName: "tracebloc", + ChartVersion: "1.3.5", + AppVersion: "1.3.5", + JobsManagerService: "http://jobs-manager." + ns + ".svc.cluster.local:8080", + IngestorSAName: "ingestor", + IngestorImageDigest: "sha256:463e236748708a5e3564569eec9173ea8cb3bcf515992d4939c5b610f3807a4a", + } + if *release != want { + t.Errorf("mismatch.\ngot: %+v\nwant: %+v", *release, want) + } +} + +func TestDiscoverParentRelease_NoReleaseFound(t *testing.T) { + cs := fake.NewClientset() // empty cluster + + _, err := DiscoverParentRelease(context.Background(), cs, "tracebloc") + if err == nil { + t.Fatal("expected error when no jobs-manager deployment exists") + } + // The error message has to be customer-actionable. Pin the + // key remediation phrase so a future refactor that loses it + // (or worse, replaces it with a stack trace) fails this test. + for _, want := range []string{"no tracebloc parent client release found", "helm install"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("expected error to mention %q, got: %s", want, err) + } + } +} + +func TestDiscoverParentRelease_MultipleReleases(t *testing.T) { + const ns = "tracebloc" + cs := fake.NewClientset( + jobsManagerDeployment("rel-a", ns, "client-1.3.5", "1.3.5", ""), + jobsManagerDeployment("rel-b", ns, "client-1.3.4", "1.3.4", ""), + ) + + _, err := DiscoverParentRelease(context.Background(), cs, ns) + if err == nil { + t.Fatal("expected error when multiple releases exist") + } + // Customer needs to see which releases the CLI found so they + // can pick (or clean up). + for _, want := range []string{"found 2", "rel-a", "rel-b"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("expected error to mention %q, got: %s", want, err) + } + } +} + +// Regression for the real-cluster discovery bug. Pre-fix, the +// selector was `app.kubernetes.io/name=jobs-manager` which never +// matches because the chart's convention is `name=` +// — so the real-cluster smoke test returned "no parent release +// found" even though one was clearly running. The fix is two-part: +// match on `name=client` (the chart name) AND filter the result +// set by Deployment-name suffix to pick out jobs-manager from its +// mysql/requests-proxy siblings, all of which share the +// chart-level labels. +func TestDiscoverParentRelease_FiltersOutSiblingDeployments(t *testing.T) { + const ns = "tracebloc" + // All three deployments share the chart-level labels — only + // the name suffix distinguishes them. + mysqlClient := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mysql-client", + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/name": "client", + "app.kubernetes.io/instance": "tracebloc", + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "client-1.3.5", + }, + }, + } + requestsProxy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tracebloc-requests-proxy", + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/name": "client", + "app.kubernetes.io/instance": "tracebloc", + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "client-1.3.5", + }, + }, + } + + cs := fake.NewClientset( + mysqlClient, + requestsProxy, + jobsManagerDeployment("tracebloc", ns, "client-1.3.5", "1.3.5", ""), + jobsManagerService("jobs-manager", ns), + ) + + release, err := DiscoverParentRelease(context.Background(), cs, ns) + if err != nil { + t.Fatalf("DiscoverParentRelease: %v", err) + } + // Should pick jobs-manager, NOT mysql-client or requests-proxy. + if release.ReleaseName != "tracebloc" { + t.Errorf("ReleaseName = %q, want %q (siblings should be filtered out by name suffix)", + release.ReleaseName, "tracebloc") + } +} + +func TestDiscoverParentRelease_FallsBackToReleasePrefixedService(t *testing.T) { + // Some older chart versions exposed the Service as + // "-jobs-manager" rather than the unprefixed + // "jobs-manager". The discover code probes both names and + // picks whichever resolves. This test seeds ONLY the + // release-prefixed Service and asserts the FQDN reflects it. + const ns = "tracebloc" + cs := fake.NewClientset( + jobsManagerDeployment("custom", ns, "client-1.3.5", "1.3.5", ""), + jobsManagerService("custom-jobs-manager", ns), + // NOTE: no "jobs-manager" service + ) + + release, err := DiscoverParentRelease(context.Background(), cs, ns) + if err != nil { + t.Fatalf("DiscoverParentRelease: %v", err) + } + wantFQDN := "http://custom-jobs-manager." + ns + ".svc.cluster.local:8080" + if release.JobsManagerService != wantFQDN { + t.Errorf("JobsManagerService = %q, want %q", release.JobsManagerService, wantFQDN) + } +} + +func TestChartVersionFromLabel(t *testing.T) { + cases := map[string]string{ + "client-1.3.5": "1.3.5", + "client-2.0.0-rc": "2.0.0-rc", + "client": "client", // unexpected shape — return as-is + "other-1.0.0": "other-1.0.0", // non-client chart — return as-is + "": "", + } + for in, want := range cases { + t.Run(in, func(t *testing.T) { + if got := chartVersionFromLabel(in); got != want { + t.Errorf("chartVersionFromLabel(%q) = %q, want %q", in, got, want) + } + }) + } +} diff --git a/internal/cluster/kubeconfig.go b/internal/cluster/kubeconfig.go new file mode 100644 index 0000000..2a94380 --- /dev/null +++ b/internal/cluster/kubeconfig.go @@ -0,0 +1,164 @@ +// Package cluster handles everything the CLI needs to talk to a +// Kubernetes cluster running the tracebloc parent client chart: +// +// - Locating the customer's kubeconfig + picking a context/namespace +// - Discovering the parent release (jobs-manager Deployment + its +// labels) +// - Minting an ingestor ServiceAccount token via TokenRequest +// +// Everything in this package is exercise-able without a real cluster +// thanks to client-go's fake clientset; see *_test.go. +package cluster + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// KubeconfigOptions captures every knob the customer can turn when +// telling the CLI which cluster + namespace to talk to. Defaults +// mirror kubectl: $KUBECONFIG > ~/.kube/config; current-context if +// --context unset; the context's default namespace if --namespace +// unset; "default" if even the context doesn't pin one. +// +// All fields are zero-value-safe — an empty Options{} produces the +// same behaviour as kubectl run with no flags. +type KubeconfigOptions struct { + // Path overrides $KUBECONFIG and ~/.kube/config when set. + // Honors path expansion (e.g. "~/.kube/config" → absolute). + Path string + + // Context picks a non-current-context from the loaded + // kubeconfig. Empty = use whatever current-context points at. + Context string + + // Namespace overrides the context's default. Empty = pull from + // the context's namespace, falling back to "default". + Namespace string +} + +// ResolvedConfig is what Load() returns: a usable rest.Config plus +// the source-of-truth metadata that the rest of the CLI needs (the +// namespace it resolved to, the context name it used). The +// metadata is separate from rest.Config because rest.Config doesn't +// carry the context name out of the box — it's discarded after the +// merge — and we want to show the customer "you're talking to +// context X" in diagnostic output. +type ResolvedConfig struct { + RestConfig *rest.Config + Context string + Namespace string + + // ServerURL is the cluster's API server URL (the value from + // the context's cluster.server field). Exposed for `tracebloc + // cluster info` to show "you're targeting " so the + // customer can sanity-check before doing anything destructive. + ServerURL string +} + +// Load resolves the kubeconfig + flags into a rest.Config ready for +// kubernetes.NewForConfig(). It does NOT make any network calls — +// pure parsing of local files + env. Network errors only happen +// when the caller subsequently uses the returned RestConfig. +func Load(opts KubeconfigOptions) (*ResolvedConfig, error) { + // NewDefaultClientConfigLoadingRules() wires up the full + // kubectl-style lookup chain: ExplicitPath > $KUBECONFIG + // (multi-file merging) > ~/.kube/config. Constructing + // ClientConfigLoadingRules{} directly with an empty + // ExplicitPath does NOT fall back to those defaults — it + // just refuses to load anything ("no configuration has been + // provided" was the symptom). Surfaced during the first + // real-cluster smoke test of `tracebloc cluster info`. + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + if explicit := expandPath(opts.Path); explicit != "" { + // Setting ExplicitPath bypasses the env-var merge + // (intentional — when a customer says "use THIS file", + // we don't mix it with their env-var-listed others). + loadingRules.ExplicitPath = explicit + } + + overrides := &clientcmd.ConfigOverrides{} + if opts.Context != "" { + overrides.CurrentContext = opts.Context + } + if opts.Namespace != "" { + // Setting Context.Namespace overrides the kubeconfig's + // per-context default. This matches `kubectl --namespace` + // semantics. + overrides.Context.Namespace = opts.Namespace + } + + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides) + + restConfig, err := loader.ClientConfig() + if err != nil { + return nil, fmt.Errorf("building rest config from kubeconfig: %w", err) + } + + // Pull the namespace the way kubectl does: explicit override + // wins; otherwise the context's namespace; otherwise "default". + // loader.Namespace() handles that fallback chain. + ns, _, err := loader.Namespace() + if err != nil { + return nil, fmt.Errorf("resolving namespace from kubeconfig: %w", err) + } + + // rest.Config doesn't carry the context name out of the box — + // fish it back out of the raw kubeconfig so diagnostics can + // show it. RawConfig() does a fresh read each call; cache if + // this ever becomes hot (it isn't). + rawCfg, err := loader.RawConfig() + if err != nil { + return nil, fmt.Errorf("reading raw kubeconfig: %w", err) + } + contextName := overrides.CurrentContext + if contextName == "" { + contextName = rawCfg.CurrentContext + } + + return &ResolvedConfig{ + RestConfig: restConfig, + Context: contextName, + Namespace: ns, + ServerURL: restConfig.Host, + }, nil +} + +// NewClientset is a tiny wrapper that constructs a kubernetes +// clientset from a ResolvedConfig. Exists so callers don't have to +// import k8s.io/client-go/kubernetes themselves for the common case +// of "I have a ResolvedConfig, I want a clientset." +func NewClientset(rc *ResolvedConfig) (kubernetes.Interface, error) { + cs, err := kubernetes.NewForConfig(rc.RestConfig) + if err != nil { + return nil, fmt.Errorf("constructing kubernetes clientset: %w", err) + } + return cs, nil +} + +// expandPath handles `~/.kube/config` → `/home/user/.kube/config` +// since clientcmd's ExplicitPath wants an absolute path. Empty +// strings pass through unchanged (they signal "use defaults" to +// clientcmd). +func expandPath(p string) string { + if p == "" { + return "" + } + if p[0] != '~' { + return p + } + home, err := os.UserHomeDir() + if err != nil { + // Best-effort: return the unexpanded path and let + // clientcmd's error surface mention it. Failing here would + // hide the more useful "tried to read ~/.kube/config and + // got X" error downstream. + return p + } + return filepath.Join(home, p[1:]) +} diff --git a/internal/cluster/kubeconfig_test.go b/internal/cluster/kubeconfig_test.go new file mode 100644 index 0000000..39709e2 --- /dev/null +++ b/internal/cluster/kubeconfig_test.go @@ -0,0 +1,84 @@ +package cluster + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExpandPath(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Skipf("os.UserHomeDir failed in this environment, skipping: %v", err) + } + + cases := []struct { + in string + wantFunc func(string) bool + }{ + { + in: "", + wantFunc: func(got string) bool { return got == "" }, + }, + { + in: "/absolute/path/to/kubeconfig", + wantFunc: func(got string) bool { return got == "/absolute/path/to/kubeconfig" }, + }, + { + in: "relative/path/kubeconfig", + wantFunc: func(got string) bool { return got == "relative/path/kubeconfig" }, + }, + { + in: "~/.kube/config", + wantFunc: func(got string) bool { + want := filepath.Join(home, ".kube/config") + return got == want + }, + }, + { + in: "~/some/file", + wantFunc: func(got string) bool { + want := filepath.Join(home, "some/file") + return got == want + }, + }, + } + + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + got := expandPath(c.in) + if !c.wantFunc(got) { + t.Errorf("expandPath(%q) = %q (failed predicate)", c.in, got) + } + }) + } +} + +// Load() involves real filesystem + clientcmd parsing; it's covered +// indirectly by the cluster-info integration path. A unit test +// would require writing a temp kubeconfig and feeding it back — +// useful but lower-priority than the discover/token tests since +// the clientcmd library is well-trodden. Pin the smallest contract: +// an empty Options{} doesn't panic. +func TestLoad_EmptyOptionsDoesNotPanic(t *testing.T) { + // May succeed or fail depending on whether the test runner has + // a kubeconfig available; either is fine. The point of this + // test is to ensure the function returns rather than crashes. + _, err := Load(KubeconfigOptions{}) + if err != nil && !looksLikeKubeconfigError(err.Error()) { + // Defense-in-depth: if Load() ever starts wrapping + // errors in a way that loses the "kubeconfig" context, + // flag it so the customer-facing diagnostic stays clear. + t.Logf("Load() failed with unfamiliar error: %v", err) + } +} + +func looksLikeKubeconfigError(s string) bool { + for _, marker := range []string{"kubeconfig", "config", "context", "namespace"} { + if strings.Contains(strings.ToLower(s), marker) { + return true + } + } + return false +} diff --git a/internal/cluster/token.go b/internal/cluster/token.go new file mode 100644 index 0000000..0c9a962 --- /dev/null +++ b/internal/cluster/token.go @@ -0,0 +1,214 @@ +package cluster + +import ( + "context" + "fmt" + + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// IngestorToken is what the customer (well, the CLI) authenticates +// to jobs-manager with. Bearer-token over HTTPS to +// /internal/submit-ingestion-run. The token is short-lived (10 +// minutes default) so it can be regenerated per `tracebloc dataset +// push` invocation without long-term-credential concerns. +type IngestorToken struct { + // Token is the raw bearer token string. Treat as sensitive; + // don't log it. Diagnostics print SHA256(Token)[:8] instead. + Token string + + // ExpirationSeconds matches the request — actual server-side + // expiration may be capped by cluster policy + // (--service-account-max-token-expiration on kube-apiserver). + // We don't try to parse the JWT to read its `exp` claim + // because the customer's diagnostic ("token expires in ~N + // min") is accurate enough using the requested value. + ExpirationSeconds int64 + + // Source records how the token was obtained — TokenRequest + // (the modern path) or a static secret (the fallback for + // clusters where the user can't call TokenRequest). Surfaced + // in `tracebloc cluster info` so the customer can see which + // path their RBAC allowed. + Source TokenSource +} + +// TokenSource enumerates the two ways the CLI can obtain an +// ingestor SA token. Adding a third (e.g. a customer-managed +// long-lived secret reference) is a v0.2 extension. +type TokenSource int + +const ( + // TokenSourceTokenRequest means we called the + // /serviceaccounts/{name}/token subresource. The modern, + // rotation-friendly path. Requires `create` permission on + // "serviceaccounts/token" for the user's kubeconfig. + TokenSourceTokenRequest TokenSource = iota + + // TokenSourceStaticSecret means we fell back to reading a + // pre-existing Secret of type + // kubernetes.io/service-account-token that references the + // ingestor SA. Older k8s (pre-1.24 default) auto-created + // these; on 1.24+ admins must create them by hand. Tokens + // from this source are long-lived (no expiration), which is + // less ideal but unblocks customers whose RBAC denies + // TokenRequest. + TokenSourceStaticSecret +) + +func (s TokenSource) String() string { + switch s { + case TokenSourceTokenRequest: + return "TokenRequest" + case TokenSourceStaticSecret: + return "static secret" + default: + return "unknown" + } +} + +// MintIngestorToken obtains a bearer token for the ingestor +// ServiceAccount in the given namespace. Tries TokenRequest first; +// if that's denied (RBAC) or unavailable (older API server) falls +// back to looking for a pre-existing service-account-token Secret. +// Returns an error with a clear remediation message if neither +// works. +// +// audiences may be nil/empty for the default API-server audience. +// Today jobs-manager's TokenReview-based validation accepts the +// default audience, so callers usually pass nil; future audiences +// (e.g. a forthcoming jobs-manager-specific audience) plug in here +// without API breakage. +func MintIngestorToken( + ctx context.Context, + cs kubernetes.Interface, + namespace, saName string, + expirationSeconds int64, + audiences []string, +) (*IngestorToken, error) { + // Try TokenRequest first. + req := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: &expirationSeconds, + Audiences: audiences, + }, + } + tr, err := cs.CoreV1().ServiceAccounts(namespace). + CreateToken(ctx, saName, req, metav1.CreateOptions{}) + + if err == nil { + return &IngestorToken{ + Token: tr.Status.Token, + ExpirationSeconds: expirationSeconds, + Source: TokenSourceTokenRequest, + }, nil + } + + // TokenRequest didn't work. Distinguish the cases we can fall + // back from (permission denied / not-found / unsupported) + // from cases we can't (e.g. context cancelled, network down). + if !isTokenRequestRecoverable(err) { + return nil, fmt.Errorf( + "minting token for ServiceAccount %s/%s via TokenRequest: %w", + namespace, saName, err, + ) + } + + // Fallback: look for a Secret of type + // kubernetes.io/service-account-token in the same namespace + // whose annotations point at our SA. + tok, fallbackErr := findStaticTokenSecret(ctx, cs, namespace, saName) + if fallbackErr != nil { + // Surface BOTH the original TokenRequest failure and the + // fallback failure — they give the customer different + // remediation options. + return nil, fmt.Errorf( + "no usable ingestor token. TokenRequest failed: %v. "+ + "Fallback to static secret also failed: %w. "+ + "Remediation: either grant your user the `create` verb on "+ + "`serviceaccounts/token` (RBAC), or have an admin create a "+ + "long-lived Secret of type kubernetes.io/service-account-token "+ + "that references the %s ServiceAccount in namespace %s.", + err, fallbackErr, saName, namespace, + ) + } + return tok, nil +} + +// isTokenRequestRecoverable reports whether a failed TokenRequest +// call is one we can fall back from (permission denied, API not +// available, SA missing) vs one we can't (context cancelled, +// network errors, etc.). +// +// Falling back on a network error would mask the network problem; +// the customer would see a less useful "static secret not found" +// instead of "couldn't reach the API server." +// +// Defers to k8s.io/apimachinery/pkg/api/errors for the classification +// rather than re-implementing the HTTP-status checks. The stdlib +// helpers key off Status.Reason (the typed enum) — that's the +// canonical interpretation, and our test file already constructs its +// fake errors via apierrors.NewForbidden, which sets both Code and +// Reason. Earlier versions of this file rolled their own helpers +// that read Status.Code numerically; Bugbot correctly flagged the +// divergence as a future-bug-waiting-to-happen for non-standard +// status errors. +func isTokenRequestRecoverable(err error) bool { + if err == nil { + return false + } + // Permission denied (StatusForbidden), SA missing + // (StatusNotFound), API not available on this cluster — typically + // pre-1.22 (StatusMethodNotAllowed/MethodNotSupported). + return apierrors.IsForbidden(err) || + apierrors.IsNotFound(err) || + apierrors.IsMethodNotSupported(err) +} + +// findStaticTokenSecret looks for a long-lived +// service-account-token Secret bound to the given SA. The legacy +// pre-k8s-1.24 way of authenticating an SA: kubectl auto-created +// these on SA creation. On modern clusters admins create them by +// hand when needed. +func findStaticTokenSecret(ctx context.Context, cs kubernetes.Interface, namespace, saName string) (*IngestorToken, error) { + secrets, err := cs.CoreV1().Secrets(namespace).List(ctx, metav1.ListOptions{ + FieldSelector: "type=kubernetes.io/service-account-token", + }) + if err != nil { + // If the user can't even list Secrets, surface that + // directly — they may be missing more RBAC than just + // TokenRequest, and "static secret fallback failed" is + // less informative than the underlying error. + return nil, fmt.Errorf("listing service-account-token secrets in %s: %w", namespace, err) + } + + for _, s := range secrets.Items { + // The Secret is bound to an SA via the + // `kubernetes.io/service-account.name` annotation. (The + // `.uid` annotation pins it to that exact SA instance — + // we don't check that because we want any historically + // created token-secret for the named SA to work.) + if s.Annotations[corev1.ServiceAccountNameKey] != saName { + continue + } + token, ok := s.Data["token"] + if !ok || len(token) == 0 { + continue // empty token field — skip + } + return &IngestorToken{ + Token: string(token), + ExpirationSeconds: 0, // static secrets don't expire client-side + Source: TokenSourceStaticSecret, + }, nil + } + + return nil, fmt.Errorf( + "no Secret of type kubernetes.io/service-account-token bound to "+ + "ServiceAccount %s found in namespace %s", + saName, namespace, + ) +} diff --git a/internal/cluster/token_test.go b/internal/cluster/token_test.go new file mode 100644 index 0000000..64e1c60 --- /dev/null +++ b/internal/cluster/token_test.go @@ -0,0 +1,186 @@ +package cluster + +import ( + "context" + "errors" + "strings" + "testing" + + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func TestMintIngestorToken_TokenRequest_HappyPath(t *testing.T) { + const ns = "tracebloc" + cs := fake.NewClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "ingestor", Namespace: ns}, + }) + + // The fake clientset doesn't implement the TokenRequest + // subresource by default — we have to inject a reactor. This + // mirrors what the real API server does: returns a stamped + // Status.Token. + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + ca, ok := action.(k8stesting.CreateAction) + if !ok { + return false, nil, nil + } + if ca.GetSubresource() != "token" { + return false, nil, nil + } + tr := ca.GetObject().(*authenticationv1.TokenRequest) + tr.Status.Token = "fake-token-deadbeef" + return true, tr, nil + }) + + tok, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err != nil { + t.Fatalf("MintIngestorToken: %v", err) + } + if tok.Token != "fake-token-deadbeef" { + t.Errorf("Token = %q, want %q", tok.Token, "fake-token-deadbeef") + } + if tok.Source != TokenSourceTokenRequest { + t.Errorf("Source = %v, want TokenSourceTokenRequest", tok.Source) + } + if tok.ExpirationSeconds != 600 { + t.Errorf("ExpirationSeconds = %d, want 600", tok.ExpirationSeconds) + } +} + +func TestMintIngestorToken_FallsBackToStaticSecret(t *testing.T) { + const ns = "tracebloc" + + // Pre-seed a service-account-token Secret bound to the + // ingestor SA. This is what older clusters auto-created on SA + // creation; modern clusters require admins to create it by + // hand. + staticSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingestor-token-abc123", + Namespace: ns, + Annotations: map[string]string{ + corev1.ServiceAccountNameKey: "ingestor", + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{ + "token": []byte("static-token-cafebabe"), + }, + } + cs := fake.NewClientset( + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "ingestor", Namespace: ns}, + }, + staticSecret, + ) + + // Make TokenRequest fail with a recoverable error + // (Forbidden = the user lacks RBAC, which is the dominant + // reason customers fall through to the static-secret path). + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + if ca, ok := action.(k8stesting.CreateAction); ok && ca.GetSubresource() == "token" { + return true, nil, apierrors.NewForbidden( + corev1.Resource("serviceaccounts/token"), + "ingestor", + errors.New("user cannot create serviceaccounts/token"), + ) + } + return false, nil, nil + }) + + tok, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err != nil { + t.Fatalf("MintIngestorToken: %v", err) + } + if tok.Token != "static-token-cafebabe" { + t.Errorf("Token = %q, want fallback static-token-cafebabe", tok.Token) + } + if tok.Source != TokenSourceStaticSecret { + t.Errorf("Source = %v, want TokenSourceStaticSecret", tok.Source) + } + if tok.ExpirationSeconds != 0 { + t.Errorf("ExpirationSeconds = %d, want 0 (static secrets don't expire client-side)", tok.ExpirationSeconds) + } +} + +func TestMintIngestorToken_NoFallbackOnUnrecoverableError(t *testing.T) { + // Network errors (or any non-403/404/405 error) shouldn't + // trigger the static-secret fallback — they'd mask the real + // underlying problem. Pin that behavior. + const ns = "tracebloc" + cs := fake.NewClientset() + + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + if ca, ok := action.(k8stesting.CreateAction); ok && ca.GetSubresource() == "token" { + return true, nil, errors.New("simulated network failure: i/o timeout") + } + return false, nil, nil + }) + + _, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err == nil { + t.Fatal("expected non-recoverable error to propagate, got nil") + } + if !strings.Contains(err.Error(), "i/o timeout") { + t.Errorf("expected the network-error message to surface, got: %v", err) + } +} + +func TestMintIngestorToken_BothPathsFailWithRemediation(t *testing.T) { + // Worst case: TokenRequest forbidden AND no static secret. The + // error must surface both failures plus an actionable + // remediation hint. + const ns = "tracebloc" + cs := fake.NewClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "ingestor", Namespace: ns}, + }) + + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + if ca, ok := action.(k8stesting.CreateAction); ok && ca.GetSubresource() == "token" { + return true, nil, apierrors.NewForbidden( + corev1.Resource("serviceaccounts/token"), + "ingestor", + errors.New("forbidden"), + ) + } + return false, nil, nil + }) + + _, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err == nil { + t.Fatal("expected error when both TokenRequest + static fallback fail") + } + for _, want := range []string{ + "TokenRequest failed", + "static secret also failed", + "Remediation", + "create", // remediation mentions granting the verb + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("expected combined error to mention %q, got: %s", want, err) + } + } +} + +func TestTokenSource_String(t *testing.T) { + cases := map[TokenSource]string{ + TokenSourceTokenRequest: "TokenRequest", + TokenSourceStaticSecret: "static secret", + TokenSource(99): "unknown", + } + for s, want := range cases { + if got := s.String(); got != want { + t.Errorf("TokenSource(%d).String() = %q, want %q", s, got, want) + } + } +} From d2a7906de59861992981b83d7e26aa1b60fc27ba Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Fri, 22 May 2026 11:03:38 +0500 Subject: [PATCH 03/17] feat(dataset): pre-flight for `dataset push` (PR-a of #151) (#8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dataset): pre-flight for `dataset push` (PR-a of #151) Phase 3 (tracebloc/client#151) lands in two PRs to keep diffs reviewable. PR-a (this one) is the no-op-safe path: synthesize the ingest spec from flags, validate against the embedded ingest.v1 schema, walk the local directory, discover the cluster's parent release + shared PVC. Print a summary, then either stop (--dry-run) or fail cleanly with "wait for PR-b" so a customer who pulled the PR-a binary doesn't see "0 files transferred" confusion. PR-b adds the novel-engineering piece: ephemeral stage Pod (alpine 3.20 pinned by digest), client-go remotecommand SPDY executor + tar stream, schollz/progressbar/v3, SIGINT-safe cleanup. It plugs into the "TODO: PR-b" branch at the end of runDatasetPush() without touching anything upstream. Surface area added: internal/push/ (new package) spec.go SpecArgs -> ingest.v1.json-conforming map. Path is leave-validation-to-schema; the schema is the single source of truth. walk.go Discover() walks /labels.csv + /images/, enforces v0.1 size caps (1 GiB total, 500 MiB per file), accepts {.jpg,.jpeg,.png,.webp} case-insensitively. internal/cluster/pvc.go (new file) DiscoverSharedPVC reads the chart's `client-pvc` (hardcoded by _helpers.tpl, not parameterized) in the namespace, verifies it is Bound, returns access modes. IsReadWriteMany() drives the stage-Pod scheduling warning in PR-b. internal/cli/dataset.go (new file) `tracebloc dataset push ` command. Order: 1. flag->spec synthesize + schema-validate (stderr diagnostic, exit 2 — same wording style as ingest validate) 2. walk local layout + size caps (exit 3) 3. load kubeconfig + clientset (exit 3) 4. discover parent release (exit 4) 5. discover shared PVC (exit 4) 6. print pre-flight summary 7. --dry-run stop, or exit 6 "wait for PR-b" internal/cli/root.go (1-line wire-up) Register the new command. Why image_classification only: the epic (tracebloc/client#147) v0.1 non-goals defer other categories to v0.2 as one-PR additions. The --category flag does NOT pre-validate so the schema's enum check produces the canonical error (rather than CLI-side drift). Why exit code 6 for "not implemented yet": distinct from the existing schema (2), local (3), discovery (4), and Phase-2's token-mint (5) codes. Tests assert the exact code so the contract sticks across PR-b's refactor. Tests: spec_test.go Build() ↔ embedded schema contract (the core "this must produce something the validator accepts" pin) walk_test.go Layout + size caps, all happy + sad paths (case-insensitive ext, skip .DS_Store, missing files, byte-sum sanity) pvc_test.go Fake-clientset coverage: happy path, not-found diagnostic, Pending-phase diagnostic, mixed-mode IsReadWriteMany dataset_test.go CLI integration: schema-fail exit 2, walk-fail exit 3, kubeconfig-fail exit 3, cobra args check Locally: vet, test -race -cover, gofmt -s, errcheck — all green. Coverage: push 81.0%, cluster 83.2%, schema 80.7%, cli 52.0%. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(dataset): drop duplicate pluralS, reuse existing plural helper Cursor Bugbot (Low severity) on PR #8: pluralS() in dataset.go was an exact duplicate of plural() already defined in ingest.go — same cli package, identical body. Removed pluralS, the schema-error diagnostic now calls plural() directly. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): reject path-traversal table names (Bugbot, Medium) Cursor Bugbot on PR #8 (commit 4240097), Medium severity: the `table` value flows unsanitized into the /data/shared// PVC path. The embedded ingest.v1 schema only enforces minLength:1 on `table` — no pattern — so --table=../../etc would resolve (via path-cleaning) to /etc, and PR-b's stage Pod would write outside the intended subtree, potentially clobbering another table's staged data. Fix: - push.ValidateTableName(table): rejects anything not matching ^[A-Za-z0-9_]+$ — the intersection of "valid unquoted MySQL identifier" and "safe single path segment". Called as step 1 of runDatasetPush, before SpecArgs.Build(). - StagedPrefix hardened: drops path.Join (whose ".."-cleaning is the silent footgun) for plain concatenation, and panics if handed an unsafe name. Callers MUST ValidateTableName first; the panic is a defense-in-depth backstop so a PR-b call site that forgets validation fails loudly in tests instead of silently escaping the prefix. - Build() doc gains the precondition note. The upstream half — adding a `pattern` constraint to the schema's `table` field so the helm flow + jobs-manager get the same guard — is filed as tracebloc/data-ingestors#116. Once that lands and the CLI re-syncs the schema, ValidateTableName collapses to a thin wrapper. Tests: - TestValidateTableName_Accepts / _RejectsUnsafe: the security regression pin (../../etc, ../foo, slashes, dots, etc.) - TestStagedPrefix_PanicsOnUnsafeName: the backstop - TestDatasetPush_TraversalTableName_ExitsTwo: CLI-layer, --table=../../etc → exit 2 before any cluster work Locally: vet, test -race -cover, gofmt -s, errcheck — all green. Coverage: push 83.3%, cluster 83.2%, schema 80.7%, cli 52.2%. Co-Authored-By: Claude Opus 4.7 (1M context) * refactor(push): export HumanBytes, drop the copy in dataset.go Cursor Bugbot on PR #8 (commit 837157f), Low severity: humanBytesForSummary in dataset.go was a line-for-line copy of humanBytes in walk.go — both new in this PR. A future tweak (TiB support, precision change) would likely patch one and not the other, drifting the size shown in an over-cap error from the size shown in the dry-run summary. Fix: export push.HumanBytes as the single implementation; delete the dataset.go copy. Both the over-cap diagnostics and the pre-flight summary now format through the same function. Locally: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): reject a directory named labels.csv in Discover Cursor Bugbot on PR #8 (commit 9915866), Low severity: Discover checked imagesStat.IsDir() for the images/ entry but never the reverse for labels.csv. A directory literally named "labels.csv" passes os.Stat, so the pre-flight would accept it and PR-b's tar stream would later fail confusingly trying to read a directory as a CSV. Added the symmetric labelsStat.IsDir() guard with a clear error. TestDiscover_LabelsCSVIsDirectory pins it. Locally: vet, test -race -cover, gofmt -s, errcheck — all green. Coverage: push 83.8%. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- internal/cli/dataset.go | 341 +++++++++++++++++++++++++++++++++++ internal/cli/dataset_test.go | 249 +++++++++++++++++++++++++ internal/cli/root.go | 1 + internal/cluster/pvc.go | 146 +++++++++++++++ internal/cluster/pvc_test.go | 117 ++++++++++++ internal/push/spec.go | 186 +++++++++++++++++++ internal/push/spec_test.go | 173 ++++++++++++++++++ internal/push/walk.go | 244 +++++++++++++++++++++++++ internal/push/walk_test.go | 247 +++++++++++++++++++++++++ 9 files changed, 1704 insertions(+) create mode 100644 internal/cli/dataset.go create mode 100644 internal/cli/dataset_test.go create mode 100644 internal/cluster/pvc.go create mode 100644 internal/cluster/pvc_test.go create mode 100644 internal/push/spec.go create mode 100644 internal/push/spec_test.go create mode 100644 internal/push/walk.go create mode 100644 internal/push/walk_test.go diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go new file mode 100644 index 0000000..d740e0f --- /dev/null +++ b/internal/cli/dataset.go @@ -0,0 +1,341 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/push" + "github.com/tracebloc/cli/internal/schema" +) + +// newDatasetCmd wires the `tracebloc dataset` subtree. The dominant +// verb is `push`, introduced in Phase 3 (tracebloc/client#151) and +// landing across two PRs: PR-a (this one) implements the +// no-op-safe pre-flight; PR-b adds the actual file streaming via +// an ephemeral Pod + tar-over-exec. Future verbs (`dataset list`, +// `dataset rm`) hang off this parent in v0.2. +func newDatasetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "dataset", + Short: "Manage datasets in the parent client release", + Long: `Commands for staging and managing datasets on the cluster's +shared PVC. + +Today: ` + "`dataset push`" + ` stages a local directory and (in PR-b) +submits an ingestion run. ` + "`tracebloc cluster info`" + ` is the +pre-flight you'd typically run before the first push.`, + } + cmd.AddCommand(newDatasetPushCmd()) + return cmd +} + +// newDatasetPushCmd implements `tracebloc dataset push `. +// +// PR-a scope (what this implements today): +// +// - Synthesize the ingest spec from flags (`internal/push.SpecArgs.Build`) +// - Validate it against the embedded ingest.v1 schema +// - Walk the local directory + enforce v0.1 size caps +// - Discover cluster, parent release, and shared PVC +// - Print a single-screen "ready to stage" summary +// - --dry-run stops here (and so does this PR — actual staging +// errors out with "coming in PR-b" until #151 PR-b merges) +// +// PR-b will add the ephemeral stage Pod, tar-over-exec stream, +// progress bar, and SIGINT-safe cleanup — slotting into the +// "TODO: PR-b" branch below without touching anything above it. +func newDatasetPushCmd() *cobra.Command { + var ( + // Kubeconfig flags — same conventions as `cluster info`. + // Promoting these to persistent on the root is a v0.2 + // follow-up (tracebloc/cli#3); for now they live on each + // command that needs them. + kubeconfigPath string + contextOverride string + nsOverride string + + // Ingest-spec flags. The set is intentionally + // image_classification-only for v0.1 per epic #147 + // non-goals; other categories are one-PR additions in v0.2. + table string + category string + intent string + labelColumn string + + // Operations flags. + dryRun bool + + // Ingestor SA name override (only matters once PR-b mints + // a token to talk to the future stage-pod-creation hook). + // Plumbed today so PR-b doesn't have to touch flag wiring. + ingestorSAName string + ) + + cmd := &cobra.Command{ + Use: "push ", + Short: "Stage a local dataset to the cluster's shared PVC", + Long: `Stages a local image_classification dataset to the parent client +release's shared PVC, then (in PR-b) submits an ingestion run. + +Expected local layout: + + / + labels.csv (required) + images/ (required) + 001.jpg + 002.jpg + ... + +Accepted image extensions: .jpg, .jpeg, .png, .webp (case-insensitive). + +v0.1 caps the dataset at 1 GiB total + 500 MiB per file. Larger +datasets need the v0.2 cloud-source story (S3/GCS/HTTPS sources) — +see tracebloc/client#147 non-goals. + +Exit codes: + 0 staged + (in PR-b) submitted successfully + 2 schema validation failed (synthesized spec rejected) + 3 local-layout or kubeconfig error + 4 cluster reachable but parent release missing + 6 pre-flight succeeded but the actual stage step isn't + implemented yet (PR-b for #151 will deliver it)`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runDatasetPush(cmd.Context(), cmd.OutOrStdout(), cmd.ErrOrStderr(), + runDatasetPushArgs{ + LocalPath: args[0], + Kubeconfig: kubeconfigPath, + Context: contextOverride, + Namespace: nsOverride, + Spec: push.SpecArgs{Table: table, Category: category, Intent: intent, LabelColumn: labelColumn}, + DryRun: dryRun, + IngestorSAName: ingestorSAName, + }) + }, + } + + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", "", + "path to the kubeconfig file (default: $KUBECONFIG, then ~/.kube/config)") + cmd.Flags().StringVar(&contextOverride, "context", "", + "name of the kubeconfig context to use (default: kubeconfig's current-context)") + cmd.Flags().StringVarP(&nsOverride, "namespace", "n", "", + "namespace where the parent tracebloc/client release is installed") + + // Required spec flags. We DON'T mark them required-at-cobra-level + // because cobra's "required flag" error message is terse and + // pre-empts our richer schema-driven diagnostic. Instead, the + // schema validator catches missing/empty values with the canonical + // JSON-pointer-anchored error. + cmd.Flags().StringVar(&table, "table", "", + "destination table name (MySQL identifier; matches /data/shared/
/ on the PVC)") + cmd.Flags().StringVar(&category, "category", "image_classification", + "task category (v0.1 only supports image_classification; see tracebloc/client#147 non-goals)") + cmd.Flags().StringVar(&intent, "intent", "", + "intent: train|test") + cmd.Flags().StringVar(&labelColumn, "label-column", "", + "column name in labels.csv that holds the label") + + cmd.Flags().BoolVar(&dryRun, "dry-run", false, + "validate + discover + walk, but don't create any cluster resources") + cmd.Flags().StringVar(&ingestorSAName, "ingestor-sa", "", + "override the ingestor ServiceAccount name (default: \"ingestor\"); "+ + "set this if you customized ingestionAuthz.serviceAccountName in the parent client chart") + + return cmd +} + +// runDatasetPushArgs collects every parameter runDatasetPush needs, +// so the body stays testable without going through cobra. The cobra +// RunE wrapper above is the ONLY caller in production; tests +// construct one of these directly. +type runDatasetPushArgs struct { + LocalPath string + Kubeconfig string + Context string + Namespace string + Spec push.SpecArgs + DryRun bool + IngestorSAName string +} + +// runDatasetPush is the PR-a slim implementation. It performs every +// pre-flight check and prints a summary; the actual file staging +// is gated behind a clear "not yet implemented" error so PR-a +// merging doesn't silently advertise a feature it can't deliver. +// +// Step order is "fail fast, fail local" — every step that doesn't +// need the cluster runs before any that does, so a customer with +// a bad label-column or oversized dataset gets the diagnostic in +// milliseconds without a kubeconfig round-trip. +func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPushArgs) error { + // 1. Validate the table name BEFORE anything else. It's both + // the MySQL identifier and the /data/shared/
/ PVC + // subdirectory — an unsanitized traversal name (../../etc) + // would escape that subtree once PR-b's stage Pod writes to + // it. The embedded schema only checks minLength on `table`, + // so this CLI-side guard is the real fix. SpecArgs.Build() + // below calls StagedPrefix, which panics on an unsafe name — + // so this check MUST come first. + if err := push.ValidateTableName(a.Spec.Table); err != nil { + return &exitError{code: 2, err: err} + } + + // 2. Synthesize the spec from flags + validate against schema. + // Catches "bad category", "missing intent" etc. BEFORE we + // touch the filesystem or the cluster. The error formatter + // is the same one ingest validate uses, so a customer who + // YAML'd manually first sees identical wording. + spec := a.Spec.Build() + specBytes, err := yaml.Marshal(spec) + if err != nil { + return &exitError{code: 3, err: fmt.Errorf("marshaling synthesized spec: %w", err)} + } + v, err := schema.NewV1Validator() + if err != nil { + return &exitError{code: 3, err: fmt.Errorf("loading embedded schema: %w", err)} + } + _, errs, parseErr := v.ValidateYAML(specBytes) + if parseErr != nil { + // "Parse" failing on a spec we marshaled ourselves is a + // programming error, not a customer error — surface it + // with the bytes so we can diagnose. Exit 3 (the + // "internal" bucket) matches the marshal-failure branch + // above. + return &exitError{code: 3, err: fmt.Errorf("internal: re-parsing synthesized spec: %w\n%s", parseErr, specBytes)} + } + if len(errs) > 0 { + // Use the SAME formatter `ingest validate` uses, so the + // experience is identical whether the customer authored + // YAML by hand or via flags. Diagnostics go to stderr + // (matching ingest validate) so a downstream pipe of + // stdout (e.g. piping the summary to jq once that's a + // JSON output mode) isn't polluted by error text. Exit 2 + // is reserved for schema violations across the CLI. + _, _ = fmt.Fprintf(errOut, "synthesized spec failed schema validation (%d issue%s):\n", + len(errs), plural(len(errs))) + _, _ = fmt.Fprintln(errOut, schema.FormatErrors(errs)) + return &exitError{code: 2, err: errors.New("synthesized spec failed schema validation; check the flag values above")} + } + + // 3. Walk the local directory. Enforces layout + size caps; + // customer sees a clear pointer to expected layout if they + // pass the wrong directory. + layout, err := push.Discover(a.LocalPath) + if err != nil { + return &exitError{code: 3, err: err} + } + + // 4. Cluster discovery — same kubeconfig path as `cluster info`. + // Errors mirror that command's exit-code contract (3 for + // kubeconfig, 4 for missing release) so behaviour is + // consistent across pre-flight commands. + resolved, err := cluster.Load(cluster.KubeconfigOptions{ + Path: a.Kubeconfig, + Context: a.Context, + Namespace: a.Namespace, + }) + if err != nil { + return &exitError{code: 3, err: fmt.Errorf("loading kubeconfig: %w", err)} + } + cs, err := cluster.NewClientset(resolved) + if err != nil { + return &exitError{code: 3, err: err} + } + release, err := cluster.DiscoverParentRelease(ctx, cs, resolved.Namespace) + if err != nil { + return &exitError{code: 4, err: err} + } + if a.IngestorSAName != "" { + release.IngestorSAName = a.IngestorSAName + } + + // 5. PVC discovery. New in this PR — confirms the chart's + // shared-data PVC is Bound before we waste time provisioning + // a Pod that can't mount it. + pvc, err := cluster.DiscoverSharedPVC(ctx, cs, resolved.Namespace) + if err != nil { + return &exitError{code: 4, err: err} + } + + // 6. Print the pre-flight summary. The output is the same in + // dry-run and (eventually) live mode — only the "what + // happens next" line differs. Customers iterating on a + // bad layout see this every attempt, so it's worth keeping + // skimmable: one fact per line, aligned by column. + printPushPreflight(out, layout, release, pvc, spec, a.DryRun) + + // 7. Dry-run stop. Acknowledged success. + if a.DryRun { + _, _ = fmt.Fprintln(out, "Dry-run complete — no cluster resources were created.") + return nil + } + + // 8. The actual staging branch lands in PR-b. Failing here + // rather than silently returning success means a customer + // who pulled PR-a's binary and ran without --dry-run gets + // a clear "wait for PR-b" signal instead of "0 files + // transferred" confusion in Phase 4. + return &exitError{code: 6, err: errors.New( + "pre-flight succeeded but the actual file staging step isn't " + + "implemented yet — wait for tracebloc/client#151 PR-b. " + + "Re-run with --dry-run to validate without this error.")} +} + +// printPushPreflight is the customer-facing summary. Mirrors +// `cluster info`'s layout for consistency: section header, +// indented key:value rows. Kept here (not on the layout/release/pvc +// types) because the formatting is policy and lives with the CLI, +// not the data. +func printPushPreflight( + out io.Writer, + layout *push.LocalLayout, + release *cluster.ParentRelease, + pvc *cluster.SharedPVC, + spec map[string]any, + dryRun bool, +) { + // Explicit-discard the writer errors throughout — same rationale + // as cli/cluster.go and cli/ingest.go: a pipe-write failure + // shouldn't convert success into failure. The exit code is + // the contract. + _, _ = fmt.Fprintf(out, "Local dataset:\n") + _, _ = fmt.Fprintf(out, " root: %s\n", layout.Root) + _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) + _, _ = fmt.Fprintf(out, " images: %d files\n", len(layout.Images)) + _, _ = fmt.Fprintf(out, " total size: %s\n", push.HumanBytes(layout.TotalBytes)) + _, _ = fmt.Fprintln(out) + + _, _ = fmt.Fprintf(out, "Target cluster:\n") + _, _ = fmt.Fprintf(out, " release: %s (chart %s)\n", release.ReleaseName, release.ChartVersion) + _, _ = fmt.Fprintf(out, " jobs-manager: %s\n", release.JobsManagerService) + _, _ = fmt.Fprintf(out, " shared PVC: %s (%s)\n", pvc.ClaimName, pvc.Phase) + if !pvc.IsReadWriteMany() { + // Warn but don't block — RWO clusters still work, the + // scheduler will co-locate the stage Pod with the existing + // mounter. Phase 3 PR-b will surface the same warning at + // pod-create time too. + _, _ = fmt.Fprintf(out, " access: %v (warn: not ReadWriteMany — stage Pod will co-locate)\n", pvc.AccessModes) + } + _, _ = fmt.Fprintln(out) + + _, _ = fmt.Fprintf(out, "Synthesized ingest spec:\n") + _, _ = fmt.Fprintf(out, " table: %s\n", spec["table"]) + _, _ = fmt.Fprintf(out, " category: %s\n", spec["category"]) + _, _ = fmt.Fprintf(out, " intent: %s\n", spec["intent"]) + _, _ = fmt.Fprintf(out, " label column: %s\n", spec["label"]) + _, _ = fmt.Fprintf(out, " destination: %s\n", push.StagedPrefix(spec["table"].(string))) + _, _ = fmt.Fprintln(out) + + if !dryRun { + _, _ = fmt.Fprintf(out, "Next: stage %d files (%s) → %s (coming in PR-b for #151)\n", + 1+len(layout.Images), push.HumanBytes(layout.TotalBytes), + push.StagedPrefix(spec["table"].(string))) + _, _ = fmt.Fprintln(out) + } +} diff --git a/internal/cli/dataset_test.go b/internal/cli/dataset_test.go new file mode 100644 index 0000000..5a4ef69 --- /dev/null +++ b/internal/cli/dataset_test.go @@ -0,0 +1,249 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// imgcLayout drops a minimum-viable image_classification directory +// under t.TempDir() and returns its path. Mirrors push.imgcDir +// (tests can't import test helpers across packages, so we duplicate +// the few lines). +func imgcLayout(t *testing.T) string { + t.Helper() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "001.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image: %v", err) + } + return root +} + +// execDatasetPush drives the full cobra dispatch for the push +// command and returns the exit code + captured stdout/stderr. +// Mirrors the execIngestValidate helper from ingest_test.go — same +// rationale about not sharing *cobra.Command across cases (cobra +// holds flag state on the command tree). +// +// kubeconfigPath is required because every push invocation tries +// kubeconfig load before any cluster work; tests that want to +// stop EARLIER (at schema validation or layout walk) still need a +// kubeconfig path that resolves predictably. We feed in a path +// that's guaranteed to fail os.Stat so the kubeconfig branch +// errors out consistently when reached — and tests assert on the +// EARLIER stage's exit code, which fires before kubeconfig. +func execDatasetPush(t *testing.T, args []string) (exitCode int, stdout, stderr string) { + t.Helper() + root := NewRootCmd(BuildInfo{Version: "test"}) + var so, se bytes.Buffer + root.SetOut(&so) + root.SetErr(&se) + + // Always inject a guaranteed-bad kubeconfig path so tests that + // "fall through" the local pre-checks into kubeconfig load + // get a deterministic exit 3 (not a flaky "depends on whether + // you have a real kubeconfig" outcome). + cmdArgs := append([]string{"dataset", "push", + "--kubeconfig=/tmp/tracebloc-cli-test-nonexistent-" + t.Name()}, + args...) + root.SetArgs(cmdArgs) + + err := root.Execute() + return ExitCodeFromError(err), so.String(), se.String() +} + +// TestDatasetPush_BadCategory_ExitsTwo: the synthesized spec gets +// fed through the embedded schema before any cluster work; an +// invalid category surfaces with exit 2 (the CLI-wide "schema +// violation" code). +func TestDatasetPush_BadCategory_ExitsTwo(t *testing.T) { + root := imgcLayout(t) + code, _, stderr := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=definitely-not-a-category", + "--intent=train", + "--label-column=label", + }) + if code != 2 { + t.Fatalf("expected exit 2 for schema failure, got %d", code) + } + // The same FormatErrors output ingest validate uses, routed + // to stderr so downstream pipes of stdout aren't polluted. + // Checking for "category" + "image_classification" surfacing + // means we're getting the JSON-pointer-anchored diagnostic + + // the enum-list expansion, not a generic message. + // Check three things: (1) the JSON-pointer-style "category" + // anchor surfaces, (2) the enum-list expansion happened (so + // "image_classification" appears as a valid option), (3) our + // "synthesized spec" framing is on the right output stream. + // The wording "synthesized spec" is intentionally distinct + // from ingest validate's "schema validation failed" — it tells + // the customer the CLI synthesized the YAML; they didn't + // author it. + for _, want := range []string{"category", "image_classification", "synthesized spec failed schema validation"} { + if !strings.Contains(stderr, want) { + t.Errorf("expected stderr to mention %q, got:\n%s", want, stderr) + } + } +} + +// TestDatasetPush_TraversalTableName_ExitsTwo is the security +// regression pin at the CLI layer. --table=../../etc must be +// rejected with exit 2 BEFORE any spec synthesis or cluster work — +// the table name flows into the /data/shared/
/ PVC path, +// and a traversal value would let PR-b's stage Pod escape that +// subtree. Bugbot flagged this on PR #8 commit 4240097. +func TestDatasetPush_TraversalTableName_ExitsTwo(t *testing.T) { + root := imgcLayout(t) + for _, bad := range []string{"../../etc", "../foo", "foo/bar"} { + t.Run(bad, func(t *testing.T) { + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=" + bad, + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 2 { + t.Fatalf("expected exit 2 for traversal table name %q, got %d", bad, code) + } + }) + } +} + +// TestDatasetPush_MissingIntent_ExitsTwo: pins the "intent is +// required" diagnostic path — different schema violation but the +// same exit-code class. +func TestDatasetPush_MissingIntent_ExitsTwo(t *testing.T) { + root := imgcLayout(t) + code, _, stderr := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=image_classification", + // intent omitted + "--label-column=label", + }) + if code != 2 { + t.Fatalf("expected exit 2 for missing intent, got %d", code) + } + if !strings.Contains(stderr, "intent") { + t.Errorf("expected stderr to mention 'intent', got:\n%s", stderr) + } +} + +// TestDatasetPush_NonexistentLocalPath_ExitsThree: the layout walk +// runs AFTER schema validation, so an invalid local path with +// otherwise-valid flags surfaces at the walk stage with exit 3 +// (the "local input or kubeconfig" code). +// +// We only assert on the exit code, not the error text — cobra's +// SilenceErrors=true means root.Execute() doesn't surface the +// returned error to stderr (main.go does that). Mirrors +// ingest_test.go's TestIngestValidate_UnreadableFileExitsThree +// pattern; the error-content surface is exercised at the package +// level (internal/push.Discover's own tests). +func TestDatasetPush_NonexistentLocalPath_ExitsThree(t *testing.T) { + code, _, _ := execDatasetPush(t, []string{ + "/tmp/tracebloc-cli-test-no-such-dir-" + t.Name(), + "--table=t1", + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 3 { + t.Fatalf("expected exit 3 for missing local path, got %d", code) + } +} + +// TestDatasetPush_MissingLabelsCSV_ExitsThree: most likely "real +// world" wrong-layout case — customer has images but forgot +// labels.csv. Pins the exit-code contract for the common failure +// mode; the diagnostic-text content is covered by +// internal/push.TestDiscover_MissingLabelsCSV. +func TestDatasetPush_MissingLabelsCSV_ExitsThree(t *testing.T) { + root := t.TempDir() + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write img: %v", err) + } + + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 3 { + t.Fatalf("expected exit 3 for missing labels.csv, got %d", code) + } +} + +// TestDatasetPush_BadKubeconfig_ExitsThree: schema + layout both +// pass; kubeconfig load fails because the injected path doesn't +// exist. The exit-code contract matches `cluster info`'s — same +// class of failure (3 = local input problem) surfaces with the +// same code regardless of which command tripped it. +func TestDatasetPush_BadKubeconfig_ExitsThree(t *testing.T) { + root := imgcLayout(t) + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 3 { + t.Fatalf("expected exit 3 for bad kubeconfig, got %d", code) + } +} + +// TestDatasetPush_RequiresExactlyOneArg: cobra-level Args check +// pins the command signature. Two positional args, or zero, should +// fail before the runner even fires. +func TestDatasetPush_RequiresExactlyOneArg(t *testing.T) { + cases := []struct { + name string + args []string + }{ + { + name: "no positional", + args: []string{ + "--table=t1", "--category=image_classification", + "--intent=train", "--label-column=label", + }, + }, + { + name: "two positionals", + args: []string{ + "./a", "./b", + "--table=t1", "--category=image_classification", + "--intent=train", "--label-column=label", + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + code, _, _ := execDatasetPush(t, c.args) + if code == 0 { + t.Errorf("expected non-zero exit for %s, got 0", c.name) + } + }) + } +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 0d790ae..1dc4184 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -66,6 +66,7 @@ roadmap. Subsequent phases land subcommands incrementally.`, root.AddCommand(newVersionCmd(info)) root.AddCommand(newIngestCmd()) root.AddCommand(newClusterCmd()) + root.AddCommand(newDatasetCmd()) return root } diff --git a/internal/cluster/pvc.go b/internal/cluster/pvc.go new file mode 100644 index 0000000..fd744d0 --- /dev/null +++ b/internal/cluster/pvc.go @@ -0,0 +1,146 @@ +package cluster + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// SharedPVCClaimName is the chart's hardcoded shared-data PVC name. +// +// From tracebloc/client's templates/_helpers.tpl: +// +// {{- define "tracebloc.clientDataPvc" -}} +// client-pvc +// {{- end }} +// +// The helper isn't parameterized by release name (yet) — every +// installation of the chart creates a PVC literally named +// "client-pvc". We probe by claim name rather than by labels here +// because the chart's labels include the helm release name and we +// already discovered the release via DiscoverParentRelease — once +// we have the namespace, the name is unambiguous. +// +// If a customer renames the PVC (out-of-band patch, mostly), they +// need the v0.2 follow-up that reads the name from +// jobs-manager's volume-mount spec instead of hardcoding. Tracked +// as a future ticket alongside #7 (the ingestor-SA-name discovery). +const SharedPVCClaimName = "client-pvc" + +// SharedPVCMountPath is where jobs-manager mounts the shared PVC +// inside its container. The CLI's staging Pod (Phase 3 PR-b) uses +// the same mount path so any tooling that introspects "where files +// live" in either context agrees. From jobs-manager-deployment.yaml: +// +// volumeMounts: +// - name: shared-volume +// mountPath: "/data/shared" +const SharedPVCMountPath = "/data/shared" + +// SharedPVC describes the chart's shared-data PVC after discovery. +// Carries enough metadata for Phase 3 PR-b to construct a stage Pod +// that can mount the same claim. +type SharedPVC struct { + // ClaimName is the metadata.name of the PVC, always + // SharedPVCClaimName today. Wrapped in a field rather than + // re-using the constant so a future "discovered via labels" + // implementation can vary the name without changing callers. + ClaimName string + + // MountPath is the in-container directory where the chart's + // pods mount this claim — SharedPVCMountPath today. Same + // rationale as ClaimName for being a field. + MountPath string + + // AccessModes is the resolved access-mode list on the PVC's + // spec. Surfaces in `tracebloc cluster info` (eventually) and + // drives the stage-Pod scheduling story: + // + // - ReadWriteMany: stage Pod can schedule on any node + // - ReadWriteOnce: stage Pod must land on the same node + // as whatever else is using the PVC (jobs-manager, + // mysql-client). PR-b will surface a warning here so + // RWO clusters get diagnostic guidance up front. + AccessModes []corev1.PersistentVolumeAccessMode + + // Phase is the PVC's current bind phase. We check Bound — an + // Unbound PVC means the cluster never provisioned storage + // (e.g. missing StorageClass), and the stage Pod would hang + // indefinitely waiting for a volume. + Phase corev1.PersistentVolumeClaimPhase +} + +// DiscoverSharedPVC verifies the chart's shared-data PVC exists in +// the given namespace and returns its metadata. Returns a friendly +// error if the PVC isn't there or isn't Bound — both are real +// situations Phase 3's pre-flight catches before we waste time +// constructing a Pod that can't mount anything. +func DiscoverSharedPVC(ctx context.Context, cs kubernetes.Interface, namespace string) (*SharedPVC, error) { + pvc, err := cs.CoreV1().PersistentVolumeClaims(namespace). + Get(ctx, SharedPVCClaimName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // The 80% case for "the CLI's pre-flight failed against + // a cluster that has SOME tracebloc release installed + // but not the parent client chart" — the parent-release + // check (DiscoverParentRelease) catches the no-release + // case first, so reaching here usually means the chart + // was installed with a customized PVC name. + return nil, fmt.Errorf( + "no PersistentVolumeClaim named %q found in namespace %q. "+ + "The chart's _helpers.tpl pins this name; if your install "+ + "renamed it out-of-band, the CLI doesn't yet support that "+ + "(read-name-from-jobs-manager is a v0.2 follow-up). "+ + "Verify with: kubectl get pvc -n %s", + SharedPVCClaimName, namespace, namespace) + } + // Forbidden / network / other — surface as-is so the + // customer can RBAC-debug. Wrapping rather than substituting + // because the underlying %w already carries the useful info. + return nil, fmt.Errorf("reading PVC %s/%s: %w", + namespace, SharedPVCClaimName, err) + } + + if pvc.Status.Phase != corev1.ClaimBound { + // Unbound PVCs are a real cluster-config issue — pre-flight + // surfacing of this is much more helpful than the stage Pod + // silently pending forever. The most common cause is a + // missing StorageClass (the chart's default is the cluster + // default, which may not exist on EKS without + // gp2/gp3-csi configured). + return nil, fmt.Errorf( + "PVC %s/%s is in phase %q, not Bound. "+ + "The shared volume hasn't been provisioned — "+ + "check that the cluster has a usable StorageClass "+ + "(kubectl get sc) and that the PVC's storageClassName matches.", + namespace, SharedPVCClaimName, pvc.Status.Phase) + } + + return &SharedPVC{ + ClaimName: pvc.Name, + MountPath: SharedPVCMountPath, + AccessModes: pvc.Spec.AccessModes, + Phase: pvc.Status.Phase, + }, nil +} + +// IsReadWriteMany reports whether the PVC accepts simultaneous +// mounts from multiple nodes. The Phase 3 stage Pod cares about +// this because RWO claims force same-node scheduling — and if the +// existing mounter (jobs-manager) is on a different node than +// where the scheduler wants to put our stage Pod, the Pod will +// pend indefinitely. PR-b will surface a pre-flight warning when +// this returns false; the dataset push still proceeds (eventually +// succeeds when the scheduler co-locates), it just takes longer. +func (p *SharedPVC) IsReadWriteMany() bool { + for _, m := range p.AccessModes { + if m == corev1.ReadWriteMany { + return true + } + } + return false +} diff --git a/internal/cluster/pvc_test.go b/internal/cluster/pvc_test.go new file mode 100644 index 0000000..3260ed7 --- /dev/null +++ b/internal/cluster/pvc_test.go @@ -0,0 +1,117 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// seedPVC returns a corev1.PersistentVolumeClaim wired to look like +// the chart's shared-data claim. Lets each test mutate one field +// (phase, access mode) to exercise its specific branch. +func seedPVC(phase corev1.PersistentVolumeClaimPhase, modes ...corev1.PersistentVolumeAccessMode) *corev1.PersistentVolumeClaim { + if len(modes) == 0 { + modes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + } + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: SharedPVCClaimName, + Namespace: "tracebloc", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: modes, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: phase, + }, + } +} + +func TestDiscoverSharedPVC_HappyPath(t *testing.T) { + cs := fake.NewClientset(seedPVC(corev1.ClaimBound)) + got, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("DiscoverSharedPVC: %v", err) + } + if got.ClaimName != SharedPVCClaimName { + t.Errorf("ClaimName = %q, want %q", got.ClaimName, SharedPVCClaimName) + } + if got.MountPath != SharedPVCMountPath { + t.Errorf("MountPath = %q, want %q", got.MountPath, SharedPVCMountPath) + } + if got.Phase != corev1.ClaimBound { + t.Errorf("Phase = %v, want Bound", got.Phase) + } + if !got.IsReadWriteMany() { + t.Errorf("IsReadWriteMany() = false on a RWX-seeded PVC") + } +} + +func TestDiscoverSharedPVC_NotFound(t *testing.T) { + // Empty cluster — the parent-release check should normally + // have failed first, but we still pin the diagnostic for the + // case where customers renamed the PVC out-of-band. + cs := fake.NewClientset() + _, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err == nil { + t.Fatal("DiscoverSharedPVC returned nil error on empty cluster") + } + // Diagnostic must point at the v0.2 follow-up and `kubectl + // get pvc` command — those are the actionable bits. + for _, want := range []string{SharedPVCClaimName, "v0.2", "kubectl get pvc"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %s", want, err) + } + } +} + +func TestDiscoverSharedPVC_UnboundPhase(t *testing.T) { + // Most common cause: missing StorageClass on EKS where the + // admin removed gp2-default before installing the chart. The + // pre-flight diagnostic must call out StorageClass explicitly + // so customers know what to fix. + cs := fake.NewClientset(seedPVC(corev1.ClaimPending)) + _, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err == nil { + t.Fatal("DiscoverSharedPVC returned nil error on Pending PVC") + } + for _, want := range []string{"Pending", "not Bound", "StorageClass"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %s", want, err) + } + } +} + +func TestIsReadWriteMany_RWO(t *testing.T) { + // ReadWriteOnce — common on cheap-storage clusters (single-node + // minikube, single-zone EBS). Phase 3 still works against RWO + // (the scheduler co-locates), but PR-b will print a warning. + // Pin the API surface that the warning logic will key off. + cs := fake.NewClientset(seedPVC(corev1.ClaimBound, corev1.ReadWriteOnce)) + got, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("DiscoverSharedPVC: %v", err) + } + if got.IsReadWriteMany() { + t.Errorf("IsReadWriteMany() = true on a RWO-seeded PVC") + } +} + +func TestIsReadWriteMany_RWXMixedWithOther(t *testing.T) { + // A few cloud providers (specifically EFS-on-EKS) advertise + // both RWX and ROX. As long as RWX is in the list, our stage + // Pod can schedule freely. + cs := fake.NewClientset(seedPVC(corev1.ClaimBound, + corev1.ReadOnlyMany, corev1.ReadWriteMany)) + got, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("DiscoverSharedPVC: %v", err) + } + if !got.IsReadWriteMany() { + t.Errorf("IsReadWriteMany() = false on a mixed-mode PVC including RWX") + } +} diff --git a/internal/push/spec.go b/internal/push/spec.go new file mode 100644 index 0000000..ec727b0 --- /dev/null +++ b/internal/push/spec.go @@ -0,0 +1,186 @@ +// Package push owns the `tracebloc dataset push` flow: synthesizing +// an ingest spec from CLI flags, walking the customer's local data +// directory, and (in a follow-up PR) staging files into the cluster's +// shared PVC via an ephemeral Pod + tar-over-exec stream. +// +// Phase 3 lands in two PRs (per tracebloc/client#151): +// +// - PR-a (this one): the no-op-safe path — spec synthesis, local +// layout discovery, PVC discovery, --dry-run. Everything up to +// "ready to copy files" without actually copying. +// - PR-b (next): the novel-engineering core — ephemeral stage Pod, +// client-go remotecommand executor, tar stream, progress bar, +// SIGINT-safe cleanup. +// +// The split keeps each diff reviewable. PR-a's purpose is "fail fast +// before we touch the cluster"; PR-b is "now actually push the bytes". +package push + +import ( + "fmt" + "path" + "regexp" +) + +// tableNamePattern is the safe character set for a table name. It +// must satisfy TWO independent constraints simultaneously: +// +// 1. A valid unquoted MySQL identifier — the chart's ingestor +// CREATEs a table with this exact name. +// 2. A safe single path segment — the name becomes the +// /data/shared/
/ subdirectory on the PVC. +// +// The intersection of "MySQL identifier" and "single safe path +// component" is [A-Za-z0-9_]: letters, digits, underscore. No +// slashes, no dots — which is what closes the path-traversal hole +// (see ValidateTableName). +// +// All the real-world example tables (chest_xrays_train, +// cats_dogs_train) match this; it's the conventional snake_case +// table-naming style anyway. +var tableNamePattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`) + +// ValidateTableName rejects table names that aren't safe as both a +// MySQL identifier and a single PVC path segment. +// +// Why a CLI-side check rather than the schema: the embedded +// ingest.v1.json only enforces `minLength: 1` on `table` — no +// `pattern`. Without this guard, --table=../../etc would flow into +// the /data/shared/
/ PVC path; PR-b's stage Pod would then +// write outside the intended subtree and could clobber another +// table's data. Tightening the upstream schema with a `pattern` +// is the proper long-term fix (it would protect the helm flow + +// jobs-manager too) but needs a change to tracebloc/data-ingestors' +// schema, which the schema-drift CI check pins — filed as +// tracebloc/data-ingestors#116. Once that lands and we re-sync, +// this guard can collapse to a thin "schema says so" wrapper. +// +// Callers MUST run this before SpecArgs.Build() or StagedPrefix(), +// both of which assume a validated name. +func ValidateTableName(table string) error { + if table == "" { + return fmt.Errorf("table name is required (set --table)") + } + if !tableNamePattern.MatchString(table) { + return fmt.Errorf( + "table name %q is invalid: must match [A-Za-z0-9_]+ "+ + "(letters, digits, underscore only). The table name is "+ + "used both as the MySQL table identifier and as the "+ + "/data/shared/
/ subdirectory on the cluster PVC, "+ + "so slashes, dots, and path-traversal sequences are "+ + "rejected.", + table) + } + return nil +} + +// SpecArgs is the user-facing flag set for `tracebloc dataset push`. +// +// It's intentionally narrower than the full ingest.v1.json schema — +// for v0.1, only the image_classification minimum is exposed. The +// epic (#147) defers all other categories to v0.2 as one-PR +// additions; the flag set will grow when those land. +// +// Validation is NOT enforced here. Build() produces an +// ingest.v1.json-conforming map, and the caller pipes it through +// internal/schema's V1 validator. Duplicating "intent must be +// train|test" or similar in Go-side code would drift from the +// embedded schema — the schema is the single source of truth. +type SpecArgs struct { + // Table is the destination MySQL table name in the cluster. Also + // used as the per-table subdirectory under /data/shared/ so + // pushes of multiple tables don't collide on the PVC. + Table string + + // Category pins the task family. For v0.1, only + // "image_classification" is supported end-to-end — the epic + // non-goals defer other categories to v0.2. The flag accepts + // other values so the schema's enum check produces the canonical + // error message (rather than a CLI-side "unknown category" + // that drifts from the schema). + Category string + + // Intent is "train" or "test" per the schema's enum. Same + // rationale as Category for not pre-validating here. + Intent string + + // LabelColumn is the column name in labels.csv that holds the + // label. The schema accepts either a shorthand string (this + // field) or a {column, policy} object; v0.1 only emits the + // shorthand because passthrough is the only policy + // image_classification cares about. + LabelColumn string +} + +// Build produces the ingest.v1.json-conforming spec map. The +// returned map is YAML-marshalable and ready to feed to +// internal/schema's V1 validator. +// +// PVC paths (csv, images) are constructed under +// /data/shared/
/ to match the chart's mount convention +// surfaced by Phase 2's cluster discovery — jobs-manager mounts +// client-pvc at /data/shared, and the per-table subdir prevents +// cross-table collisions when a customer pushes multiple datasets +// to the same release. +// +// File-name conventions inside that subdir (labels.csv, +// images/) are dictated by the local layout that internal/push.Discover +// requires. Keeping the layout convention in lock-step on both +// sides — the CLI's view of "what local files we expect" and the +// spec's view of "where they'll live in the cluster" — means a +// successful Discover guarantees a runnable spec. +// +// PRECONDITION: a.Table must have passed ValidateTableName. Build +// calls StagedPrefix, which panics on an unsafe name. +func (a SpecArgs) Build() map[string]any { + prefix := StagedPrefix(a.Table) + return map[string]any{ + "apiVersion": "tracebloc.io/v1", + "kind": "IngestConfig", + "category": a.Category, + "table": a.Table, + "intent": a.Intent, + // Trailing slash on `images` matches the schema example + // (data-ingestors/examples/yaml/image_classification.yaml, + // line 14). The ingestor treats it as a directory glob. + "csv": path.Join(prefix, "labels.csv"), + "images": path.Join(prefix, "images") + "/", + "label": a.LabelColumn, + } +} + +// StagedPrefix returns the in-cluster destination directory the CLI +// writes files into for a given table. Used in two places that +// MUST agree: +// +// 1. Phase 3 (this PR + PR-b): the path the ephemeral stage Pod +// creates and tars files into. +// 2. The csv/images fields in Build() above, which jobs-manager +// reads to know where the ingestor Job will find them. +// +// Exported because Phase 3's PR-b (stage Pod construction) needs +// it from the same place, and Phase 4 (submit) might want to print +// it as part of "what we pushed." +// +// PRECONDITION: table must already have passed ValidateTableName. +// This function panics on an unsafe name rather than returning an +// escape path — a name that escapes /data/shared is a caller bug +// (validation was skipped), and a panic surfaces it loudly in +// tests instead of silently letting PR-b's stage Pod write to, +// say, /etc. Every production call path runs ValidateTableName +// first (see cli.runDatasetPush), so the panic is unreachable in +// correct code. +func StagedPrefix(table string) string { + // Deliberately NOT path.Join here: path.Join cleans ".." + // segments, which is exactly the silent traversal we're + // guarding against. Plain concatenation keeps the name as a + // literal segment so the assertion below can detect a bad one. + prefix := "/data/shared/" + table + if !tableNamePattern.MatchString(table) { + panic(fmt.Sprintf( + "push.StagedPrefix: unsafe table name %q — caller must "+ + "ValidateTableName before constructing a PVC path", + table)) + } + return prefix +} diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go new file mode 100644 index 0000000..8f19716 --- /dev/null +++ b/internal/push/spec_test.go @@ -0,0 +1,173 @@ +package push + +import ( + "testing" + + "gopkg.in/yaml.v3" + + "github.com/tracebloc/cli/internal/schema" +) + +// TestBuild_ImageClassificationMinimum_PassesSchema is the contract +// test that pins Phase 3's flag → spec synthesis to the embedded +// schema. The whole point of Build() is "produce something the +// canonical validator accepts" — if a refactor breaks that, every +// `dataset push` invocation fails after kubeconfig load but before +// the user sees a useful error, so we want this caught in CI. +func TestBuild_ImageClassificationMinimum_PassesSchema(t *testing.T) { + args := SpecArgs{ + Table: "chest_xrays_train", + Category: "image_classification", + Intent: "train", + LabelColumn: "image_label", + } + spec := args.Build() + + // Round-trip via YAML because the validator's public API is + // YAML-input. The map → JSON → YAML → parse-back chain is a + // microscopic cost per `dataset push` invocation; not worth + // adding a Validate(parsed) method to internal/schema for v0.1. + specBytes, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("yaml.Marshal: %v", err) + } + + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + _, errs, parseErr := v.ValidateYAML(specBytes) + if parseErr != nil { + t.Fatalf("ValidateYAML returned parse error on our own output: %v\n%s", parseErr, specBytes) + } + if len(errs) != 0 { + t.Fatalf("synthesized spec failed schema validation: %s\nspec:\n%s", + schema.FormatErrors(errs), specBytes) + } +} + +// TestBuild_PathsMatchStagedPrefix pins the contract between Phase 3 +// (where the CLI puts files on the PVC) and Phase 4 (what paths the +// submitted spec tells jobs-manager to look at). If these ever +// drift, the ingestor Job spawned by jobs-manager won't find the +// files we just staged — a silent "0 rows ingested" outcome that's +// hard to debug. +func TestBuild_PathsMatchStagedPrefix(t *testing.T) { + const table = "cats_dogs" + spec := SpecArgs{ + Table: table, + Category: "image_classification", + Intent: "train", + LabelColumn: "label", + }.Build() + + prefix := StagedPrefix(table) + wantCSV := prefix + "/labels.csv" + wantImages := prefix + "/images/" + + if got := spec["csv"].(string); got != wantCSV { + t.Errorf("spec.csv = %q, want %q", got, wantCSV) + } + if got := spec["images"].(string); got != wantImages { + t.Errorf("spec.images = %q, want %q", got, wantImages) + } +} + +// TestBuild_LeavesValidationToSchema asserts that Build() does NOT +// pre-validate. A garbage category goes through unchanged so the +// schema's enum check produces the canonical error message. The +// alternative — duplicating the schema's enum in Go — would drift +// the moment data-ingestors adds a new category and we forget to +// mirror it here. +func TestBuild_LeavesValidationToSchema(t *testing.T) { + spec := SpecArgs{ + Table: "x", + Category: "definitely-not-a-real-category", + Intent: "train", + LabelColumn: "label", + }.Build() + + if got := spec["category"].(string); got != "definitely-not-a-real-category" { + t.Errorf("Build() pre-validated category; want raw passthrough, got %q", got) + } +} + +// TestStagedPrefix_PerTableIsolation pins the contract that two +// concurrent pushes for different tables don't write to the same +// PVC subdirectory. If this ever returns the same path for +// different tables, parallel `dataset push` calls would race on +// labels.csv overwrites. +func TestStagedPrefix_PerTableIsolation(t *testing.T) { + if a, b := StagedPrefix("cats"), StagedPrefix("dogs"); a == b { + t.Errorf("StagedPrefix(%q) == StagedPrefix(%q) = %q, want distinct", "cats", "dogs", a) + } + if got := StagedPrefix("table_a"); got != "/data/shared/table_a" { + t.Errorf("StagedPrefix(%q) = %q, want /data/shared/table_a", "table_a", got) + } +} + +// TestValidateTableName_Accepts pins the names that MUST pass — +// the real-world example tables plus a few edge shapes (single +// char, leading underscore, mixed case, digits). A regression +// that rejects a valid name would break legitimate pushes. +func TestValidateTableName_Accepts(t *testing.T) { + for _, name := range []string{ + "cats_dogs", + "chest_xrays_train", + "t1", + "ABC", + "table_123", + "_leading_underscore", + "9starts_with_digit", // valid MySQL identifier + safe path segment + } { + if err := ValidateTableName(name); err != nil { + t.Errorf("ValidateTableName(%q) = %v, want nil", name, err) + } + } +} + +// TestValidateTableName_RejectsUnsafe is the security-regression +// pin. The path-traversal cases (../../etc, ../foo) are the ones +// Bugbot flagged on PR #8 — if this test ever goes green with +// those removed, the traversal hole is back open. +func TestValidateTableName_RejectsUnsafe(t *testing.T) { + cases := map[string]string{ + "empty": "", + "parent traversal": "../../etc", + "single parent": "../foo", + "embedded slash": "foo/bar", + "embedded dot": "foo.bar", + "bare dot": ".", + "absolute": "/etc/passwd", + "space": "my table", + "dash": "cats-dogs", // not a valid unquoted MySQL identifier + "trailing newline": "foo\n", + "shell metachar": "foo;rm", + "null-ish unicode": "foo\x00bar", + } + for desc, table := range cases { + if err := ValidateTableName(table); err == nil { + t.Errorf("%s: ValidateTableName(%q) = nil, want a rejection error", desc, table) + } + } +} + +// TestStagedPrefix_PanicsOnUnsafeName pins the defense-in-depth +// backstop: if a caller skips ValidateTableName and hands a +// traversal name straight to StagedPrefix, it must panic rather +// than silently return an escape path. PR-b adds new call sites +// for StagedPrefix — this test guards against one of them +// forgetting the validation step. +func TestStagedPrefix_PanicsOnUnsafeName(t *testing.T) { + for _, unsafe := range []string{"../../etc", "foo/bar", ""} { + t.Run(unsafe, func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("StagedPrefix(%q) did not panic; an unsafe "+ + "name must panic, not return an escape path", unsafe) + } + }() + _ = StagedPrefix(unsafe) + }) + } +} diff --git a/internal/push/walk.go b/internal/push/walk.go new file mode 100644 index 0000000..dd95571 --- /dev/null +++ b/internal/push/walk.go @@ -0,0 +1,244 @@ +package push + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Size limits enforced before we touch the cluster. Both caps are +// soft engineering choices, not protocol limits — they exist +// because tar-over-exec via client-go's remotecommand executor has +// a memory profile that degrades steeply past ~1 GB total transfer. +// Customers hitting these get pointed at the v0.2 cloud-source +// story (S3/GCS/HTTPS sources, currently in epic non-goals). +// +// The single-file cap is stricter than the total cap because the +// streaming buffer for one file lives in memory longer than the +// inter-file overhead — a 1 GB single file is worse than ten 100 MB +// files for the executor's working-set. +const ( + // MaxTotalBytes is the v0.1 ceiling on the sum of all files in + // a single `dataset push`. Picked from the epic's stated + // "anything above ~1GB needs the cloud-source story (v0.2)." + MaxTotalBytes int64 = 1 * 1024 * 1024 * 1024 + + // MaxSingleFileBytes caps any single file. Tuned via the + // data-ingestors templates' largest sample image (~30 KB) and + // a 10000x safety margin for typical user uploads. Files in the + // hundreds-of-MB range work in testing but degrade noticeably. + MaxSingleFileBytes int64 = 500 * 1024 * 1024 +) + +// LocalLayout describes a validated local directory ready to stage. +// All paths are absolute, resolved against the customer's working +// directory before this struct is returned. +type LocalLayout struct { + // Root is the absolute path the customer passed (after cleanup). + Root string + + // LabelsCSV is the absolute path to labels.csv inside Root. + // Required for image_classification. + LabelsCSV string + + // Images is the list of absolute paths to image files under + // Root/images/. Order is filesystem-walk order — Discover + // doesn't sort, so callers that need determinism (e.g. + // reproducible-build tests) sort before use. + Images []string + + // TotalBytes is the sum of all files Discover will stage — + // labels.csv plus every entry in Images. Pre-computed during + // the walk so the size-cap check + the progress bar (PR-b) + // can read it without re-stat'ing. + TotalBytes int64 +} + +// imageExtensions accepts the file types the chart's +// image_classification ingestor processes by default. From +// data-ingestors' FileTypeValidator(images) defaults: .jpg, .jpeg, +// .png. The chart's defaults file (see chartversion 1.3.5+) also +// accepts .webp; we mirror that here so customers on a recent +// chart can stage webp files without hitting "no images found." +// +// Comparison is case-insensitive — filesystems vary (case-sensitive +// on Linux, case-preserving-but-insensitive on macOS default APFS). +var imageExtensions = map[string]struct{}{ + ".jpg": {}, + ".jpeg": {}, + ".png": {}, + ".webp": {}, +} + +// Discover walks rootDir and validates it matches the layout Phase 3 +// expects for image_classification: +// +// - /labels.csv (required) +// - /images/*.{jpg,jpeg,png,webp} (at least one file) +// +// Returns specific errors keyed to the layout mistakes a customer +// is most likely to hit — these surface as the CLI's diagnostic +// output before any cluster work, so they're a primary UX surface. +// +// Enforces both v0.1 size caps (MaxTotalBytes, MaxSingleFileBytes); +// over-cap returns ErrTooBig with a pointer to the cloud-source +// story. +func Discover(rootDir string) (*LocalLayout, error) { + abs, err := filepath.Abs(rootDir) + if err != nil { + return nil, fmt.Errorf("resolving %q: %w", rootDir, err) + } + + st, err := os.Stat(abs) + if err != nil { + // Stat covers both "path doesn't exist" and "permission + // denied" via the wrapped fs.PathError; the customer sees + // the underlying message which is already clear. + return nil, fmt.Errorf("reading dataset directory %q: %w", abs, err) + } + if !st.IsDir() { + return nil, fmt.Errorf( + "%q is not a directory; pass the directory containing labels.csv + images/", + abs) + } + + layout := &LocalLayout{Root: abs} + + // labels.csv (required). + labelsPath := filepath.Join(abs, "labels.csv") + labelsStat, err := os.Stat(labelsPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing labels.csv in %q. The CLI expects "+ + "/labels.csv + /images/ for image_classification; "+ + "see https://github.com/tracebloc/client/issues/147 for the "+ + "v0.1 layout contract.", + abs) + } + return nil, fmt.Errorf("stat labels.csv: %w", err) + } + if labelsStat.IsDir() { + // A directory literally named "labels.csv" passes the + // os.Stat above — without this check the pre-flight would + // accept it, and PR-b's tar stream would fail confusingly + // trying to read a directory as a CSV. Symmetric with the + // imagesStat.IsDir() check below. + return nil, fmt.Errorf( + "%q is a directory, not a file. labels.csv must be the "+ + "CSV file holding the image_id,label rows.", + labelsPath) + } + if labelsStat.Size() > MaxSingleFileBytes { + return nil, sizeError("labels.csv", labelsStat.Size(), MaxSingleFileBytes) + } + layout.LabelsCSV = labelsPath + layout.TotalBytes += labelsStat.Size() + + // images/ subdir (required, must contain at least one + // image-extension file). + imagesDir := filepath.Join(abs, "images") + imagesStat, err := os.Stat(imagesDir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing images/ subdirectory in %q. The CLI expects "+ + "/labels.csv + /images/*.{jpg,jpeg,png,webp}.", + abs) + } + return nil, fmt.Errorf("stat images/: %w", err) + } + if !imagesStat.IsDir() { + return nil, fmt.Errorf("%q exists but is not a directory", imagesDir) + } + + // Walk just the images/ directory — we don't recurse, image + // classification's layout is flat. If a customer has nested + // subdirs (e.g. images/cats/ + images/dogs/), that's a + // different category convention and out of scope for v0.1. + entries, err := os.ReadDir(imagesDir) + if err != nil { + return nil, fmt.Errorf("reading images/: %w", err) + } + for _, entry := range entries { + if entry.IsDir() { + // Silently skip subdirectories so a stray .DS_Store or + // thumbnails dir doesn't error out the whole walk. + // We DO surface the count of accepted images at the + // end, so a customer with all-nested-subdirs gets + // "0 images found" which is the right diagnostic. + continue + } + ext := strings.ToLower(filepath.Ext(entry.Name())) + if _, ok := imageExtensions[ext]; !ok { + continue + } + info, err := entry.Info() + if err != nil { + return nil, fmt.Errorf("stat %q: %w", entry.Name(), err) + } + if info.Size() > MaxSingleFileBytes { + return nil, sizeError(filepath.Join("images", entry.Name()), + info.Size(), MaxSingleFileBytes) + } + layout.Images = append(layout.Images, filepath.Join(imagesDir, entry.Name())) + layout.TotalBytes += info.Size() + } + + if len(layout.Images) == 0 { + return nil, fmt.Errorf( + "no image files found in %q. Expected .jpg, .jpeg, .png, or .webp; "+ + "got %d non-image entries.", + imagesDir, len(entries)) + } + + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. "+ + "For larger datasets, the cloud-source path (S3/GCS/HTTPS) "+ + "is on the v0.2 roadmap — see tracebloc/client#147 non-goals. "+ + "Workaround for v0.1: split the push into multiple smaller "+ + "tables, or stage directly via the existing helm flow.", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + + return layout, nil +} + +// sizeError builds the over-the-single-file-cap error with the same +// human-readable framing as the total-cap branch above. Centralized +// so the message stays consistent if we tune the wording later. +func sizeError(relPath string, got, cap int64) error { + return fmt.Errorf( + "file %q is %s, exceeds v0.1 single-file cap of %s. "+ + "For larger files, see tracebloc/client#147's v0.2 cloud-source story.", + relPath, HumanBytes(got), HumanBytes(cap)) +} + +// HumanBytes renders a byte count in the shortest readable unit. +// Not internationalized — the CLI is English-only for v0.1. +// +// Exported because the CLI's pre-flight summary (internal/cli) needs +// the identical formatting — keeping one implementation here means +// the size shown in an over-cap error and the size shown in the +// dry-run summary can never drift (Bugbot flagged the earlier +// copy-pasted variant on PR #8). +func HumanBytes(n int64) string { + const ( + KiB = 1024 + MiB = 1024 * KiB + GiB = 1024 * MiB + ) + switch { + case n >= GiB: + return fmt.Sprintf("%.2f GiB", float64(n)/float64(GiB)) + case n >= MiB: + return fmt.Sprintf("%.2f MiB", float64(n)/float64(MiB)) + case n >= KiB: + return fmt.Sprintf("%.2f KiB", float64(n)/float64(KiB)) + default: + return fmt.Sprintf("%d B", n) + } +} diff --git a/internal/push/walk_test.go b/internal/push/walk_test.go new file mode 100644 index 0000000..8095415 --- /dev/null +++ b/internal/push/walk_test.go @@ -0,0 +1,247 @@ +package push + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// imgcDir builds a valid image_classification layout under t.TempDir() +// and returns its absolute path. Used as the happy-path baseline that +// individual negative-case tests mutate. +func imgcDir(t *testing.T, withImages ...string) string { + t.Helper() + root := t.TempDir() + + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n002.jpg,dog\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images/: %v", err) + } + if len(withImages) == 0 { + withImages = []string{"001.jpg", "002.jpg"} + } + for _, name := range withImages { + // 100 bytes of stub data per image — keeps the total-bytes + // math predictable in TestDiscover_TotalBytesSum without + // generating real image headers. + if err := os.WriteFile(filepath.Join(imagesDir, name), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image %s: %v", name, err) + } + } + return root +} + +func TestDiscover_HappyPath(t *testing.T) { + root := imgcDir(t) + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if got.Root == "" || !filepath.IsAbs(got.Root) { + t.Errorf("Root = %q, want non-empty absolute path", got.Root) + } + if filepath.Base(got.LabelsCSV) != "labels.csv" { + t.Errorf("LabelsCSV basename = %q, want labels.csv", filepath.Base(got.LabelsCSV)) + } + if len(got.Images) != 2 { + t.Errorf("len(Images) = %d, want 2", len(got.Images)) + } +} + +func TestDiscover_TotalBytesSum(t *testing.T) { + // Two 100-byte images + the inline labels.csv string (39 bytes: + // "image_id,label\n001.jpg,cat\n002.jpg,dog\n"). 100+100+39 = 239. + // This pins the pre-cluster size summary the dry-run output + // prints — if we ever undercount, customers see "0 bytes" + // pre-push and panic. + root := imgcDir(t) + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + const want = int64(100 + 100 + 39) + if got.TotalBytes != want { + t.Errorf("TotalBytes = %d, want %d", got.TotalBytes, want) + } +} + +func TestDiscover_AcceptsAllImageExtensions(t *testing.T) { + // Mirror the chart's FileTypeValidator(images) defaults — if a + // customer's image-set has .png + .webp, both should stage. + root := imgcDir(t, "a.jpg", "b.jpeg", "c.png", "d.webp", "e.JPG") + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if len(got.Images) != 5 { + t.Errorf("len(Images) = %d, want 5 (case-insensitive); names=%v", + len(got.Images), got.Images) + } +} + +func TestDiscover_SkipsNonImageFiles(t *testing.T) { + // .DS_Store, thumbnails.db, sibling .yaml etc. should be + // silently skipped — not error-out, not stage. The "no image + // files" diagnostic only fires when *zero* accepted files + // remain after filtering. + root := imgcDir(t, "real.jpg") + if err := os.WriteFile(filepath.Join(root, "images", ".DS_Store"), + make([]byte, 50), 0o644); err != nil { + t.Fatalf("write .DS_Store: %v", err) + } + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if len(got.Images) != 1 { + t.Errorf("len(Images) = %d, want 1; .DS_Store should be filtered", len(got.Images)) + } +} + +func TestDiscover_MissingLabelsCSV(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "images"), 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "images", "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image: %v", err) + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected missing-labels error") + } + if !strings.Contains(err.Error(), "missing labels.csv") { + t.Errorf("error = %q, want it to mention missing labels.csv", err) + } +} + +func TestDiscover_MissingImagesDir(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("x"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected missing-images-dir error") + } + if !strings.Contains(err.Error(), "missing images/") { + t.Errorf("error = %q, want it to mention missing images/", err) + } +} + +func TestDiscover_NoAcceptedImageExtensions(t *testing.T) { + // images/ exists but only contains .gif and .bmp — neither + // in our accept-set. Customer should see "no image files" + // pointing at the accepted extensions, not a successful walk + // with len(Images)==0 that then succeeds the dry-run. + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("x"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + for _, n := range []string{"old.gif", "old.bmp"} { + if err := os.WriteFile(filepath.Join(imagesDir, n), []byte("x"), 0o644); err != nil { + t.Fatalf("write %s: %v", n, err) + } + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected no-images error") + } + if !strings.Contains(err.Error(), "no image files") { + t.Errorf("error = %q, want it to mention no image files", err) + } +} + +func TestDiscover_LabelsCSVIsDirectory(t *testing.T) { + // A directory literally named "labels.csv" — os.Stat succeeds, + // so without the IsDir guard the pre-flight would accept it and + // PR-b's tar stream would fail confusingly. Symmetric with the + // images/ check. Bugbot flagged the missing guard on PR #8. + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "labels.csv"), 0o755); err != nil { + t.Fatalf("mkdir labels.csv/: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images/: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image: %v", err) + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected labels.csv-is-a-directory error") + } + if !strings.Contains(err.Error(), "is a directory") { + t.Errorf("error = %q, want it to mention 'is a directory'", err) + } +} + +func TestDiscover_NotADirectory(t *testing.T) { + // Customer passes a path to a single file instead of a dir. + // This is a common autocomplete-mistake (tab-completing + // into the file). Should be caught early with a clear error. + root := t.TempDir() + filePath := filepath.Join(root, "looks-like-data.tar") + if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + _, err := Discover(filePath) + if err == nil { + t.Fatal("Discover returned nil error; expected not-a-directory error") + } + if !strings.Contains(err.Error(), "not a directory") { + t.Errorf("error = %q, want it to mention not a directory", err) + } +} + +func TestDiscover_OverSingleFileCap(t *testing.T) { + // Use a fake-size pattern: create a real small file but assert + // the cap logic by exercising the human-readable error format + // at the boundary. We can't easily generate a 500MB+ file in + // CI without slowing the suite — instead pin the human-bytes + // formatter (which is what the customer sees) via its own + // boundary test below, and exercise sizeError() directly. + got := sizeError("images/big.jpg", 600*1024*1024, MaxSingleFileBytes).Error() + for _, want := range []string{"images/big.jpg", "600.00 MiB", "500.00 MiB", "v0.2", "cloud-source"} { + if !strings.Contains(got, want) { + t.Errorf("sizeError missing %q in: %s", want, got) + } + } +} + +func TestHumanBytes(t *testing.T) { + // Boundary check: the formatter is what surfaces in every + // diagnostic, so a regression here makes the error messages + // unreadable for the customer. Pin a few representative values. + cases := []struct { + in int64 + want string + }{ + {0, "0 B"}, + {1023, "1023 B"}, + {1024, "1.00 KiB"}, + {1024 * 1024, "1.00 MiB"}, + {1024 * 1024 * 1024, "1.00 GiB"}, + {500 * 1024 * 1024, "500.00 MiB"}, + } + for _, c := range cases { + if got := HumanBytes(c.in); got != c.want { + t.Errorf("HumanBytes(%d) = %q, want %q", c.in, got, c.want) + } + } +} From b3efc62232d3adfbcf247f4c871a4e2fa2d19a0c Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Fri, 22 May 2026 19:21:17 +0500 Subject: [PATCH 04/17] feat(dataset): stage Pod + tar-over-exec stream (PR-b of #151) (#9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dataset): stage Pod + tar-over-exec stream (PR-b of #151) Completes Phase 3 (tracebloc/client#151). PR-a delivered the no-op-safe pre-flight; this PR-b plugs in the actual file staging — ephemeral Pod, client-go SPDY executor, tar stream, SIGINT-safe cleanup. ## What lands internal/push/ (5 new files): - pod.go Stage Pod spec builder + Create/Wait/Delete lifecycle. Alpine 3.20 pinned by digest, PSA- restricted security context, activeDeadline- Seconds=600 in-cluster self-kill defense-in-depth, random 8-hex-char suffix for parallel-push collision avoidance. - stream.go Executor interface (SPDYExecutor in prod, fakeExecutor in tests). StreamLayout wires a tar.Writer through an io.Pipe to the exec stdin, running `tar -xf - -C /data/shared/
` in the Pod. Captures remote stderr into the error surface so customers see "no space left on device" verbatim, not a generic exec failure. - progress.go schollz/progressbar/v3 wrapper, TTY-detected via golang.org/x/term. No-op in CI / non-TTY output so the bar doesn't pollute log streams. - orphan.go Scans for stage Pods labeled managed-by=tracebloc-cli older than 5 min; renders an actionable warning with the kubectl-delete command. Warn-only in v0.1 (auto-cleanup is v0.2 — can't distinguish "crashed previous push" from "still-running parallel push from another workstation"). - stage.go The Stage() orchestrator. Order: orphan scan → CreateStagePod → defer DeleteStagePod with a FRESH context (SIGINT-safe — defers fire even when the parent ctx is cancelled) → WaitForReady → StreamLayout → cleanup. On any error past Create, the deferred delete still runs. internal/cli/dataset.go: - Replace the exit-6 stub with push.Stage(...) wired to cluster-discovered ns/PVC/SA + the new SPDYExecutor. - Add v0.1 category gate (Medium-1 from PR-a self-review). Runs BEFORE schema validation so unsupported-but-schema- valid categories like tabular_classification get the actionable "v0.1 supports only image_classification" message instead of the schema's confusing "missing property 'schema'". - Add --stage-pod-image flag for air-gapped customers. - Update exit-code doc: drop exit 6 (PR-b stub), add exit 7 (staging-step failed, distinct from pre-flight 3/4). internal/push/stage_test.go covers Medium-2 from PR-a review: TestStage_IngestorSANameFlowsToPod pins that the --ingestor-sa override actually lands on the stage Pod's ServiceAccountName. ## Tricky bits Pipe deadlock guard: when the executor returns early (ctx cancel, remote tar dies immediately), the tar-write goroutine would block on pipe.Write forever because nothing reads. Fix: close the pipe Reader after exec.Exec returns, which unblocks the writer with io.ErrClosedPipe. Caught by TestStage_CancelledContext_StillCleansUp. SIGINT safety: the deferred DeleteStagePod uses a fresh ctx with a 30s deadline rather than the (possibly cancelled) parent ctx. Without this, Ctrl-C right after pod-create leaks an orphan Pod. activeDeadlineSeconds=600 is the in-cluster backstop for the truly-pathological case (hard-kill, network partition between laptop and cluster). errcheck cleanup: writeLayoutTar uses a named return + deferred close that promotes a tar trailer-write error if and only if the function otherwise succeeded — silent truncation would be much worse than a noisy error. ## Test plan - [x] go vet ./... — green - [x] go test -race -cover ./... — green (push 79.8%, cluster 83.2%, schema 80.7%, cli 52.2%) - [x] gofmt -s -l . — no drift - [x] errcheck ./... — green - [x] go build — binary builds - [x] Local smoke: --category=tabular_classification → exit 2 with "v0.1 supports only image_classification" message (the new gate's actionable diagnostic) - [ ] Real EKS smoke (manual; out of CI scope) ## Open items deferred Per PR-a review's Low/Nit list: - runDatasetPushArgs.Context shadowing context.Context - printPushPreflight rendering AccessModes with %v - testutil package consolidation for imgcDir / imgcLayout - dataset_test kubeconfig path traversal via subtests - Discover hint for nested-image-dirs These are tracked for v0.2 cleanup. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): DNS-1123-safe Pod names + wire SIGINT to ctx (Bugbot, High+Med) Two real bugs caught by Bugbot's re-review on PR-b: ## 1. High — Stage Pod name breaks DNS-1123 BuildStagePodSpec used the raw table name as a Pod name segment. ValidateTableName accepts [A-Za-z0-9_]+ (MySQL identifier rules), but K8s Pod names follow DNS-1123 subdomain rules: lowercase alnum + hyphen, must start/end alnum. The canonical example throughout tracebloc docs (cats_dogs_train, snake_case) would fail Pod create post-pre-flight — worst-of-both-worlds UX (pre-flight says "good!" then create fails). Fix: transform the table name into a DNS-1123-safe segment for the Pod name only — lowercase, _→-, trim leading/trailing hyphens, cap length, fallback to "tbl" for the pathological all-underscore case. The original verbatim name still goes in the tracebloc.io/table label so orphan warnings stay traceable. TestDNS1123SafeTableSegment covers cats_dogs → cats-dogs, MyTable → mytable, _leading_underscore → leading-underscore, _ → tbl, 50-char → 30-char truncation. Each result is cross-validated against an inline DNS-1123 regex check. ## 2. Medium — SIGINT skips Pod cleanup push.Stage documented SIGINT-safe cleanup via context cancellation + fresh-ctx deferred DeleteStagePod, but cmd/tracebloc/main.go called Execute() without signal.NotifyContext. Default Go runtime behaviour on SIGINT is to exit without running defers — so every Ctrl-C during staging leaked an orphan Pod until activeDeadlineSeconds (10 min) fired. The docstring was false advertising. Fix: signal.NotifyContext(ctx, SIGINT, SIGTERM) in main, passed to ExecuteContext. First Ctrl-C cancels the cobra ctx → push.Stage's in-flight HTTP cancels → deferred cleanup runs (with its own fresh ctx, also kept). Second Ctrl-C does normal hard-kill (stop() in defer un-registers the handler) — important if the cleanup itself hangs. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): surface NotFound/Forbidden + propagate rand error (Bugbot) Two new findings from Bugbot's re-review on commit 35ae044: ## Medium — WaitForStagePodReady polled through terminal errors The poll callback returned (false, nil) for EVERY Pods.Get error, which meant Pod-deleted-out-of-band (NotFound) or RBAC-revoked-mid- push (Forbidden) waited the full 60s timeout despite the situation being unrecoverable. Customer-facing impact: 60s of "Waiting for stage Pod to be Ready..." for an error that should surface in <1s. Fix: classify the error. NotFound + Forbidden terminate the poll immediately (return (false, err)); everything else stays transient (network blip, brief API unavailability). The apierrors.Is{NotFound,Forbidden} helpers were already imported from the previous fixes, so this is a single-block change. Two new tests pin the contract via t.Now() bounds — if a regression makes either case transient again, the test waits the full 60s and the assertion catches it. ## Low — BuildStagePodSpec swallowed crypto/rand error `suffix, _ := randomSuffix(4)` — if crypto/rand failed (rare but possible on systems with exhausted entropy), the suffix was empty and the Pod name became `tracebloc-stage-cats-dogs-` with a bare trailing hyphen. DNS-1123 rejects that → opaque API error from kube-apiserver instead of a clear local diagnostic. Fix: change BuildStagePodSpec signature to (*Pod, error) and propagate the rand failure. CreateStagePod (the prod caller) already returns an error; tests use a mustBuildStagePod helper that t.Fatal's on the rare path. 8 test call sites updated via find-and-replace, plus the override-image test by hand. The third inline comment ("SIGINT skips pod cleanup", bug_id 24ab9106) is a stale carry-over of the prior-round finding I already fixed in 35ae044 — same bug_id, GitHub auto-shifted the anchor onto the new commit. Replied in-thread pointing at the fix. Bugbot's review-body confirms "2 NEW findings", not 3. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): close symlink size-cap bypass + Phase=Failed short-circuit Round 4 of Bugbot fixes on PR-b. Two new findings on commit 7baa321 (plus 3 stale carry-overs from earlier rounds — same bug IDs as already-fixed findings, GitHub keeps re-anchoring them onto each new commit; Bugbot's review-body confirms "2 NEW findings"). ## Medium-Security — symlink images bypassed size caps The vulnerability: Discover sized image entries via DirEntry.Info() (Lstat-equivalent: returns the symlink's own ~100-byte size). writeTarFile read them via os.Stat + os.Open (which follow symlinks). So a customer with `images/evil.jpg` symlinked to `/var/log/system.log` (multi-GB) or `~/.ssh/id_rsa` could: 1. Pass Discover's MaxSingleFileBytes + MaxTotalBytes caps trivially (cap is 1 GiB; the symlink itself is ~100 bytes). 2. Stream the target's FULL contents into the cluster PVC, where the cluster admin can read them via `kubectl exec`. This is both a size-cap bypass AND an arbitrary-local-file disclosure. Worse, it can fire unintentionally — a customer who ran `ln -s ~/datasets/big-images images` to share data across projects would silently stream gigabytes during what they thought was a small test push. Fix in three layers: 1. walk.go: os.Stat → os.Lstat for the labels.csv check (so the mode bits include ModeSymlink) + new rejectSymlink() helper called for both labels.csv and each image entry. v0.1 rejects symlinks outright with a clear "v0.1 doesn't allow symlinks (security: ...)" message pointing at `cp -L` for materializing and the v0.2 cloud-source story for distributed data. 2. stream.go: writeTarFile's os.Stat → os.Lstat + symlink check too. Defense-in-depth — Discover is the primary fix, but a future refactor that calls writeTarFile directly (e.g. a new resume-from-partial-transfer code path in v0.2) would re-introduce the hole without this layer. 3. New tests pin both rejections (labels.csv-as-symlink + image- as-symlink). Skipped on Windows where symlinks require admin. ## Medium — WaitForStagePodReady didn't short-circuit on Phase=Failed Companion to the NotFound/Forbidden short-circuit landed in the previous commit. The poll positively-terminated on Ready=True but never negatively-terminated on Phase=Failed (container crashed at startup: PSA rejection, image crashloop, OOM at container init) or Phase=Succeeded (stage container's sleep exited unexpectedly). Same 60s timeout symptom for an outcome that's actually decided in <5s. Fix: in the poll body, check Phase after the conditions loop; return a structured error including the podReadyTimeoutHint output (container-status reason + message) so the customer sees "terminated in phase Failed (OOMKilled — ...)" instead of a generic timeout. New test TestWaitForStagePodReady_FailedPhaseIsTerminal pins the <3s elapsed contract for an OOMKilled startup. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): Lstat images dir + table-len cap + hermetic re-push + progress defer Round 5 of Bugbot fixes on PR-b. The "5 potential issues" review-body count corresponds to 5 NEW findings; the other 5 inline comments are stale carry-overs of bug IDs already fixed in earlier rounds (24ab9106, 15e29a3e, 258869d3, 5c577688, 7fcf137f — GitHub auto-anchors them onto each new commit). ## Medium — Symlinked images/ directory bypassed all checks Round 4 fixed symlinked FILES inside images/ (Lstat on each entry). It missed the case where images/ ITSELF is a symlink — `ln -s /etc images` passed os.Stat (which follows symlinks), the IsDir check succeeded, and the loop walked /etc. Same disclosure pattern as the round-4 finding, one layer up. Fix: os.Stat → os.Lstat on imagesDir + rejectSymlink before the IsDir check. Symmetric with the labels.csv treatment from round 4. New test TestDiscover_RejectsSymlinkedImagesDir pins the rejection. ## Medium — Table label could exceed Kubernetes 63-char limit ValidateTableName accepted [A-Za-z0-9_]+ without a length cap, but the stage Pod's tracebloc.io/table label carries the raw name. K8s label values cap at 63 chars (DNS-1123 label rule), so a 100-char name passed pre-flight then failed Pod creation with an opaque label-validation error. Fix: cap at 63 chars (also matches MySQL's 64-char identifier limit with 1 char of headroom for any future ingestion-run-id suffix). New MaxTableNameLength const + boundary test pin the cap. ## Medium — Re-push left stale PVC files The remote command was `mkdir -p && tar -xf - -C `. If a previous push had 3 images and the new push has 2, the PVC ends up with the union — and labels.csv now disagrees with the stage directory, silently breaking ingestion. Fix: `rm -rf && mkdir -p && tar -xf - -C `. Safe because dest = StagedPrefix(table) = /data/shared/
and ValidateTableName has ensured `table` is a single safe segment, so rm -rf can only nuke that one per-table subdir, never the parent /data/shared or sibling tables. TestStreamLayout_RemoteCommand updated to assert the rm AND its ordering before mkdir. ## Low — Progress bar not Finish'd on early Stage failure Stage's deferred Finish lives inside StreamLayout, so a failure earlier in the lifecycle (CreateStagePod, WaitForStagePodReady) returned without clearing the TTY bar. Visual artifact on the customer's terminal after a Pod-create failure. Fix: defer progress.Finish() at the construction site in runDatasetPush. Schollz Finish is idempotent so double-call on happy path is a no-op. ## Hard-link bypass — documented, not fixed Bugbot also flagged that hard links to outside files aren't rejected. Filesystem mode bits don't distinguish a hard link from a regular file the way ModeSymlink distinguishes symlinks, and a high-Nlink check has too many false positives. The implicit security boundary is the CLI's process-level read permissions: a customer can only hard-link files they already have read access to, so this isn't a privilege escalation. v0.2 may add openat(O_NOFOLLOW) sandboxing if customers need harder isolation. Documented in rejectSymlink's docstring as a known limitation alongside the v0.2 plan. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): 30m activeDeadline + portable tar paths (Bugbot round 6) Round 6. Review-body says "2 NEW + 3 unresolved" — Bugbot itself is flagging the 9 carry-overs at this point. Two new findings, both real: ## High — Pod activeDeadlineSeconds (600s) cut too close The deadline starts at Pod CREATION, but image pull (up to 60s) + WaitForStagePodReady ceiling (60s) + worst-case stream (1 GiB at 2 MB/s ≈ 8.5 min) eat the budget. Near-cap customers on slow uplinks could see the kubelet terminate mid-transfer with no useful diagnostic. Fix: bump StagePodActiveDeadline from 600 → 1800 (30 min). Budget breakdown in the const's docstring; comfortable margin for variance. Cost is "an idle alpine Pod with sleep idles for ~22min after a successful push" — ~5 MiB cluster RAM, zero CPU, deleted seconds later by the defer'd cleanup anyway. Test pin: TestStagePodActiveDeadline_CoversFullLifecycle asserts the floor at 1500s so a future regression to 600 is caught. ## Medium — Windows tar paths used backslashes `filepath.Join("images", filepath.Base(abs))` produces `images\file.jpg` on Windows. USTAR / POSIX tar requires forward slashes; the Linux stage Pod's `tar -xf` either rejects or extracts as a flat-named file (collapsing the images/ subdir the ingest spec expects). Breaks the Windows-built CLI. Fix: switch to `path.Join` for the tar HEADER name. Keep `filepath.Join` everywhere else (where the OS-native separator is the right thing). Test pin TestStreamLayout_TarPathsAreForwardSlash asserts no backslashes in any tar entry name, regardless of host OS. ## Carry-overs Bugbot's review body now explicitly says "There are 3 total unresolved" — it knows the carry-overs aren't on the new commit anymore. Counts I'm tracking as known-fixed across earlier rounds: 24ab9106 (SIGINT, r3), 15e29a3e (Pod-Get spin, r3), 258869d3 (rand error, r3), 5c577688 (Failed Pod, r4), 7fcf137f (symlink files, r4), de426248 (hard links, r5 — documented limitation), cd462de9 (symlink dir, r5), a8e9e5c7 (label len, r5), 0b296807 (progress finish, r5). Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): wait-error labels + orphan-scan precision + transactional re-push Round 7 of Bugbot fixes on PR-b. Review-body confirmed "4 NEW + ...". All four real, all four fixed: ## Medium — Wait errors mislabeled as timeout WaitForStagePodReady wrapped EVERY poll exit (NotFound, Forbidden, Phase=Failed, ctx.Canceled from SIGINT) in "did not become Ready within 60s." Misleading: the customer who hit Ctrl-C 2 seconds into the wait would see a "timed out" diagnostic. Fix: branch on errors.Is(err, context.DeadlineExceeded). True timeout keeps the "did not become Ready within %s" wording; early-exit cases surface as "did not reach Ready state: %w" with the actual cause front and center. ## Medium — Orphan-delete hint would nuke parallel pushes FormatOrphansWarning's kubectl-delete command used the `managed-by=tracebloc-cli,component=stage-pod` label selector, which matches every stage Pod in the namespace including LEGITIMATE RUNNING ONES from parallel pushes (other workstations, or even this one's just-started Pod). A customer copy-pasting the suggested cleanup could silently kill someone else's in-progress push. Fix: list specific orphan Pod names in the delete command — `kubectl delete pod -n ...`. Test regression- pins the absence of the label-selector form. ## Medium — Re-push deleted before transfer succeeded Round 5's hermetic-re-push fix (rm -rf $DEST && mkdir + tar) satisfied "no stale files" but not "preserve on failure." A tar mid-stream failure (Ctrl-C, network drop, container OOM) left the customer with NOTHING on the PVC — the previous push's data already nuked, the new push aborted before completing. Fix: extract to .staging, swap on success: rm -rf .staging # cleanup any prior failure mkdir -p .staging && tar -xf - -C .staging rm -rf && mv .staging # swap on success rm -rf .staging # defensive cleanup The shell's && sequencing means swap only fires if tar succeeded. Lost the prior `exec /bin/tar` micro-optimization (can't exec mid-chain) — fine, the shell process is tiny. The window between the destination rm and the mv is sub-ms; closing it fully would need a double-mv (old/new/cleanup) which is v0.2 territory. stream_test.go pins: - mkdir-p of .staging happens - tar extracts to .staging (not directly to dest) - mv from .staging to dest exists - tar happens BEFORE any rm of $DEST (transactional property) - no rm of the parent /data/shared (sibling-table safety) ## Medium — Orphan scan flagged active pushes FindOrphanStagePods's only filter was age > 5min. But pod.go itself budgets ~8.5 min for a 1 GiB transfer — a legitimately-running near-cap push would be mislabeled as orphan by the same customer's next concurrent push, and the (newly-fixed) delete hint would now correctly target that specific running Pod by name, killing it. Fix: skip Phase=Running pods entirely. A Running stage Pod is presumed to be doing real work; activeDeadlineSeconds is its cluster-side safety net. Pods in non-Running phases past grace (Pending stuck on image pull, Failed from crash) still flag — those are the genuine orphan shapes. Two new tests: - TestFindOrphanStagePods_SkipsRunningPods: 30-min-old Running Pod doesn't surface as orphan - TestFindOrphanStagePods_FlagsNonRunningPastGrace: 30-min-old Failed Pod DOES surface (regression guard — narrowing the filter should reduce false positives, not eliminate the warning entirely) Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): unique staging suffix + stream-time size-cap recheck Round 8 of Bugbot fixes. Two new findings on commit 42c5e93, both real: ## Medium — Parallel pushes corrupted staging Two concurrent `dataset push` runs for the same `--table` shared one `$DEST.staging` path on the PVC. Each Pod's `rm -rf $DEST.staging`, `tar -xf -`, `mv` sequence raced against the other, with no coordination. Worst case: push A's `rm -rf` wipes push B's mid-extraction state, B's tar then writes into a partially-removed dir, customer ends up with an interleaved- corrupted-and-half-committed dataset. Fix: each invocation generates a fresh 8-hex-char suffix for the staging dir name (`$DEST.staging-<8hex>`), reusing pod.go's randomSuffix. Two parallel pushes now have distinct staging dirs — no interleaved-write hole. The FINAL `rm $DEST && mv $STAGING $DEST` step is still last-write-wins under concurrent commits, but that's an acceptable v0.1 semantic (concurrent pushes for the same table are inherently undefined; whichever commits last "wins" with a COHERENT dataset, not an interleaved one). New test TestStreamLayout_StagingSuffixIsUniquePerInvocation pins the contract by calling StreamLayout twice and asserting distinct staging suffixes. ## Medium — Stream skipped size-cap enforcement (TOCTOU) Pre-flight Discover checked MaxSingleFileBytes + MaxTotalBytes, but writeTarFile / writeLayoutTar streamed files later via os.Lstat → io.Copy with NO re-check. A file that grew between pre-flight and stream (legitimate dataset prep racing the push, or adversarial overwrites) would silently upload past the 500 MiB / 1 GiB advertised caps. Fix in three layers: 1. writeTarFile re-checks the single-file cap (os.Lstat size vs MaxSingleFileBytes) right before WriteHeader. A file that grew gets a clear "exceeds v0.1 single-file cap" error using the same sizeError formatter Discover uses. 2. writeTarFile now returns (int64, error) — the declared header size. writeLayoutTar accumulates this into a running total and aborts mid-stream if it exceeds MaxTotalBytes. Tests added below to pin the running-total contract. 3. io.Copy → io.CopyN(tw, f, st.Size()) caps the actual byte transfer at the declared header size. Without the cap, a file that grew between Lstat and io.Copy would overflow the tar header — header says N bytes, body has > N → tar archive corruption. CopyN-and-trust truncates instead. Closes both the metadata-side (header size) and body-side (byte stream) halves of the stream-time TOCTOU window. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): hoist randomSuffix above goroutine + clean orphan staging dirs Round 9 of Bugbot fixes. Both findings real: ## Medium — Tar goroutine could deadlock on suffix error The randomSuffix call landed AFTER the tar-write goroutine spawned. If crypto/rand failed (rare but possible on entropy-starved systems — same scenario the round-3 BuildStagePodSpec fix handles), the function returned without closing the pipe or waiting on the goroutine. The goroutine would then block once the ~64 KB pipe buffer filled, leaking forever. Fix: hoist randomSuffix (and the entire remoteCmd composition) ABOVE the io.Pipe + goroutine setup. The function now bails on suffix failure before touching the pipe at all, so there's nothing to deadlock on. ## Medium — Failed pushes leaked .staging- dirs Round 8's unique-per-invocation suffix fixed parallel-push interleaving but created a new leak: if THIS push fails before the final `mv` step, the .staging- dir lingers on the PVC. Round 8's `rm -rf $STAGING` at the start only cleans up THIS invocation's path, not the previous-failed one. Repeated failed pushes accumulate unbounded storage on the shared PVC. Fix: append a defensive cleanup pass to the remote script: find $(dirname $DEST) -maxdepth 1 -name "$(basename $DEST).staging-*" \\ -mmin +60 -exec rm -rf {} + 2>/dev/null || true Constraints baked into the pattern: - -mmin +60 (1 hour) is 2x activeDeadlineSeconds — anything older HAS to be from a dead stage Pod, so we can't race with a parallel push's in-progress staging - -name pattern scopes to THIS table's staging siblings only; other tables' .staging-* dirs are none of our business - 2>/dev/null || true — find failures don't fail the whole stream (defensive cleanup is best-effort, the customer's actual push already committed before this runs) - Uses ';' (semicolon) instead of '&&' so it runs even if the main push sequence failed somehow — orphan cleanup should happen regardless Updated TestStreamLayout_RemoteCommand asserts the find pattern is present with the exact -mmin window and table-scoped -name. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): backup-and-rollback dir swap (Bugbot r10) Bugbot's tenth-round single finding: the previous r7 pattern of `rm $DEST && mv $STAGING $DEST` had a small but real failure window — if the rm succeeded and the mv failed (ENOSPC, fs-level rename error, transient kernel weirdness), the customer's previous dataset was already gone and the new data was stranded under .staging- where the ingestor wouldn't see it. Fix: backup-and-rollback. Rename the existing $DEST to a unique $DEST.old- sibling FIRST; only then mv $STAGING into $DEST; on the latter's failure, restore the backup. On success, drop the backup. { [ -e $DEST ] && mv $DEST $DEST.old- || true; } && { mv $STAGING $DEST || { [ -e $DEST.old- ] && mv $DEST.old- $DEST; exit 1; }; } && rm -rf $DEST.old- Properties this gives us: - tar fails: $DEST untouched (transactional from r7) - backup mv fails: $DEST untouched - main mv fails: backup at .old- survives; customer recovers via `kubectl exec -- mv .old- $DEST` - final rm fails: just leaves an .old- cruft, picked up by the r9 find -mmin +60 sweep (extended to also catch .old-* siblings) - first-ever push (no pre-existing $DEST): `[ -e $DEST ] && ...` short-circuits, backup mv silently skipped via `|| true` The .old- suffix reuses the same random as .staging- so two parallel pushes can't collide on the backup name. Both .staging-* AND .old-* now flow through the orphan-cleanup find pattern (with the `-o` alternation), so r9's leak-prevention covers both halves of the swap. Test updates pin: - Backup mv (DEST → .old) appears BEFORE primary mv (.staging → DEST) — rollback contract - Backup and staging suffixes MUST agree (rollback-target correctness) - find pattern includes both -name "...staging-*" and -name "...old-*" Local: vet, test -race -cover, gofmt -s, errcheck — all green. This is round 10. The new-finding rate has been decaying: 5→2→2→2→1. We're approaching the asymptote. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(push): propagate remote-script failures + prefer tar error (Bugbot r11) Two new findings on r10's commit, both real: ## Medium — Remote script masked failures via POSIX `;` + `|| true` The previous shell used `&&-chain ; find ... || true`. POSIX `;` runs the next command regardless of prior exit status, and the trailing `|| true` then forces exit 0. A failed tar or mv earlier in the && chain would set $? to non-zero, then `;` would clobber it back to 0 via the find's `|| true`. Result: a remote push that actually failed would return exit 0 to the exec subprocess, and StreamLayout would happily report "Staged N files" on a catastrophic failure. Fix: rewrite the script using `set -e` + explicit newline- separated statements + an explicit `if ! mv; then rollback; exit 1; fi` for the swap. With set -e: - Any non-guarded non-zero exits the script with that status. - `cmd || true` continues to suppress (find cleanup stays best-effort). - `if ! cmd; then ...; fi` is treated as guarded, so set -e doesn't preempt the explicit rollback handler. Multi-line shell-c args work across busybox sh (alpine 3.20), dash, and bash equally. No portability regression. ## Medium — Stream error masked the upstream tar error After r8 added stream-time MaxSingleFileBytes / MaxTotalBytes rechecks in writeLayoutTar, the LOCAL tar build can legitimately fail mid-stream. When it does, the goroutine's CloseWithError propagates to the pipe reader; exec sees broken-pipe and returns a generic "exec stream against ns/pod failed" error. The previous code checked streamErr FIRST and returned the exec-flavored framing. Customer saw "streaming files failed" instead of the actually-actionable "dataset exceeded v0.1 total cap of 1.00 GiB after streaming ..." diagnostic from the tar side. Fix: swap the order. Check tarErr first — when both are non-nil, the tar side is the upstream cause; the broken-pipe streamErr is downstream noise. streamErr-only (no tarErr) is still the real network/RBAC/remote-tar case and gets surfaced with the exec wording. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Round 11 finding rate: 2 new (1+2+5+2+4+2+2+1+2 → still trending roughly downward, though not monotone). One more clean round and we're at the asymptote. Co-Authored-By: Claude Opus 4.7 (1M context) * chore: doc/cleanup cross-cutting drift from 11 rounds of bugbot fixes Independent review pass over PR #9 (post-bugbot-r11). Five findings, none caught by bugbot — all cross-cutting doc/style: 1. stream.go: dead comment block above remoteCmd composition. The block was a holdover from the && era; it claimed "&& sequencing means the swap only fires if tar succeeded" but we switched to `set -e` in r11. The canonical pipeline doc lives below the composition; collapsed the upstairs block into a single semantic- guarantees summary (HERMETIC/TRANSACTIONAL/PARALLEL-SAFE/EXIT- FAITHFUL) and removed the contradictory && reference. 2. stage.go: StageOptions docstring referenced a non-existent `OrphanLogger` field. Stale from an early design where orphan warnings were emitted via a logger callback rather than the integrated FormatOrphansWarning we ended up with. Corrected to the actual nil-able fields (Progress, Out). 3. orphan.go: OrphanGracePeriod comment claimed "5 minutes is generous: a healthy stage Pod is fully done in ~30 seconds... under a couple minutes at the 1 GiB cap." That directly contradicts pod.go's StagePodActiveDeadline budget (~8.5 min for a 1 GiB transfer at 2 MB/s, plus image pull, plus ready wait). Rewrote to reflect the post-r7 semantic: Running Pods are never flagged regardless of age; the 5-min grace targets the Pending/Failed/Unknown shapes that are genuinely stuck. 4. orphan.go: joinNames was a 9-line wrapper around `strings.Join( names, " ")`. The comment justifying it ("single point of change for tests") doesn't hold — the tests assert on the final output string, not on this helper's surface. Inlined. 5. cli/dataset.go: runDatasetPush docstring still said "the PR-a slim implementation. It performs every pre-flight check... the actual file staging is gated behind a clear 'not yet implemented' error." PR-b (this PR) actually implements the staging — the doc lagged the code. Updated to reflect the complete Phase 3 flow. No behavioral change. Tests + lint green. The doc fixes matter because future readers (humans and the next bugbot pass) get contradictory signals when the comments disagree with the code. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .gitignore | 6 +- cmd/tracebloc/main.go | 26 +- go.mod | 8 +- go.sum | 18 ++ internal/cli/dataset.go | 174 +++++++---- internal/cli/dataset_test.go | 58 ++-- internal/push/orphan.go | 196 ++++++++++++ internal/push/orphan_test.go | 220 ++++++++++++++ internal/push/pod.go | 528 +++++++++++++++++++++++++++++++++ internal/push/pod_test.go | 560 +++++++++++++++++++++++++++++++++++ internal/push/progress.go | 132 +++++++++ internal/push/spec.go | 37 ++- internal/push/spec_test.go | 19 ++ internal/push/stage.go | 153 ++++++++++ internal/push/stage_test.go | 288 ++++++++++++++++++ internal/push/stream.go | 462 +++++++++++++++++++++++++++++ internal/push/stream_test.go | 428 ++++++++++++++++++++++++++ internal/push/walk.go | 84 +++++- internal/push/walk_test.go | 119 ++++++++ 19 files changed, 3419 insertions(+), 97 deletions(-) create mode 100644 internal/push/orphan.go create mode 100644 internal/push/orphan_test.go create mode 100644 internal/push/pod.go create mode 100644 internal/push/pod_test.go create mode 100644 internal/push/progress.go create mode 100644 internal/push/stage.go create mode 100644 internal/push/stage_test.go create mode 100644 internal/push/stream.go create mode 100644 internal/push/stream_test.go diff --git a/.gitignore b/.gitignore index 72a79d0..ac2c77c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ # Build output — populated by `go build ./cmd/tracebloc` at the repo -# root. Release artifacts go through dist/ (see Phase 5 / release -# workflow); local hacking puts a binary here. +# root, or by `go build -o ./bin/tracebloc ./cmd/tracebloc` for the +# convention some folks prefer. Release artifacts go through dist/ +# (see Phase 5 / release workflow); local hacking puts a binary here. /tracebloc +/bin/ # Cross-compiled binaries staged by the release workflow. /dist/ diff --git a/cmd/tracebloc/main.go b/cmd/tracebloc/main.go index 8066305..978cde4 100644 --- a/cmd/tracebloc/main.go +++ b/cmd/tracebloc/main.go @@ -18,8 +18,11 @@ package main import ( + "context" "fmt" "os" + "os/signal" + "syscall" "github.com/tracebloc/cli/internal/cli" ) @@ -35,11 +38,32 @@ var ( ) func main() { + // Wire SIGINT / SIGTERM into the cobra root command's context. + // Long-running operations (e.g. push.Stage in `dataset push`) + // propagate ctx down through every k8s API call, so cancelling + // here triggers ctx.Done() → in-flight HTTP cancels → defers + // fire in normal stack unwind → orphan Pod gets cleaned up. + // + // Without this wire, Ctrl-C goes straight through Go's runtime + // signal handler and exits the process WITHOUT running defers + // — which silently broke the SIGINT-safe cleanup contract in + // push.Stage's docstring. Bugbot flagged this on PR-b. + // + // signal.NotifyContext (Go 1.16+) is the stdlib pattern for + // this. The stop func unregisters the handler so a *second* + // SIGINT does the normal hard-kill — important for the case + // where the cleanup itself hangs (e.g. API server unreachable + // even past the 30s cleanup deadline). The customer's second + // Ctrl-C should always work. + ctx, stop := signal.NotifyContext(context.Background(), + syscall.SIGINT, syscall.SIGTERM) + defer stop() + err := cli.NewRootCmd(cli.BuildInfo{ Version: version, GitSHA: gitSHA, BuildDate: buildDate, - }).Execute() + }).ExecuteContext(ctx) if err == nil { return } diff --git a/go.mod b/go.mod index c792eb3..56b4e7b 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,9 @@ go 1.26.0 require ( github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 + github.com/schollz/progressbar/v3 v3.19.0 github.com/spf13/cobra v1.8.1 + golang.org/x/term v0.39.0 golang.org/x/text v0.33.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.36.1 @@ -28,13 +30,17 @@ require ( github.com/go-openapi/swag v0.23.0 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/moby/spdystream v0.5.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/x448/float16 v0.8.4 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect @@ -42,13 +48,13 @@ require ( golang.org/x/net v0.49.0 // indirect golang.org/x/oauth2 v0.34.0 // indirect golang.org/x/sys v0.40.0 // indirect - golang.org/x/term v0.39.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.1 // indirect k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect diff --git a/go.sum b/go.sum index 9a05a37..a49a7f6 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -27,6 +31,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -42,6 +48,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -53,11 +65,15 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= +github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -114,6 +130,8 @@ k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= +k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go index d740e0f..6034e03 100644 --- a/internal/cli/dataset.go +++ b/internal/cli/dataset.go @@ -15,11 +15,11 @@ import ( ) // newDatasetCmd wires the `tracebloc dataset` subtree. The dominant -// verb is `push`, introduced in Phase 3 (tracebloc/client#151) and -// landing across two PRs: PR-a (this one) implements the -// no-op-safe pre-flight; PR-b adds the actual file streaming via -// an ephemeral Pod + tar-over-exec. Future verbs (`dataset list`, -// `dataset rm`) hang off this parent in v0.2. +// verb is `push`, completed in Phase 3 (tracebloc/client#151) across +// PR-a (pre-flight: spec synth, validation, layout walk, cluster +// discovery) and PR-b (this one: ephemeral stage Pod + tar-over- +// exec stream + progress bar + SIGINT-safe cleanup). Future verbs +// (`dataset list`, `dataset rm`) hang off this parent in v0.2. func newDatasetCmd() *cobra.Command { cmd := &cobra.Command{ Use: "dataset", @@ -27,9 +27,13 @@ func newDatasetCmd() *cobra.Command { Long: `Commands for staging and managing datasets on the cluster's shared PVC. -Today: ` + "`dataset push`" + ` stages a local directory and (in PR-b) -submits an ingestion run. ` + "`tracebloc cluster info`" + ` is the -pre-flight you'd typically run before the first push.`, +Today: ` + "`dataset push`" + ` stages a local directory to the cluster's +shared PVC. Submission to jobs-manager (so the ingestor Job actually +runs) lands in Phase 4 (` + "`tracebloc/client#152`" + `); for now the staged +files are picked up by the existing helm ` + "`tracebloc/ingestor`" + ` flow. + +` + "`tracebloc cluster info`" + ` is the pre-flight you'd typically run +before the first push.`, } cmd.AddCommand(newDatasetPushCmd()) return cmd @@ -37,19 +41,21 @@ pre-flight you'd typically run before the first push.`, // newDatasetPushCmd implements `tracebloc dataset push `. // -// PR-a scope (what this implements today): +// Phase 3 scope (now complete across PR-a + PR-b): // // - Synthesize the ingest spec from flags (`internal/push.SpecArgs.Build`) // - Validate it against the embedded ingest.v1 schema // - Walk the local directory + enforce v0.1 size caps // - Discover cluster, parent release, and shared PVC -// - Print a single-screen "ready to stage" summary -// - --dry-run stops here (and so does this PR — actual staging -// errors out with "coming in PR-b" until #151 PR-b merges) +// - Print a single-screen pre-flight summary +// - Either --dry-run stop, OR create an ephemeral stage Pod +// (alpine 3.20 pinned by digest, PSA-restricted security +// context), tar local files into it via an SPDY exec stream +// with a progress bar, then defer-delete the Pod // -// PR-b will add the ephemeral stage Pod, tar-over-exec stream, -// progress bar, and SIGINT-safe cleanup — slotting into the -// "TODO: PR-b" branch below without touching anything above it. +// Phase 4 (`tracebloc/client#152`) hooks the submit-to-jobs-manager +// step into the bottom of this command, replacing the "manually +// kick off helm ingestor" workaround in the success message. func newDatasetPushCmd() *cobra.Command { var ( // Kubeconfig flags — same conventions as `cluster info`. @@ -71,10 +77,18 @@ func newDatasetPushCmd() *cobra.Command { // Operations flags. dryRun bool - // Ingestor SA name override (only matters once PR-b mints - // a token to talk to the future stage-pod-creation hook). - // Plumbed today so PR-b doesn't have to touch flag wiring. + // Ingestor SA name override. Used as the ServiceAccountName + // of the ephemeral stage Pod, so the Pod inherits whatever + // imagePullSecrets + PSA exemptions the admin already + // configured for that SA. ingestorSAName string + + // Stage Pod image override. Defaults to the digest-pinned + // alpine that ships with the CLI; air-gapped customers + // override this to an image their registry mirror serves. + // Pin by digest in your override too — tag-only references + // drift silently and break "all my pushes worked yesterday." + stagePodImage string ) cmd := &cobra.Command{ @@ -99,12 +113,13 @@ datasets need the v0.2 cloud-source story (S3/GCS/HTTPS sources) — see tracebloc/client#147 non-goals. Exit codes: - 0 staged + (in PR-b) submitted successfully - 2 schema validation failed (synthesized spec rejected) + 0 files staged successfully (Phase 4 will add: submitted + completed) + 2 schema validation failed (synthesized spec rejected) or + v0.1-unsupported category passed 3 local-layout or kubeconfig error - 4 cluster reachable but parent release missing - 6 pre-flight succeeded but the actual stage step isn't - implemented yet (PR-b for #151 will deliver it)`, + 4 cluster reachable but parent release / shared PVC missing + 7 pre-flight succeeded but staging the files failed + (Pod creation, image pull, exec stream, or remote tar error)`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runDatasetPush(cmd.Context(), cmd.OutOrStdout(), cmd.ErrOrStderr(), @@ -116,6 +131,7 @@ Exit codes: Spec: push.SpecArgs{Table: table, Category: category, Intent: intent, LabelColumn: labelColumn}, DryRun: dryRun, IngestorSAName: ingestorSAName, + StagePodImage: stagePodImage, }) }, } @@ -146,6 +162,9 @@ Exit codes: cmd.Flags().StringVar(&ingestorSAName, "ingestor-sa", "", "override the ingestor ServiceAccount name (default: \"ingestor\"); "+ "set this if you customized ingestionAuthz.serviceAccountName in the parent client chart") + cmd.Flags().StringVar(&stagePodImage, "stage-pod-image", "", + "override the ephemeral stage Pod's image (default: digest-pinned alpine 3.20 baked into the CLI). "+ + "Pin by digest in your override too — tag-only refs drift silently.") return cmd } @@ -162,12 +181,13 @@ type runDatasetPushArgs struct { Spec push.SpecArgs DryRun bool IngestorSAName string + StagePodImage string } -// runDatasetPush is the PR-a slim implementation. It performs every -// pre-flight check and prints a summary; the actual file staging -// is gated behind a clear "not yet implemented" error so PR-a -// merging doesn't silently advertise a feature it can't deliver. +// runDatasetPush is the full Phase 3 implementation: pre-flight +// checks, then either --dry-run stop or stage Pod + tar stream + +// cleanup. Phase 4 (#152) will hook submit-to-jobs-manager after +// the staging step. // // Step order is "fail fast, fail local" — every step that doesn't // need the cluster runs before any that does, so a customer with @@ -177,7 +197,7 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush // 1. Validate the table name BEFORE anything else. It's both // the MySQL identifier and the /data/shared/
/ PVC // subdirectory — an unsanitized traversal name (../../etc) - // would escape that subtree once PR-b's stage Pod writes to + // would escape that subtree once the stage Pod writes to // it. The embedded schema only checks minLength on `table`, // so this CLI-side guard is the real fix. SpecArgs.Build() // below calls StagedPrefix, which panics on an unsafe name — @@ -186,7 +206,23 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush return &exitError{code: 2, err: err} } - // 2. Synthesize the spec from flags + validate against schema. + // 2. v0.1 category gate. Runs BEFORE schema validation because + // schema-valid-but-unsupported categories (e.g. + // tabular_classification) would otherwise fail with the + // schema's "missing property 'schema'" message — confusing + // for the customer who has no way to set --schema in v0.1. + // Nonsense categories (typos) also hit this gate; the + // "only image_classification in v0.1" message is more + // actionable than the schema's 11-option enum list anyway. + // Bugbot's review-on-self caught the missing gate on PR-a. + if a.Spec.Category != "" && a.Spec.Category != "image_classification" { + return &exitError{code: 2, err: fmt.Errorf( + "category %q is not supported in v0.1 (only image_classification). "+ + "Other categories are one-PR additions in v0.2 — see "+ + "tracebloc/client#147 non-goals.", a.Spec.Category)} + } + + // 3. Synthesize the spec from flags + validate against schema. // Catches "bad category", "missing intent" etc. BEFORE we // touch the filesystem or the cluster. The error formatter // is the same one ingest validate uses, so a customer who @@ -223,7 +259,7 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush return &exitError{code: 2, err: errors.New("synthesized spec failed schema validation; check the flag values above")} } - // 3. Walk the local directory. Enforces layout + size caps; + // 4. Walk the local directory. Enforces layout + size caps; // customer sees a clear pointer to expected layout if they // pass the wrong directory. layout, err := push.Discover(a.LocalPath) @@ -231,7 +267,7 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush return &exitError{code: 3, err: err} } - // 4. Cluster discovery — same kubeconfig path as `cluster info`. + // 5. Cluster discovery — same kubeconfig path as `cluster info`. // Errors mirror that command's exit-code contract (3 for // kubeconfig, 4 for missing release) so behaviour is // consistent across pre-flight commands. @@ -255,36 +291,74 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush release.IngestorSAName = a.IngestorSAName } - // 5. PVC discovery. New in this PR — confirms the chart's - // shared-data PVC is Bound before we waste time provisioning - // a Pod that can't mount it. + // 6. PVC discovery — confirms the chart's shared-data PVC is + // Bound before we waste time provisioning a Pod that can't + // mount it. pvc, err := cluster.DiscoverSharedPVC(ctx, cs, resolved.Namespace) if err != nil { return &exitError{code: 4, err: err} } - // 6. Print the pre-flight summary. The output is the same in - // dry-run and (eventually) live mode — only the "what - // happens next" line differs. Customers iterating on a - // bad layout see this every attempt, so it's worth keeping - // skimmable: one fact per line, aligned by column. + // 7. Print the pre-flight summary. The output is the same in + // dry-run and live mode — only the "what happens next" line + // differs. Customers iterating on a bad layout see this + // every attempt, so it's worth keeping skimmable: one fact + // per line, aligned by column. printPushPreflight(out, layout, release, pvc, spec, a.DryRun) - // 7. Dry-run stop. Acknowledged success. + // 8. Dry-run stop. Acknowledged success. if a.DryRun { _, _ = fmt.Fprintln(out, "Dry-run complete — no cluster resources were created.") return nil } - // 8. The actual staging branch lands in PR-b. Failing here - // rather than silently returning success means a customer - // who pulled PR-a's binary and ran without --dry-run gets - // a clear "wait for PR-b" signal instead of "0 files - // transferred" confusion in Phase 4. - return &exitError{code: 6, err: errors.New( - "pre-flight succeeded but the actual file staging step isn't " + - "implemented yet — wait for tracebloc/client#151 PR-b. " + - "Re-run with --dry-run to validate without this error.")} + // 9. Stage the files: create ephemeral Pod → wait Ready → tar + // stream → cleanup. The deferred cleanup inside push.Stage + // runs on success and failure (including ctx cancellation + // from a SIGINT handler), so no orphan Pod is left behind. + // + // Exit code 7 ("staging failed") is distinct from the + // pre-flight codes so customers can branch on whether the + // failure was their environment vs the actual data transfer. + progress := push.NewProgress(out, layout.TotalBytes, + fmt.Sprintf("Staging %s", a.Spec.Table)) + // Defer Finish so a failure path that returns BEFORE + // StreamLayout (e.g. CreateStagePod fails on PSA rejection, + // WaitForStagePodReady times out) still clears the TTY + // progress UI. push.StreamLayout's own deferred Finish would + // otherwise be unreachable. Calling Finish twice on the same + // schollz bar is a no-op, so the double-call on the happy + // path is safe. Bugbot flagged on PR-b round 5. + defer progress.Finish() + stageErr := push.Stage(ctx, push.StageOptions{ + Client: cs, + Executor: &push.SPDYExecutor{ + Config: resolved.RestConfig, + Client: cs, + }, + Namespace: resolved.Namespace, + IngestorSAName: release.IngestorSAName, + PVCClaimName: pvc.ClaimName, + PVCMountPath: pvc.MountPath, + Layout: layout, + Table: a.Spec.Table, + StagePodImage: a.StagePodImage, + Progress: progress, + Out: out, + }) + if stageErr != nil { + return &exitError{code: 7, err: stageErr} + } + + // 10. Phase 4 (submit to jobs-manager + watch + summary) hooks + // in here — tracebloc/client#152. For now PR-b leaves the + // dataset staged on the PVC and exits 0, which the customer + // can then chase manually via `helm install ingestor ...` if + // they need the ingestion to actually run today. + _, _ = fmt.Fprintln(out) + _, _ = fmt.Fprintln(out, "Dataset staged. Submission to jobs-manager arrives in Phase 4 (#152);") + _, _ = fmt.Fprintln(out, "in the meantime, the existing helm ingestor flow can pick up the staged files.") + return nil } // printPushPreflight is the customer-facing summary. Mirrors @@ -333,7 +407,7 @@ func printPushPreflight( _, _ = fmt.Fprintln(out) if !dryRun { - _, _ = fmt.Fprintf(out, "Next: stage %d files (%s) → %s (coming in PR-b for #151)\n", + _, _ = fmt.Fprintf(out, "Next: stage %d files (%s) → %s\n", 1+len(layout.Images), push.HumanBytes(layout.TotalBytes), push.StagedPrefix(spec["table"].(string))) _, _ = fmt.Fprintln(out) diff --git a/internal/cli/dataset_test.go b/internal/cli/dataset_test.go index 5a4ef69..96d7ea0 100644 --- a/internal/cli/dataset_test.go +++ b/internal/cli/dataset_test.go @@ -63,39 +63,33 @@ func execDatasetPush(t *testing.T, args []string) (exitCode int, stdout, stderr return ExitCodeFromError(err), so.String(), se.String() } -// TestDatasetPush_BadCategory_ExitsTwo: the synthesized spec gets -// fed through the embedded schema before any cluster work; an -// invalid category surfaces with exit 2 (the CLI-wide "schema -// violation" code). -func TestDatasetPush_BadCategory_ExitsTwo(t *testing.T) { +// TestDatasetPush_UnsupportedCategory_ExitsTwo: v0.1 only supports +// image_classification end-to-end (epic #147 non-goals); the +// CLI-side gate runs before schema validation so a customer who +// passes --category=tabular_classification gets the actionable +// "v0.1 supports only image_classification" message instead of +// the schema's confusing "missing property 'schema'" (which the +// customer has no way to supply in v0.1). Bugbot review-on-self +// caught the missing gate on PR-a; landing it here. +func TestDatasetPush_UnsupportedCategory_ExitsTwo(t *testing.T) { root := imgcLayout(t) - code, _, stderr := execDatasetPush(t, []string{ - root, - "--table=t1", - "--category=definitely-not-a-category", - "--intent=train", - "--label-column=label", - }) - if code != 2 { - t.Fatalf("expected exit 2 for schema failure, got %d", code) - } - // The same FormatErrors output ingest validate uses, routed - // to stderr so downstream pipes of stdout aren't polluted. - // Checking for "category" + "image_classification" surfacing - // means we're getting the JSON-pointer-anchored diagnostic + - // the enum-list expansion, not a generic message. - // Check three things: (1) the JSON-pointer-style "category" - // anchor surfaces, (2) the enum-list expansion happened (so - // "image_classification" appears as a valid option), (3) our - // "synthesized spec" framing is on the right output stream. - // The wording "synthesized spec" is intentionally distinct - // from ingest validate's "schema validation failed" — it tells - // the customer the CLI synthesized the YAML; they didn't - // author it. - for _, want := range []string{"category", "image_classification", "synthesized spec failed schema validation"} { - if !strings.Contains(stderr, want) { - t.Errorf("expected stderr to mention %q, got:\n%s", want, stderr) - } + for _, badCategory := range []string{ + "tabular_classification", // schema-valid but v0.1-unsupported + "object_detection", // ditto + "definitely-not-a-category", // nonsense; gate catches this too + } { + t.Run(badCategory, func(t *testing.T) { + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=" + badCategory, + "--intent=train", + "--label-column=label", + }) + if code != 2 { + t.Fatalf("expected exit 2 for unsupported category %q, got %d", badCategory, code) + } + }) } } diff --git a/internal/push/orphan.go b/internal/push/orphan.go new file mode 100644 index 0000000..65ab4c0 --- /dev/null +++ b/internal/push/orphan.go @@ -0,0 +1,196 @@ +package push + +import ( + "context" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// OrphanGracePeriod is how long a non-Running stage Pod has to live +// before we flag it as an orphan. Running Pods are NEVER flagged +// regardless of age (see FindOrphanStagePods's Phase=Running skip, +// Bugbot r7) — Pods past pod.go's StagePodActiveDeadline of 30 min +// are still legitimate "in-progress active push" candidates, and +// any per-age cap would produce false positives for near-cap pushes. +// +// 5 minutes targets the genuinely-stuck shapes: +// +// - Phase=Pending past 5 min: image pull failure that didn't +// resolve, scheduling stuck on missing nodes, PSA rejection +// that took kubelet a while to surface +// - Phase=Failed past 5 min: container crashed at startup or +// during stream, CLI didn't get a chance to clean up +// - Phase=Unknown past 5 min: node went away (network partition, +// kubelet crash) and the Pod metadata is stranded +// +// All three shapes are genuinely orphan — no still-running push +// will ever progress past Pending/Failed/Unknown into Ready, so +// 5 min is comfortable for "stuck enough to surface as a warning." +const OrphanGracePeriod = 5 * time.Minute + +// Orphan describes a stage Pod found by the orphan scan. Carries +// enough metadata for the warning surface to render an actionable +// "delete with: kubectl delete pod X -n Y" hint. +type Orphan struct { + // Name is the metadata.name. + Name string + + // Namespace is the resolved namespace (not the chart's + // release-name, the actual API namespace). + Namespace string + + // Table is the destination table the orphan was staging for — + // read from the StagePodTableLabel set in BuildStagePodSpec. + // May be empty for very old orphans (pre-label-convention), + // but every Pod the CLI ever created carries this label. + Table string + + // Age is how long the Pod has existed (now - creationTimestamp). + // Surfaces in the warning so customers can spot stale Pods at + // a glance ("3 hours old" → almost certainly safe to delete; + // "6 minutes old" → maybe wait, might be from a slow parallel + // push). + Age time.Duration +} + +// FindOrphanStagePods lists every stage Pod the CLI has ever +// created in the namespace and returns the ones past OrphanGrace +// Period. Pods younger than that are silently filtered — they +// might be the CURRENT invocation's own Pod (the orphan scan runs +// before CreateStagePod, but in a tight enough loop the previous +// invocation's just-created Pod could plausibly still be < grace). +// +// Returns nil + nil if there are no orphans. Doesn't return an +// error for the API call failing — orphan scanning is best-effort +// (an RBAC denial on `list pods` shouldn't block the push) — the +// caller logs the scan failure but proceeds. +// +// Hence the signature returns ([]Orphan, error): the slice is +// empty on success-with-no-orphans, the error is non-nil only on +// API failures the caller might want to surface for debugging. +func FindOrphanStagePods(ctx context.Context, cs kubernetes.Interface, namespace string) ([]Orphan, error) { + selector := fmt.Sprintf("%s=%s,%s=%s", + StagePodManagedByLabel, StagePodManagedByValue, + StagePodComponentLabel, StagePodComponentValue) + + pods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + // Don't wrap — the caller (cli.runDatasetPush) will either + // log this for diagnostic value and proceed, or ignore it + // entirely. Either way, the orphan scan isn't on the + // critical path. + return nil, fmt.Errorf("listing stage Pods in %s: %w", namespace, err) + } + + now := time.Now() + var orphans []Orphan + for i := range pods.Items { + p := &pods.Items[i] + + // Phase=Running means an active push (this workstation or + // another's) is still doing real work. activeDeadlineSeconds + // is its safety net at the cluster level. Flagging it as + // orphan would produce false positives for legitimate + // slow/near-cap pushes — pod.go budgets ~8.5 minutes for + // the 1 GiB-cap transfer alone, which exceeds the 5-min + // grace below. Bugbot flagged the false-positive risk on + // PR-b round 7. + // + // Pods in non-Running phases (Pending/Failed/Unknown) past + // the grace period are still flagged — those are the + // genuine orphan shapes (stuck on image pull, crashed + // during stream, network-partitioned). + if p.Status.Phase == corev1.PodRunning { + continue + } + + age := now.Sub(p.CreationTimestamp.Time) + if age < OrphanGracePeriod { + continue + } + orphans = append(orphans, Orphan{ + Name: p.Name, + Namespace: p.Namespace, + Table: p.Labels[StagePodTableLabel], + Age: age, + }) + } + return orphans, nil +} + +// FormatOrphansWarning renders the orphan list as a multi-line +// customer-facing warning. Returns empty string when orphans is +// empty — caller can blind-print without an "if len > 0" check. +// +// The "delete with" hint is the actionable part. v0.1 doesn't +// auto-delete because: +// +// 1. We can't tell from labels alone whether the Pod is stuck +// mid-transfer for a still-live parallel push from another +// workstation, vs truly orphaned from a crash. +// 2. Deleting someone else's in-progress push silently is bad +// UX. A warn-only approach respects the principle of least +// surprise. +// +// v0.2 can layer on `--cleanup-orphans` for ops folks who want it +// automated. +func FormatOrphansWarning(orphans []Orphan) string { + if len(orphans) == 0 { + return "" + } + var s string + s += fmt.Sprintf("WARNING: %d orphan stage Pod%s detected in this namespace — likely "+ + "leftover from a previously crashed `dataset push`:\n", + len(orphans), pluralS(len(orphans))) + names := make([]string, 0, len(orphans)) + for _, o := range orphans { + tableHint := "" + if o.Table != "" { + tableHint = fmt.Sprintf(" (table: %s)", o.Table) + } + s += fmt.Sprintf(" - %s, age %s%s\n", o.Name, humanDuration(o.Age), tableHint) + names = append(names, o.Name) + } + // Use SPECIFIC Pod names in the delete command, not the + // label selector. The selector would match every stage Pod + // in the namespace, including legitimate running ones from + // parallel pushes (this workstation or another's) — copy- + // pasting a label-based delete could silently kill someone + // else's in-progress push. Bugbot flagged the over-broad + // delete on PR-b round 7. + s += "Delete with: kubectl delete pod -n " + orphans[0].Namespace + " " + + strings.Join(names, " ") + "\n" + return s +} + +// pluralS returns "s" for n != 1, else "". Local helper to keep +// internal/push self-contained (the cli package has its own copy +// for the same reason — see cli/ingest.go's plural()). +func pluralS(n int) string { + if n == 1 { + return "" + } + return "s" +} + +// humanDuration is a coarse "X hours" / "X minutes" / "X seconds" +// formatter for orphan warnings. time.Duration.String() returns +// "2h13m45s" which is technically more precise but harder to read +// in a one-line warning. +func humanDuration(d time.Duration) string { + switch { + case d >= time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + case d >= time.Minute: + return fmt.Sprintf("%dm", int(d.Minutes())) + default: + return fmt.Sprintf("%ds", int(d.Seconds())) + } +} diff --git a/internal/push/orphan_test.go b/internal/push/orphan_test.go new file mode 100644 index 0000000..271f086 --- /dev/null +++ b/internal/push/orphan_test.go @@ -0,0 +1,220 @@ +package push + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// stagePodWithAge constructs a stage-labeled Pod whose +// creationTimestamp is `age` ago. Used to seed the fake clientset +// for each orphan-test variant. Default phase is "" (unscheduled) — +// helpers below construct Running/etc explicitly. +func stagePodWithAge(name, table string, age time.Duration) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "tracebloc", + CreationTimestamp: metav1.NewTime(time.Now().Add(-age)), + Labels: map[string]string{ + StagePodManagedByLabel: StagePodManagedByValue, + StagePodComponentLabel: StagePodComponentValue, + StagePodTableLabel: table, + }, + }, + } +} + +// runningStagePodWithAge constructs a stage-labeled Pod in +// Phase=Running. Used to assert the Bugbot-r7 false-positive fix: +// a Running Pod is presumed to be an active push (this or +// another workstation's) and must NOT be flagged as an orphan +// regardless of age. +func runningStagePodWithAge(name, table string, age time.Duration) *corev1.Pod { + p := stagePodWithAge(name, table, age) + p.Status.Phase = corev1.PodRunning + return p +} + +// TestFindOrphanStagePods_NoOrphans: empty cluster returns empty, +// no error. +func TestFindOrphanStagePods_NoOrphans(t *testing.T) { + cs := fake.NewClientset() + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("len(orphans) = %d, want 0", len(got)) + } +} + +// TestFindOrphanStagePods_RecentPodFiltered: a Pod just created +// (well within OrphanGracePeriod) is silently filtered out — it +// might be the current invocation's own Pod from a parallel +// workstation push. +func TestFindOrphanStagePods_RecentPodFiltered(t *testing.T) { + cs := fake.NewClientset(stagePodWithAge("fresh", "t1", 30*time.Second)) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("recent Pod surfaced as orphan; got %d, want 0", len(got)) + } +} + +// TestFindOrphanStagePods_StaleSurfaces: a Pod past the grace +// period surfaces with its metadata intact (name, table, age). +func TestFindOrphanStagePods_StaleSurfaces(t *testing.T) { + cs := fake.NewClientset(stagePodWithAge("stale-cats", "cats_dogs", 30*time.Minute)) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 1 { + t.Fatalf("len(orphans) = %d, want 1", len(got)) + } + o := got[0] + if o.Name != "stale-cats" { + t.Errorf("Name = %q, want stale-cats", o.Name) + } + if o.Table != "cats_dogs" { + t.Errorf("Table = %q, want cats_dogs", o.Table) + } + if o.Age < 29*time.Minute { + t.Errorf("Age = %v, want at least 29m", o.Age) + } +} + +// TestFindOrphanStagePods_IgnoresNonStagePods: the chart's own +// Pods (managed-by=Helm) and arbitrary user Pods must NOT show +// up — the label selector is the safety boundary. +func TestFindOrphanStagePods_IgnoresNonStagePods(t *testing.T) { + chartPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "jobs-manager-abc", + Namespace: "tracebloc", + CreationTimestamp: metav1.NewTime(time.Now().Add(-1 * time.Hour)), + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "client", + }, + }, + } + customerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "their-app", + Namespace: "tracebloc", + CreationTimestamp: metav1.NewTime(time.Now().Add(-1 * time.Hour)), + }, + } + cs := fake.NewClientset(chartPod, customerPod) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("non-stage Pods surfaced as orphans; got %d, want 0; names=%v", len(got), got) + } +} + +// TestFormatOrphansWarning_Empty returns the empty string so the +// caller can blind-print without conditionals. +func TestFormatOrphansWarning_Empty(t *testing.T) { + if got := FormatOrphansWarning(nil); got != "" { + t.Errorf("FormatOrphansWarning(nil) = %q, want empty", got) + } +} + +// TestFormatOrphansWarning_ActionableHint: the warning must contain +// (a) the count, (b) the per-pod name + age + table, (c) a +// kubectl delete command using SPECIFIC POD NAMES (not the label +// selector — Bugbot r7 flagged that the label selector would +// match every stage Pod including parallel-push's still-running +// ones). +func TestFormatOrphansWarning_ActionableHint(t *testing.T) { + orphans := []Orphan{ + {Name: "stage-a", Namespace: "tracebloc", Table: "cats_dogs", Age: 30 * time.Minute}, + {Name: "stage-b", Namespace: "tracebloc", Table: "xrays", Age: 2 * time.Hour}, + } + got := FormatOrphansWarning(orphans) + for _, want := range []string{ + "2 orphan stage Pods", + "stage-a", "cats_dogs", "30m", + "stage-b", "xrays", "2h", + // Targeted delete — specific names, not the label + // selector. The label-selector version would nuke + // running pods from parallel pushes. + "kubectl delete pod -n tracebloc stage-a stage-b", + } { + if !strings.Contains(got, want) { + t.Errorf("warning missing %q in:\n%s", want, got) + } + } + // Regression check: the label selector MUST NOT appear in + // the delete command. If a refactor reintroduces it, + // customers could accidentally nuke their parallel-push Pod. + if strings.Contains(got, "kubectl delete pod -n tracebloc -l ") { + t.Errorf("warning uses label-selector delete (would catch parallel pushes):\n%s", got) + } +} + +// TestFindOrphanStagePods_SkipsRunningPods is the Bugbot-r7 +// false-positive fix: a Pod in Phase=Running is presumed to be an +// active push (this workstation or another's still doing work), +// and must NOT be flagged as orphan regardless of age. Near-cap +// 1 GiB pushes legitimately take >5 min (pod.go budgets ~8.5 min +// for the stream alone), so without this guard the next concurrent +// push would warn about Pods that are still actively transferring. +func TestFindOrphanStagePods_SkipsRunningPods(t *testing.T) { + // 30-minute-old Running Pod — well past OrphanGracePeriod, + // but Running means it's an active push. + cs := fake.NewClientset(runningStagePodWithAge("active-slow-push", + "big_table", 30*time.Minute)) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("Running Pod flagged as orphan; got %d, want 0", len(got)) + } +} + +// TestFindOrphanStagePods_FlagsNonRunningPastGrace: complement to +// the Running-skip — a Failed or Pending Pod past the grace period +// IS still an orphan (crashed-at-startup case, stuck-on-image-pull +// case). The Running-skip should narrow the false positives, not +// eliminate the warning entirely. +func TestFindOrphanStagePods_FlagsNonRunningPastGrace(t *testing.T) { + failedPod := stagePodWithAge("crashed", "t1", 30*time.Minute) + failedPod.Status.Phase = corev1.PodFailed + cs := fake.NewClientset(failedPod) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 1 { + t.Errorf("Failed Pod past grace not flagged; got %d, want 1", len(got)) + } +} + +// TestFormatOrphansWarning_NoTableLabel: very old orphans from +// before we added StagePodTableLabel might not have it. The +// formatter must not crash or render "(table: )". +func TestFormatOrphansWarning_NoTableLabel(t *testing.T) { + got := FormatOrphansWarning([]Orphan{ + {Name: "old-orphan", Namespace: "tracebloc", Age: 1 * time.Hour}, + }) + if strings.Contains(got, "(table: )") { + t.Errorf("warning rendered empty table parenthetical:\n%s", got) + } + if !strings.Contains(got, "old-orphan") { + t.Errorf("warning missing Pod name:\n%s", got) + } +} diff --git a/internal/push/pod.go b/internal/push/pod.go new file mode 100644 index 0000000..fb28e6d --- /dev/null +++ b/internal/push/pod.go @@ -0,0 +1,528 @@ +package push + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +// DefaultStagePodImage is the alpine image the ephemeral stage Pod +// runs. Pinned by digest at CLI build time so a customer pulling +// v0.1.x at any future date gets bit-for-bit identical behavior — +// no "alpine just shipped a glibc-on-musl regression and now my +// stage Pod crashloops" surprises. +// +// alpine:3.20 was the current 3.x stable when Phase 3 PR-b landed +// (May 2026). The image is tiny (~8 MiB), has tar/sh/busybox built +// in, and is one of the most-mirrored images on the planet — air- +// gapped customers can almost always pull it from their internal +// mirror without extra config. +// +// Override via `tracebloc dataset push --stage-pod-image=...` for: +// +// - Customers on registries that don't proxy docker.io +// - Customers running a curated base image with extra audit +// instrumentation +// - Custom forks that need a specific busybox build +// +// Bumping the pinned digest is a v0.2 task; track it alongside +// the kube-deps refresh. +const DefaultStagePodImage = "alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc" + +// StagePodLabel{Key,Value} mark every Pod the CLI creates so the +// orphan-scan logic (orphan.go) can find leftover Pods from a +// previously crashed `dataset push` invocation. Using the +// kubernetes.io/managed-by label means the chart's own resources +// (managed-by=Helm) don't get caught in our scan. +const ( + StagePodManagedByLabel = "app.kubernetes.io/managed-by" + StagePodManagedByValue = "tracebloc-cli" + + // StagePodComponentLabel narrows the orphan scan to just stage + // Pods (not, e.g., a future `tracebloc cluster doctor` Pod). + StagePodComponentLabel = "tracebloc.io/component" + StagePodComponentValue = "stage-pod" + + // StagePodTableLabel records which table the Pod was staging + // for. Useful in the orphan-warning output so the customer + // knows which `dataset push` invocation it came from. + StagePodTableLabel = "tracebloc.io/table" +) + +// StagePodActiveDeadline is the in-cluster self-kill timer. The CLI +// also deletes the Pod via defer + SIGINT handler, but those only +// fire if the CLI process is still alive — a hard kill (`kill -9`, +// OS crash, network partition between laptop and cluster) leaves +// the Pod stranded. activeDeadlineSeconds means the kubelet kills +// it even if the CLI never gets the chance. +// +// 30 minutes covers the full lifecycle, not just the stream: +// +// ~60s image pull on a fresh node + scheduler back-pressure +// ~60s WaitForStagePodReady ceiling (StagePodReadyTimeout) +// ~8.5m 1 GiB transfer at a conservative 2 MB/s +// + comfortable margin for variance +// +// activeDeadlineSeconds starts the clock at Pod CREATION (not at +// streaming start), so we have to budget for the pre-stream +// readiness portion too — the earlier 10-minute value cut it too +// fine for near-cap customers on slow uplinks (kubelet would +// terminate mid-transfer). Bugbot flagged the squeeze as High on +// PR-b round 6. +// +// 30 min is generous but cheap — an idle alpine Pod with `sleep` +// consumes ~5 MiB RAM and zero CPU on the cluster. v0.2 should +// make this configurable per push. +const StagePodActiveDeadline = 1800 + +// StagePodReadyTimeout is how long we wait for the Pod to become +// Running + Ready after CREATE. Most clusters spawn an alpine Pod +// in 5-15 seconds; 60 seconds covers image-pull from a slow mirror +// + scheduler back-pressure on busy clusters. Beyond that, something +// is wrong (no image-pull access, no schedulable node, PSP/PSA +// rejection) and the customer wants the diagnostic, not a longer +// wait. +const StagePodReadyTimeout = 60 * time.Second + +// PodSpecOptions controls the ephemeral stage Pod construction. +// Fields are intentionally narrow — every knob is one a customer +// could plausibly need to turn for an air-gapped or hardened- +// security setup. Adding fields should require a real use case. +type PodSpecOptions struct { + // Namespace is where the Pod gets created — always the + // discovered parent-release namespace. + Namespace string + + // PVCClaimName is the shared PVC to mount at /data/shared. + // Discovered by cluster.DiscoverSharedPVC (always "client-pvc" + // today, but routed through a field so a future per-customer + // override doesn't require touching this signature). + PVCClaimName string + + // PVCMountPath is where to mount the PVC inside the Pod — + // "/data/shared" by chart convention. + PVCMountPath string + + // Table is the destination table name. Used to compose the Pod + // name and the on-PVC subdirectory. MUST have already passed + // ValidateTableName; pod-name composition relies on the same + // character-class restrictions. + Table string + + // Image overrides DefaultStagePodImage. Empty = use default. + Image string + + // ServiceAccountName is the SA the Pod runs as. Phase 2's + // discovery surfaces this as `ingestor` (the chart default) or + // whatever the customer's `--ingestor-sa` flag overrides to. + // Using the chart's existing SA means the Pod inherits any + // imagePullSecrets and PSA exemptions the admin already + // configured for it. + ServiceAccountName string +} + +// BuildStagePodSpec produces the corev1.Pod for an ephemeral stage +// Pod, fully parametrized but with no cluster side-effects. +// Separated from CreateStagePod so unit tests can assert the spec +// shape without needing a fake clientset for every assertion. +// +// Returns an error only when crypto/rand fails — which is rare but +// possible on systems with exhausted entropy. Earlier versions +// swallowed that error and produced a Pod name ending with a bare +// trailing hyphen (DNS-1123 violation → opaque API server +// rejection). Bugbot flagged that as Low on PR-b; surfacing the +// error here turns "weird API error message" into "clear local +// diagnostic at the call site." +// +// Security context follows the Kubernetes Pod Security Standards +// "restricted" profile — the strictest preset, accepted on every +// PSA-enabled namespace including the chart's recommended config. +// This is intentional: the stage Pod runs in the customer's +// namespace and writes to their PVC, so being a model citizen for +// PSA defaults reduces "the Pod won't even start on my cluster" +// surface area. +func BuildStagePodSpec(opts PodSpecOptions) (*corev1.Pod, error) { + image := opts.Image + if image == "" { + image = DefaultStagePodImage + } + + suffix, err := randomSuffix(4) // 4 bytes → 8 hex chars + if err != nil { + return nil, fmt.Errorf("generating Pod-name random suffix: %w", err) + } + // Transform the table name into a DNS-1123 subdomain-safe + // segment for the Pod name. ValidateTableName accepts + // [A-Za-z0-9_]+ (MySQL identifier rules), but Kubernetes Pod + // names follow DNS-1123 — lowercase + alphanumeric + hyphen + // only, must start/end with alphanumeric. Without this + // transform, the dominant canonical example (cats_dogs_train, + // snake_case throughout the tracebloc docs) would fail Pod + // creation post-pre-flight, which is a worst-of-both-worlds + // UX (the pre-flight summary says "we're good!" then the + // create fails). Bugbot flagged the gap as High on PR-b. + // + // The original (un-transformed) table name is preserved + // verbatim in the tracebloc.io/table label below, so orphan + // warnings still surface the customer-facing identifier. + podName := fmt.Sprintf("tracebloc-stage-%s-%s", + dns1123SafeTableSegment(opts.Table), suffix) + + // Pod-level security context: runAsNonRoot is the only field + // PSA's restricted profile *requires* at the Pod level (the + // rest are container-level). Setting it here means every + // future container (if we ever add one) inherits the + // non-root constraint. + runAsNonRoot := true + runAsUser := int64(65532) // distroless's "nonroot" UID; works on any cluster + + // Container-level security context: every PSA-restricted + // requirement. Reads top-to-bottom: capabilities dropped, + // read-only root FS, no privilege escalation, no privileged, + // seccomp default. None of these prevent tar from working — + // tar reads stdin + writes to the PVC mount (which is RW), + // and doesn't need any caps to run as a non-root user. + allowPrivEsc := false + privileged := false + readOnlyRootFS := true + + activeDeadline := int64(StagePodActiveDeadline) + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: opts.Namespace, + Labels: map[string]string{ + StagePodManagedByLabel: StagePodManagedByValue, + StagePodComponentLabel: StagePodComponentValue, + StagePodTableLabel: opts.Table, + }, + Annotations: map[string]string{ + // Annotations are searchable via `kubectl describe` + // but don't constrain scheduling — perfect for + // breadcrumbs that help post-mortem an orphan. + "tracebloc.io/created-at": time.Now().UTC().Format(time.RFC3339), + "tracebloc.io/created-by": "tracebloc-cli", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: opts.ServiceAccountName, + RestartPolicy: corev1.RestartPolicyNever, + + // 10-min in-cluster self-kill — see comment on the const. + ActiveDeadlineSeconds: &activeDeadline, + + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: &runAsNonRoot, + RunAsUser: &runAsUser, + FSGroup: &runAsUser, // PVC writes need this group + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + + Containers: []corev1.Container{{ + Name: "stage", + Image: image, + // `sleep` keeps the Pod alive long enough for the + // CLI to open an exec stream and tar files in. + // activeDeadlineSeconds caps the worst case; the + // CLI deletes the Pod the moment the stream + // finishes (or fails). + // + // Why not the busybox `sleep infinity` idiom: it + // causes some PSA configurations to flag the Pod + // because the container holds the SIGTERM until + // activeDeadlineSeconds fires (sleep traps signals + // imperfectly). A finite sleep matched to the + // deadline is gentler. + Command: []string{"/bin/sleep", fmt.Sprintf("%d", StagePodActiveDeadline)}, + + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivEsc, + Privileged: &privileged, + ReadOnlyRootFilesystem: &readOnlyRootFS, + RunAsNonRoot: &runAsNonRoot, + RunAsUser: &runAsUser, + Capabilities: &corev1.Capabilities{ + // PSA restricted requires ALL capabilities + // dropped, then optionally add back + // NET_BIND_SERVICE. tar doesn't need it. + Drop: []corev1.Capability{"ALL"}, + }, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + + VolumeMounts: []corev1.VolumeMount{{ + Name: "shared", + MountPath: opts.PVCMountPath, + }, { + // tar needs a writable working dir for its + // temporary state; with ReadOnlyRootFilesystem + // it can't use /tmp on the root FS. An + // emptyDir at /tmp is the standard pattern. + Name: "tmp", + MountPath: "/tmp", + }}, + + // Conservative resource requests: tar of typical + // image_classification data uses <100 MiB RAM and + // negligible CPU. Setting requests lets the + // scheduler place us; setting limits keeps a + // runaway tar (e.g. corrupted archive triggering + // a busy loop) from impacting the cluster. + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + }, + }}, + + Volumes: []corev1.Volume{{ + Name: "shared", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: opts.PVCClaimName, + }, + }, + }, { + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }}, + }, + }, nil +} + +// CreateStagePod creates the Pod in the cluster and returns the +// metadata.name the API server assigned. The returned name is what +// callers pass to WaitForStagePodReady + DeleteStagePod. +// +// Take-name-from-API-response is deliberate: even though +// BuildStagePodSpec pre-computes a name, the API server can in +// principle rewrite it (e.g. via a mutating admission webhook +// renaming with a prefix). Reading back from the response is the +// safe contract. +func CreateStagePod(ctx context.Context, cs kubernetes.Interface, opts PodSpecOptions) (string, error) { + pod, err := BuildStagePodSpec(opts) + if err != nil { + return "", err + } + created, err := cs.CoreV1().Pods(opts.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + // Most-common failure path: PSA rejects the Pod because + // the customer's namespace policy is stricter than even + // "restricted." We surface the raw API error so the customer + // sees the PSA violation list verbatim — that's the most + // actionable thing. + return "", fmt.Errorf("creating stage Pod in namespace %q: %w", opts.Namespace, err) + } + return created.Name, nil +} + +// WaitForStagePodReady polls until the Pod's status reports +// Ready=True (the canonical "containers started and not yet +// terminated" signal). Times out per StagePodReadyTimeout with a +// diagnostic that tries to help — image-pull failures and +// scheduler back-pressure are the dominant slow paths and have +// distinct symptom strings. +// +// Returns nil on Ready, the pod object on success so callers don't +// have to re-Get for it. +func WaitForStagePodReady(ctx context.Context, cs kubernetes.Interface, namespace, podName string) (*corev1.Pod, error) { + var lastObserved *corev1.Pod + + // Poll every 1s — image-pull failures surface in seconds, and + // the Ready transition itself is instant once kubelet has + // pulled+started. Tighter polling burns API server CPU for no + // benefit; looser polling delays the user. + err := wait.PollUntilContextTimeout(ctx, 1*time.Second, StagePodReadyTimeout, true, + func(ctx context.Context) (bool, error) { + p, err := cs.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + // Distinguish terminal vs transient. Terminal + // errors (Pod deleted out-of-band, RBAC revoked + // to read Pods) MUST short-circuit the poll — + // otherwise the customer waits the full 60s + // timeout for a condition that won't change. + // Bugbot flagged the prior "everything transient" + // version as Medium on PR-b. + if apierrors.IsNotFound(err) || apierrors.IsForbidden(err) { + return false, err + } + // Transient (network blip, brief API unavail). + // Keep polling; last-observed context survives. + return false, nil + } + lastObserved = p + // Positive terminal: Ready=True. + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return true, nil + } + } + // Negative terminal: Phase=Failed (container crashed + // at startup — PSA rejection, image crashloop, OOM) + // or Phase=Succeeded (sleep exited unexpectedly — the + // stage container shouldn't terminate by itself before + // we exec into it, so this is also a failure mode). + // Without these checks the poll waits the full 60s + // for a Pod that will never become Ready. Bugbot + // flagged on PR-b round 4 as Medium; this is the + // counterpart to the NotFound/Forbidden short-circuit + // landed in the previous commit. + if p.Status.Phase == corev1.PodFailed || p.Status.Phase == corev1.PodSucceeded { + return false, fmt.Errorf( + "stage Pod %s/%s terminated in phase %q before becoming Ready%s", + namespace, podName, p.Status.Phase, podReadyTimeoutHint(p)) + } + return false, nil + }) + + if err == nil { + return lastObserved, nil + } + + // Differentiate "actual timeout expired" from "poll terminated + // early" (NotFound, Forbidden, Failed phase, ctx-canceled). + // The earlier blanket "did not become Ready within 60s" + // wording was misleading — Bugbot flagged on PR-b round 7. + hint := podReadyTimeoutHint(lastObserved) + if errors.Is(err, context.DeadlineExceeded) { + return nil, fmt.Errorf( + "stage Pod %s/%s did not become Ready within %s%s", + namespace, podName, StagePodReadyTimeout, hint) + } + // Early-exit error (terminal API error, terminal phase, + // SIGINT). Surface it as-is so the customer sees the actual + // cause without the wrong "ran out the timer" framing. + return nil, fmt.Errorf( + "stage Pod %s/%s did not reach Ready state: %w%s", + namespace, podName, err, hint) +} + +// podReadyTimeoutHint extracts the most useful diagnostic from a +// last-observed Pod status. Designed to match against the two +// dominant slow-path scenarios: +// +// 1. Image pull is slow or failed (ImagePullBackOff, +// ErrImagePull) — surface the registry message. +// 2. Pod is unschedulable (PodScheduled=False with reason) — +// surface the scheduler's message. +func podReadyTimeoutHint(p *corev1.Pod) string { + if p == nil { + return " (no Pod status observed — check API server connectivity)" + } + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" { + return fmt.Sprintf(" (last container state: %s — %s)", + cs.State.Waiting.Reason, cs.State.Waiting.Message) + } + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodScheduled && c.Status == corev1.ConditionFalse { + return fmt.Sprintf(" (scheduling: %s — %s)", c.Reason, c.Message) + } + } + return fmt.Sprintf(" (Pod phase: %s)", p.Status.Phase) +} + +// DeleteStagePod removes the Pod. Called via defer in the orchestrator +// so it runs on both success and failure (including SIGINT, which +// is wired up to cancel the parent context and let the defer fire +// in normal stack unwind). +// +// Uses background propagation + a tiny grace period because we +// don't care about giving sleep a graceful shutdown — we just want +// the Pod gone so the next push doesn't see an orphan. +func DeleteStagePod(ctx context.Context, cs kubernetes.Interface, namespace, podName string) error { + gracePeriod := int64(0) + propagation := metav1.DeletePropagationBackground + err := cs.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + PropagationPolicy: &propagation, + }) + if err != nil && !apierrors.IsNotFound(err) { + // Not-found is fine: the Pod might have already self-killed + // via activeDeadlineSeconds, or a parallel `kubectl delete` + // got there first. Either way, our goal (no orphan) is + // achieved. Returning the error here would mask the + // upstream cause of failure that triggered the defer. + return fmt.Errorf("deleting stage Pod %s/%s: %w", namespace, podName, err) + } + return nil +} + +// dns1123SafeTableSegment transforms a ValidateTableName-passed +// table name into a Kubernetes-Pod-name-compatible path segment. +// The Pod's full name is then `tracebloc-stage--<8hex>`, +// which must satisfy DNS-1123 subdomain rules: +// +// [a-z0-9]([-a-z0-9]*[a-z0-9])? +// +// (lowercase + digit + hyphen, start/end alphanumeric). +// +// Our input alphabet is [A-Za-z0-9_], so the transform is: +// +// 1. Lowercase (DNS-1123 forbids uppercase) +// 2. Replace '_' with '-' (DNS-1123 forbids underscore) +// 3. Strip leading/trailing hyphens (a name like "_leading" +// would otherwise become "-leading" → tracebloc-stage--leading +// which is OK in the middle but ugly) +// 4. Cap length at 30 chars — Pod names are bounded at 63 total, +// and "tracebloc-stage-" + 8-hex-suffix already consumes ~25 +// of those, leaving ~38 for the segment. 30 gives margin. +// 5. Fallback to "tbl" if the transform leaves an empty string +// (the pathological "_"-only-name case). +func dns1123SafeTableSegment(table string) string { + s := strings.ToLower(strings.ReplaceAll(table, "_", "-")) + s = strings.Trim(s, "-") + if len(s) > 30 { + s = s[:30] + // Truncation could leave a trailing hyphen — re-trim. + s = strings.TrimRight(s, "-") + } + if s == "" { + // Pathological all-underscore name. The label still + // carries the original, so customers can still trace + // orphan-Pod warnings back to their push. + s = "tbl" + } + return s +} + +// randomSuffix returns a hex string of length 2*n. Used to make +// stage Pod names unique across parallel `dataset push` invocations +// for the same table — without this, two CLI runs would race on the +// same Pod name and one would fail the Create with AlreadyExists. +// +// Cryptographic randomness is overkill for collision avoidance +// (8 hex chars = 32 bits of entropy is way more than needed) but +// crypto/rand is the simpler import compared to math/rand which +// needs explicit seeding. +func randomSuffix(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} diff --git a/internal/push/pod_test.go b/internal/push/pod_test.go new file mode 100644 index 0000000..6e58715 --- /dev/null +++ b/internal/push/pod_test.go @@ -0,0 +1,560 @@ +package push + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// stageOpts is the minimal PodSpecOptions for happy-path tests — +// individual cases mutate one field to exercise their specific +// branch. +func stageOpts() PodSpecOptions { + return PodSpecOptions{ + Namespace: "tracebloc", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Table: "cats_dogs", + ServiceAccountName: "ingestor", + } +} + +// mustBuildStagePod wraps BuildStagePodSpec for tests that don't +// care about the (rare) crypto/rand failure path — failing the +// test if it ever fires is the right escape from "if err != nil" +// noise in every assertion. The dedicated TestBuildStagePodSpec_ +// PropagatesRandError below covers the error path directly. +func mustBuildStagePod(t *testing.T, opts PodSpecOptions) *corev1.Pod { + t.Helper() + p, err := BuildStagePodSpec(opts) + if err != nil { + t.Fatalf("BuildStagePodSpec: %v", err) + } + return p +} + +// TestDNS1123SafeTableSegment is the High-severity regression pin +// for the Bugbot finding on PR-b. Every realistic table name in +// tracebloc docs uses snake_case (cats_dogs_train, chest_xrays_train); +// without the transform these would produce Pod names K8s rejects +// post-pre-flight, which is the worst-of-both-worlds UX. The +// transform's job: take any ValidateTableName-passed name and +// produce a DNS-1123 subdomain-safe segment for the Pod name. +func TestDNS1123SafeTableSegment(t *testing.T) { + cases := []struct { + in, want string + }{ + // Canonical happy path: snake_case → kebab-case. + {"cats_dogs", "cats-dogs"}, + {"chest_xrays_train", "chest-xrays-train"}, + // Uppercase: must lowercase. + {"MyTable", "mytable"}, + {"ABC", "abc"}, + // Already lowercase no-underscore: identity. + {"single", "single"}, + // Mixed: lowercase + underscore → hyphen. + {"Table_123", "table-123"}, + // Edge: leading underscore must be stripped (would + // otherwise produce "-leading", and while the full Pod + // name `tracebloc-stage--leading-` is valid DNS-1123 + // the consecutive hyphens are ugly). + {"_leading_underscore", "leading-underscore"}, + // Edge: trailing underscore stripped. + {"trailing_", "trailing"}, + // Edge: digit-led is valid (DNS-1123 allows it). + {"9starts_with_digit", "9starts-with-digit"}, + // Truncation: input > 30 chars caps at 30, then trims + // any trailing hyphen the cut might have left. + {strings.Repeat("a", 50), strings.Repeat("a", 30)}, + // Pathological: all-underscores → fallback "tbl". + {"_", "tbl"}, + {"___", "tbl"}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + got := dns1123SafeTableSegment(c.in) + if got != c.want { + t.Errorf("dns1123SafeTableSegment(%q) = %q, want %q", c.in, got, c.want) + } + // Defensive: the result must satisfy DNS-1123 + // subdomain rules for a Pod-name segment. Cheap + // regex check covers the contract. + if !isDNS1123SafeSegment(got) { + t.Errorf("dns1123SafeTableSegment(%q) = %q, which violates DNS-1123", + c.in, got) + } + }) + } +} + +// isDNS1123SafeSegment is a test-only helper: returns true iff s +// matches the DNS-1123 subdomain segment regex (lowercase alnum + +// hyphen, must start AND end with alnum). Matches what the K8s +// validation library checks for Pod names. +func isDNS1123SafeSegment(s string) bool { + if s == "" { + return false + } + for i, r := range s { + switch { + case r >= 'a' && r <= 'z': + case r >= '0' && r <= '9': + case r == '-': + // Hyphen forbidden at boundaries. + if i == 0 || i == len(s)-1 { + return false + } + default: + return false + } + } + return true +} + +// TestBuildStagePodSpec_Defaults pins the spec fields PR-b's +// post-create logic depends on: the SA name, the PVC mount, the +// activeDeadline, and the labels orphan.go keys off. +func TestBuildStagePodSpec_Defaults(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + + if p.Namespace != "tracebloc" { + t.Errorf("Namespace = %q, want tracebloc", p.Namespace) + } + // Pod name uses the DNS-1123-safe transform of the table name: + // "cats_dogs" → "cats-dogs". Bugbot flagged the raw-name version + // on PR-b as High severity (snake_case is the canonical naming + // style in tracebloc docs but K8s Pod names reject underscores). + if !strings.HasPrefix(p.Name, "tracebloc-stage-cats-dogs-") { + t.Errorf("Name = %q, want prefix tracebloc-stage-cats-dogs-", p.Name) + } + if got, wantLen := len(p.Name), len("tracebloc-stage-cats-dogs-")+8; got != wantLen { + t.Errorf("len(Name) = %d, want %d (8-hex-char random suffix)", got, wantLen) + } + if p.Spec.ServiceAccountName != "ingestor" { + t.Errorf("ServiceAccountName = %q, want ingestor", p.Spec.ServiceAccountName) + } + if p.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Errorf("RestartPolicy = %q, want Never", p.Spec.RestartPolicy) + } + if p.Spec.ActiveDeadlineSeconds == nil || *p.Spec.ActiveDeadlineSeconds != int64(StagePodActiveDeadline) { + t.Errorf("ActiveDeadlineSeconds = %v, want %d", p.Spec.ActiveDeadlineSeconds, StagePodActiveDeadline) + } +} + +// TestStagePodActiveDeadline_CoversFullLifecycle pins the floor on +// activeDeadlineSeconds. The earlier 600s value was too tight: the +// timer starts at Pod CREATION, but image pull (up to 60s) + +// readiness wait (up to 60s) + the worst-case stream (1 GiB at +// 2 MB/s = ~8.5 min) leaves no margin, so the kubelet could +// terminate near-cap pushes mid-transfer. Bugbot flagged as High +// on PR-b round 6. This test pins the floor at 1500s (25 min) so +// a future regression bumping it back near 600 gets caught. +func TestStagePodActiveDeadline_CoversFullLifecycle(t *testing.T) { + const minDeadline = 1500 + if StagePodActiveDeadline < minDeadline { + t.Errorf("StagePodActiveDeadline = %d, want at least %d "+ + "(must cover image-pull + readiness + worst-case 1 GiB stream "+ + "+ margin; activeDeadline starts at Pod creation not stream start)", + StagePodActiveDeadline, minDeadline) + } +} + +// TestBuildStagePodSpec_DefaultImage pins the digest-pinned image — +// if this ever drifts to a tag-only reference, air-gapped customers +// would silently get whatever alpine:3.20 resolved to that day. +func TestBuildStagePodSpec_DefaultImage(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + got := p.Spec.Containers[0].Image + if got != DefaultStagePodImage { + t.Errorf("Image = %q, want %q (default)", got, DefaultStagePodImage) + } + if !strings.Contains(got, "@sha256:") { + t.Errorf("Image = %q, want digest-pinned (@sha256:...); air-gapped customers depend on this", got) + } +} + +// TestBuildStagePodSpec_OverrideImage pins the --stage-pod-image +// flag's contract end-to-end at the spec layer. +func TestBuildStagePodSpec_OverrideImage(t *testing.T) { + opts := stageOpts() + opts.Image = "internal-mirror.example.com/alpine:3.20@sha256:abc123" + p := mustBuildStagePod(t, opts) + if got := p.Spec.Containers[0].Image; got != opts.Image { + t.Errorf("Image = %q, want override %q", got, opts.Image) + } +} + +// TestBuildStagePodSpec_LabelsForOrphanScan pins the labels orphan.go +// will key off. If these ever drift, orphan-pod detection silently +// misses leftover Pods from crashed pushes. +func TestBuildStagePodSpec_LabelsForOrphanScan(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + wantLabels := map[string]string{ + StagePodManagedByLabel: StagePodManagedByValue, + StagePodComponentLabel: StagePodComponentValue, + StagePodTableLabel: "cats_dogs", + } + for k, want := range wantLabels { + if got := p.Labels[k]; got != want { + t.Errorf("Label %s = %q, want %q", k, got, want) + } + } +} + +// TestBuildStagePodSpec_RestrictedPSA is the security-regression +// pin: every PSA-restricted requirement must hold, otherwise the +// stage Pod gets rejected on hardened namespaces (which is the +// majority of production tracebloc deployments). +func TestBuildStagePodSpec_RestrictedPSA(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + + // Pod-level: runAsNonRoot, seccomp RuntimeDefault. + psc := p.Spec.SecurityContext + if psc == nil { + t.Fatal("Pod has no SecurityContext") + } + if psc.RunAsNonRoot == nil || !*psc.RunAsNonRoot { + t.Errorf("Pod.SecurityContext.RunAsNonRoot = %v, want true", psc.RunAsNonRoot) + } + if psc.SeccompProfile == nil || psc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Errorf("Pod.SeccompProfile = %v, want RuntimeDefault", psc.SeccompProfile) + } + + // Container-level: every PSA restricted constraint. + if len(p.Spec.Containers) != 1 { + t.Fatalf("len(Containers) = %d, want 1", len(p.Spec.Containers)) + } + c := p.Spec.Containers[0] + sc := c.SecurityContext + if sc == nil { + t.Fatal("Container has no SecurityContext") + } + if sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation { + t.Errorf("AllowPrivilegeEscalation = %v, want false", sc.AllowPrivilegeEscalation) + } + if sc.Privileged == nil || *sc.Privileged { + t.Errorf("Privileged = %v, want false", sc.Privileged) + } + if sc.ReadOnlyRootFilesystem == nil || !*sc.ReadOnlyRootFilesystem { + t.Errorf("ReadOnlyRootFilesystem = %v, want true", sc.ReadOnlyRootFilesystem) + } + if sc.Capabilities == nil || len(sc.Capabilities.Drop) == 0 { + t.Fatalf("Capabilities.Drop = %v, want [ALL]", sc.Capabilities) + } + if sc.Capabilities.Drop[0] != "ALL" { + t.Errorf("Capabilities.Drop = %v, want [ALL]", sc.Capabilities.Drop) + } + if sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Errorf("Container SeccompProfile = %v, want RuntimeDefault", sc.SeccompProfile) + } +} + +// TestBuildStagePodSpec_PVCMount pins the volume + mountPath that +// the tar stream writes into. If the mount path here ever drifts +// from cluster.SharedPVCMountPath, the tar would write to the +// wrong location and Phase 4's ingestor Job would see "no files." +func TestBuildStagePodSpec_PVCMount(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + + // Volume side: the PVC reference. + var foundVol bool + for _, v := range p.Spec.Volumes { + if v.Name == "shared" { + foundVol = true + if v.PersistentVolumeClaim == nil { + t.Errorf("shared volume has no PVC source") + } else if v.PersistentVolumeClaim.ClaimName != "client-pvc" { + t.Errorf("PVC ClaimName = %q, want client-pvc", v.PersistentVolumeClaim.ClaimName) + } + } + } + if !foundVol { + t.Error("Pod has no shared volume") + } + + // Mount side: where the container sees it. + var foundMount bool + for _, m := range p.Spec.Containers[0].VolumeMounts { + if m.Name == "shared" { + foundMount = true + if m.MountPath != "/data/shared" { + t.Errorf("MountPath = %q, want /data/shared", m.MountPath) + } + } + } + if !foundMount { + t.Error("stage container has no shared mount") + } +} + +// TestBuildStagePodSpec_TmpEmptyDir pins the writable /tmp emptyDir +// that tar needs (since the root FS is read-only). Without this, +// tar would fail to create its working state and the stream would +// die mid-transfer. +func TestBuildStagePodSpec_TmpEmptyDir(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + var foundEmptyDir, foundMount bool + for _, v := range p.Spec.Volumes { + if v.Name == "tmp" { + foundEmptyDir = true + if v.EmptyDir == nil { + t.Error("tmp volume is not an EmptyDir") + } + } + } + for _, m := range p.Spec.Containers[0].VolumeMounts { + if m.Name == "tmp" && m.MountPath == "/tmp" { + foundMount = true + } + } + if !foundEmptyDir || !foundMount { + t.Errorf("tmp emptyDir mount missing (vol=%v, mount=%v)", foundEmptyDir, foundMount) + } +} + +// TestBuildStagePodSpec_RandomSuffixCollisionAvoidance: two specs +// built back-to-back must have distinct names so parallel pushes +// don't race on Create. +func TestBuildStagePodSpec_RandomSuffixCollisionAvoidance(t *testing.T) { + a := mustBuildStagePod(t, stageOpts()) + b := mustBuildStagePod(t, stageOpts()) + if a.Name == b.Name { + t.Errorf("back-to-back BuildStagePodSpec produced identical name %q; "+ + "random-suffix collision avoidance is broken", a.Name) + } +} + +// TestCreateStagePod_HappyPath: the fake clientset accepts a Create +// and returns the created object. We pin that CreateStagePod +// surfaces the assigned name back to the caller. +func TestCreateStagePod_HappyPath(t *testing.T) { + cs := fake.NewClientset() + name, err := CreateStagePod(context.Background(), cs, stageOpts()) + if err != nil { + t.Fatalf("CreateStagePod: %v", err) + } + if !strings.HasPrefix(name, "tracebloc-stage-cats-dogs-") { + t.Errorf("returned name = %q, want prefix tracebloc-stage-cats-dogs-", name) + } + // Cross-check: the Pod actually exists in the fake cluster. + if _, err := cs.CoreV1().Pods("tracebloc").Get(context.Background(), name, metav1.GetOptions{}); err != nil { + t.Errorf("Pod not found after CreateStagePod: %v", err) + } +} + +// TestCreateStagePod_APIErrorSurfaces: PSA rejections, RBAC denials, +// etc. surface verbatim so the customer sees the actionable cluster- +// side message. +func TestCreateStagePod_APIErrorSurfaces(t *testing.T) { + cs := fake.NewClientset() + cs.PrependReactor("create", "pods", + func(_ k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + corev1.Resource("pods"), "", + errors.New("user cannot create pods")) + }) + + _, err := CreateStagePod(context.Background(), cs, stageOpts()) + if err == nil { + t.Fatal("CreateStagePod returned nil error on forbidden Create") + } + if !strings.Contains(err.Error(), "creating stage Pod") { + t.Errorf("error missing CLI framing: %v", err) + } +} + +// TestWaitForStagePodReady_HappyPath: a Pod that immediately reports +// Ready=True passes through. +func TestWaitForStagePodReady_HappyPath(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tracebloc-stage-cats_dogs-abc12345", + Namespace: "tracebloc", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }}, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + p, err := WaitForStagePodReady(ctx, cs, "tracebloc", "tracebloc-stage-cats_dogs-abc12345") + if err != nil { + t.Fatalf("WaitForStagePodReady: %v", err) + } + if p == nil { + t.Fatal("returned nil Pod on success") + } +} + +// TestWaitForStagePodReady_TimeoutHint: when the Pod is stuck (e.g. +// ImagePullBackOff), the timeout error should surface the waiting- +// state reason from container statuses. This is the dominant slow +// path — air-gapped customers without the right pull secret. +func TestWaitForStagePodReady_TimeoutHint(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "stuck-pod", + Namespace: "tracebloc", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "stage", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ImagePullBackOff", + Message: "Back-off pulling image \"alpine:3.20@sha256:...\"", + }, + }, + }}, + }, + }) + + // Tight ctx timeout so the test doesn't actually wait 60s. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "stuck-pod") + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on stuck Pod") + } + if !strings.Contains(err.Error(), "ImagePullBackOff") { + t.Errorf("error missing ImagePullBackOff hint: %v", err) + } +} + +// TestWaitForStagePodReady_NotFoundIsTerminal: if the Pod gets +// deleted out-of-band mid-wait (admin cleanup, parallel test +// teardown, etc.), the poll must short-circuit instead of waiting +// the full timeout. Bugbot flagged the prior "everything transient" +// behavior as Medium on PR-b. +func TestWaitForStagePodReady_NotFoundIsTerminal(t *testing.T) { + cs := fake.NewClientset() // empty — Get returns NotFound + + // Generous overall ctx; the test should return WAY before + // StagePodReadyTimeout (60s) because NotFound terminates the + // poll immediately. If the fix regresses, this test takes the + // full 60s and gets caught by go test's -timeout. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "ghost-pod") + elapsed := time.Since(start) + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on NotFound") + } + if elapsed > 3*time.Second { + t.Errorf("WaitForStagePodReady waited %s for NotFound; expected immediate return", elapsed) + } +} + +// TestWaitForStagePodReady_ForbiddenIsTerminal: same as NotFound but +// for RBAC denial — the customer's kubeconfig might lose `get pods` +// permission mid-push (token rotation, RBAC change). The poll must +// surface that immediately, not spin until timeout. +func TestWaitForStagePodReady_ForbiddenIsTerminal(t *testing.T) { + cs := fake.NewClientset() + cs.PrependReactor("get", "pods", + func(_ k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + corev1.Resource("pods"), "test-pod", + errors.New("user cannot get pods")) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "test-pod") + elapsed := time.Since(start) + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on Forbidden") + } + if elapsed > 3*time.Second { + t.Errorf("WaitForStagePodReady waited %s for Forbidden; expected immediate return", elapsed) + } +} + +// TestWaitForStagePodReady_FailedPhaseIsTerminal: a Pod that +// crashes at startup (PSA rejection, ImagePullBackOff that escalates +// to ErrImagePull, OOMKilled during container start) lands in +// Phase=Failed and will never become Ready. The poll must +// short-circuit instead of waiting the full 60s. Bugbot flagged +// the missing check as Medium on PR-b round 4. +func TestWaitForStagePodReady_FailedPhaseIsTerminal(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "crashed-pod", + Namespace: "tracebloc", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "stage", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + Reason: "OOMKilled", + ExitCode: 137, + }, + }, + }}, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "crashed-pod") + elapsed := time.Since(start) + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on Phase=Failed") + } + if elapsed > 3*time.Second { + t.Errorf("WaitForStagePodReady waited %s for Phase=Failed; expected immediate return", elapsed) + } + if !strings.Contains(err.Error(), "Failed") { + t.Errorf("error missing Phase=Failed signal: %v", err) + } +} + +// TestDeleteStagePod_NotFoundIsOK: the Pod might be gone already +// (activeDeadlineSeconds fired, or someone kubectl-deleted it). +// Not-found shouldn't error — our goal is "Pod doesn't exist," which +// is satisfied. +func TestDeleteStagePod_NotFoundIsOK(t *testing.T) { + cs := fake.NewClientset() // empty + if err := DeleteStagePod(context.Background(), cs, "tracebloc", "nope"); err != nil { + t.Errorf("DeleteStagePod on absent Pod = %v, want nil", err) + } +} + +// TestDeleteStagePod_HappyPath: delete a real Pod, confirm it's gone. +func TestDeleteStagePod_HappyPath(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "to-delete", Namespace: "tracebloc"}, + }) + if err := DeleteStagePod(context.Background(), cs, "tracebloc", "to-delete"); err != nil { + t.Fatalf("DeleteStagePod: %v", err) + } + _, err := cs.CoreV1().Pods("tracebloc").Get(context.Background(), "to-delete", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + t.Errorf("Pod still exists after delete (err=%v)", err) + } +} diff --git a/internal/push/progress.go b/internal/push/progress.go new file mode 100644 index 0000000..f5df7b3 --- /dev/null +++ b/internal/push/progress.go @@ -0,0 +1,132 @@ +package push + +import ( + "io" + "os" + + "github.com/schollz/progressbar/v3" + "golang.org/x/term" +) + +// Progress is the narrow interface push.Stream + the stage +// orchestrator use to surface per-byte transfer progress to the +// customer. The real implementation wraps schollz/progressbar/v3 +// (NewTTYProgress); tests pass a no-op (NoOpProgress) since CI +// runs aren't TTYs and decoupling the test layer from any +// progressbar internals keeps test output clean. +// +// The interface is deliberately byte-oriented (not file-count +// oriented). Tar header bytes flow through too — they're a few +// hundred bytes per file, which makes the progress bar slightly +// over-count the "useful" bytes, but they're real bytes on the +// wire that the customer pays for anyway. The alternative +// (counting only file-body bytes) requires the progress sink to +// know what a tar header looks like, which couples it to the +// streaming format. +type Progress interface { + // Add records that n bytes have been transferred. Safe to + // call from any goroutine. + Add(n int) + + // Finish marks the transfer done. Calling Finish before all + // bytes have been added is fine — the bar just won't fill to + // 100%, which is a useful visual signal that something cut + // the stream short. + Finish() +} + +// NewProgress returns a Progress sink appropriate for the given +// output writer: +// +// - If out is a TTY (interactive terminal), return a +// progressbar/v3 that renders an animated bar with bytes/sec +// - ETA + percentage. +// - Otherwise return a no-op (CI, redirected output, piped to +// a file). A spinning bar in a CI log is noise, not a +// feature. +// +// totalBytes is the expected transfer size (from LocalLayout's +// TotalBytes plus a small tar-overhead margin). The bar normalizes +// against this; if actual transferred bytes overshoot, the bar +// caps at 100% rather than wrapping. +// +// `description` shows up to the left of the bar; "Staging cats_dogs" +// is a typical value. +func NewProgress(out io.Writer, totalBytes int64, description string) Progress { + if !isTTY(out) { + return NoOpProgress{} + } + return &ttyProgress{ + bar: progressbar.NewOptions64(totalBytes, + progressbar.OptionSetWriter(out), + progressbar.OptionSetDescription(description), + progressbar.OptionShowBytes(true), + progressbar.OptionShowCount(), + progressbar.OptionThrottle(100), // ms — keep CPU low + progressbar.OptionSetRenderBlankState(true), + progressbar.OptionSetWidth(40), + progressbar.OptionClearOnFinish(), + ), + } +} + +// isTTY reports whether out points at a real terminal. We test for +// *os.File AND golang.org/x/term IsTerminal — a *bytes.Buffer in +// tests isn't *os.File so we return false fast. Wrappers like +// io.MultiWriter that aren't *os.File also return false, which is +// the conservative choice. +func isTTY(out io.Writer) bool { + f, ok := out.(*os.File) + if !ok { + return false + } + return term.IsTerminal(int(f.Fd())) +} + +// ttyProgress is the real-bar implementation. The progressbar +// library is itself concurrency-safe under Add(), so the wrapper +// can be a thin pass-through. +type ttyProgress struct { + bar *progressbar.ProgressBar +} + +func (p *ttyProgress) Add(n int) { + // Explicit-discard the error — progressbar's Add returns an + // error if the write to the underlying writer fails, which we + // can't do anything about mid-stream (the customer's terminal + // going away during a push doesn't affect the staging + // outcome). Same rationale as the Fprintf discards elsewhere. + _ = p.bar.Add(n) +} + +func (p *ttyProgress) Finish() { + _ = p.bar.Finish() +} + +// NoOpProgress satisfies Progress without doing anything. Used in +// tests and when output is non-TTY. Exported so tests in OTHER +// packages (internal/cli) can also use it. +type NoOpProgress struct{} + +func (NoOpProgress) Add(_ int) {} +func (NoOpProgress) Finish() {} + +// progressWriter is an io.Writer that funnels write counts into a +// Progress. Used to wrap the stdin pipe-writer so every byte +// piped to the exec stream counts toward the bar. +// +// Not exported — it's an implementation detail of Stream. Tests +// validate the contract end-to-end (transferred bytes match +// expected) rather than poking the wrapper directly. +type progressWriter struct { + w io.Writer + p Progress +} + +func (pw *progressWriter) Write(b []byte) (int, error) { + n, err := pw.w.Write(b) + if n > 0 { + pw.p.Add(n) + } + return n, err +} diff --git a/internal/push/spec.go b/internal/push/spec.go index ec727b0..8aabb0a 100644 --- a/internal/push/spec.go +++ b/internal/push/spec.go @@ -40,18 +40,33 @@ import ( // table-naming style anyway. var tableNamePattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`) +// MaxTableNameLength caps `--table` at 63 chars. Two hard limits +// agree on this: +// +// 1. MySQL identifier limit: 64 chars (MySQL Reference 9.2 Schema +// Object Names). We use 63 to leave one char of headroom for +// any future "ingestion_run_id" suffix the chart might want to +// append. +// 2. Kubernetes label value limit: 63 chars (DNS-1123 label rules). +// The stage Pod's tracebloc.io/table label carries the +// untransformed table name verbatim — a longer value fails Pod +// creation post-pre-flight, which Bugbot flagged on PR-b +// round 5. +// +// 63 is a coincidence that lets both checks share the same constant. +const MaxTableNameLength = 63 + // ValidateTableName rejects table names that aren't safe as both a // MySQL identifier and a single PVC path segment. // // Why a CLI-side check rather than the schema: the embedded // ingest.v1.json only enforces `minLength: 1` on `table` — no -// `pattern`. Without this guard, --table=../../etc would flow into -// the /data/shared/
/ PVC path; PR-b's stage Pod would then -// write outside the intended subtree and could clobber another -// table's data. Tightening the upstream schema with a `pattern` -// is the proper long-term fix (it would protect the helm flow + -// jobs-manager too) but needs a change to tracebloc/data-ingestors' -// schema, which the schema-drift CI check pins — filed as +// `pattern`, no `maxLength`. Without this guard, --table=../../etc +// would flow into the /data/shared/
/ PVC path, and a 100-char +// name would make the Pod-label assignment fail post-pre-flight. +// Tightening the upstream schema is the proper long-term fix +// (it would protect the helm flow + jobs-manager too) but needs +// a change to tracebloc/data-ingestors' schema — filed as // tracebloc/data-ingestors#116. Once that lands and we re-sync, // this guard can collapse to a thin "schema says so" wrapper. // @@ -61,6 +76,14 @@ func ValidateTableName(table string) error { if table == "" { return fmt.Errorf("table name is required (set --table)") } + if len(table) > MaxTableNameLength { + return fmt.Errorf( + "table name is %d characters; the max is %d "+ + "(matches both the MySQL identifier limit and the "+ + "Kubernetes label-value limit, which the stage Pod's "+ + "tracebloc.io/table label is bound by). Use a shorter name.", + len(table), MaxTableNameLength) + } if !tableNamePattern.MatchString(table) { return fmt.Errorf( "table name %q is invalid: must match [A-Za-z0-9_]+ "+ diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go index 8f19716..05c7961 100644 --- a/internal/push/spec_test.go +++ b/internal/push/spec_test.go @@ -1,6 +1,7 @@ package push import ( + "strings" "testing" "gopkg.in/yaml.v3" @@ -126,6 +127,24 @@ func TestValidateTableName_Accepts(t *testing.T) { } } +// TestValidateTableName_RejectsTooLong: K8s label values are +// capped at 63 chars, and the stage Pod carries the raw table +// name as the tracebloc.io/table label. Without this rejection, +// a 100-char table name passes pre-flight then fails Pod create +// with an opaque label-validation error. Bugbot flagged on PR-b +// round 5. +func TestValidateTableName_RejectsTooLong(t *testing.T) { + tooLong := strings.Repeat("a", MaxTableNameLength+1) + if err := ValidateTableName(tooLong); err == nil { + t.Fatalf("ValidateTableName(%d-char name) = nil, want length-cap error", len(tooLong)) + } + // Right at the boundary: 63 chars must be accepted. + atCap := strings.Repeat("a", MaxTableNameLength) + if err := ValidateTableName(atCap); err != nil { + t.Errorf("ValidateTableName(%d-char name) = %v, want nil (at exact cap)", len(atCap), err) + } +} + // TestValidateTableName_RejectsUnsafe is the security-regression // pin. The path-traversal cases (../../etc, ../foo) are the ones // Bugbot flagged on PR #8 — if this test ever goes green with diff --git a/internal/push/stage.go b/internal/push/stage.go new file mode 100644 index 0000000..f8e9472 --- /dev/null +++ b/internal/push/stage.go @@ -0,0 +1,153 @@ +package push + +import ( + "context" + "fmt" + "io" + "time" + + "k8s.io/client-go/kubernetes" +) + +// StageOptions packages every dependency Stage needs into a single +// struct so callers (cli.runDatasetPush) can build it incrementally +// without juggling 10 positional args. +// +// Required fields are pinned in the doc; the nil-able Progress + Out +// fields get default behaviors (NoOpProgress / io.Discard) so the +// happy path is short. +type StageOptions struct { + // Client is the discovered kubernetes.Interface. + Client kubernetes.Interface + + // Executor is how the tar stream actually reaches the Pod. + // Production: a *SPDYExecutor backed by the rest.Config. + // Tests: a fakeExecutor (see stream_test.go). + Executor Executor + + // Namespace + IngestorSAName come from Phase 2's parent-release + // discovery (with optional --ingestor-sa override). + Namespace string + IngestorSAName string + + // PVCClaimName + PVCMountPath come from Phase 3 PR-a's PVC + // discovery (always "client-pvc" + "/data/shared" today). + PVCClaimName string + PVCMountPath string + + // Layout is the validated local files-to-stage. + Layout *LocalLayout + + // Table is the destination table name. MUST have already + // passed ValidateTableName. + Table string + + // StagePodImage overrides the alpine default. Empty = default. + StagePodImage string + + // Progress is the customer-facing transfer bar. Nil = no-op + // (use this in tests / non-TTY output). + Progress Progress + + // Out is where Stage prints non-progress diagnostic output + // (orphan warnings, "Created stage Pod X" etc.). Nil = io.Discard. + // Progress bar output is separate — schollz writes to its own + // configured writer (typically the same one). + Out io.Writer +} + +// Stage is Phase 3 PR-b's top-level entrypoint. Does the full +// dance: orphan scan → create stage Pod → wait Ready → tar-over- +// exec stream → defer-delete (SIGINT-safe). +// +// On success, the layout's files are on the cluster's PVC at +// /data/shared/
/{labels.csv, images/...} and the stage Pod +// has been cleaned up. Caller proceeds to Phase 4's submit step. +// +// On error, the deferred cleanup still fires — no orphan Pod left +// behind even if the stream fails partway. Phase 4's idempotency +// key handles the "retry the whole push" scenario; for v0.1 we +// don't try to resume a partial transfer. +// +// SIGINT handling: the caller is expected to wire a signal handler +// that cancels the passed-in ctx (cli/main.go pattern). When ctx +// is cancelled, the in-flight Exec stream returns ctx.Err(), the +// deferred DeleteStagePod runs with a FRESH context (so it can +// reach the API even after our parent ctx is cancelled), and the +// error propagates with the SIGINT signal preserved. +func Stage(ctx context.Context, opts StageOptions) error { + if opts.Out == nil { + opts.Out = io.Discard + } + if opts.Progress == nil { + opts.Progress = NoOpProgress{} + } + + // 1. Orphan scan — best-effort. We log either the warning or + // the scan-failure and proceed regardless. Customers who + // care about orphans get to see them; customers whose RBAC + // forbids List Pods just don't get the warning (no impact + // on the push itself). + if orphans, err := FindOrphanStagePods(ctx, opts.Client, opts.Namespace); err != nil { + _, _ = fmt.Fprintf(opts.Out, "(orphan-scan skipped: %v)\n", err) + } else if warning := FormatOrphansWarning(orphans); warning != "" { + _, _ = fmt.Fprint(opts.Out, warning) + } + + // 2. Create the stage Pod. From here on, cleanup matters. + podName, err := CreateStagePod(ctx, opts.Client, PodSpecOptions{ + Namespace: opts.Namespace, + PVCClaimName: opts.PVCClaimName, + PVCMountPath: opts.PVCMountPath, + Table: opts.Table, + Image: opts.StagePodImage, + ServiceAccountName: opts.IngestorSAName, + }) + if err != nil { + return err + } + _, _ = fmt.Fprintf(opts.Out, "Created stage Pod %s/%s\n", opts.Namespace, podName) + + // 3. Defer cleanup. The deferred call uses a FRESH context with + // its own deadline — if the parent ctx is cancelled (SIGINT, + // timeout), we still want the Pod deleted. Without this, a + // Ctrl-C right after pod-create would leak an orphan. + // + // 30s deadline on the cleanup is a safety net for the case + // where the API server itself is unreachable — better to + // print "failed to clean up stage Pod" than hang forever. + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if delErr := DeleteStagePod(cleanupCtx, opts.Client, opts.Namespace, podName); delErr != nil { + _, _ = fmt.Fprintf(opts.Out, + "WARNING: failed to delete stage Pod %s/%s (it will be activeDeadline-killed in <=%ds): %v\n", + opts.Namespace, podName, StagePodActiveDeadline, delErr) + } + }() + + // 4. Wait for the Pod to be Ready (containers up, ready to + // accept exec). Times out at StagePodReadyTimeout with + // diagnostic hints from container statuses (image pull, + // scheduling) when we have them. + _, _ = fmt.Fprintf(opts.Out, "Waiting for stage Pod to be Ready (timeout %s)...\n", StagePodReadyTimeout) + if _, err := WaitForStagePodReady(ctx, opts.Client, opts.Namespace, podName); err != nil { + return err + } + + // 5. Stream the tar. This is where actual bytes flow. The + // progress bar (if TTY) renders during this call. + _, _ = fmt.Fprintf(opts.Out, "Streaming %d files (%s) to %s...\n", + 1+len(opts.Layout.Images), HumanBytes(opts.Layout.TotalBytes), StagedPrefix(opts.Table)) + + if err := StreamLayout(ctx, opts.Executor, + opts.Namespace, podName, "stage", + opts.Layout, opts.Table, opts.Progress); err != nil { + return err + } + + // 6. Print "done" message. The deferred cleanup runs after this. + _, _ = fmt.Fprintf(opts.Out, "Staged %d files to %s\n", + 1+len(opts.Layout.Images), StagedPrefix(opts.Table)) + return nil +} diff --git a/internal/push/stage_test.go b/internal/push/stage_test.go new file mode 100644 index 0000000..b46e0cb --- /dev/null +++ b/internal/push/stage_test.go @@ -0,0 +1,288 @@ +package push + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// readyOnNextGet adds a reactor that marks any new stage Pod as +// Ready=True the moment after it's created. Without this, the fake +// clientset creates the Pod in default (empty) status and +// WaitForStagePodReady spins until its timeout. We can't easily +// patch the Pod from inside StageOptions construction, so the +// reactor is the standard pattern. +func readyOnNextGet(cs *fake.Clientset) { + cs.PrependReactor("get", "pods", + func(action k8stesting.Action) (bool, runtime.Object, error) { + ga, ok := action.(k8stesting.GetAction) + if !ok { + return false, nil, nil + } + // Let the existing tracker handle the get to find the + // real object, then patch it before returning. The + // safest way: do the get ourselves, then synthesize a + // Ready condition onto the result. + pod, err := cs.Tracker().Get(action.GetResource(), action.GetNamespace(), ga.GetName()) + if err != nil { + return false, nil, nil + } + p := pod.(*corev1.Pod) + p.Status.Phase = corev1.PodRunning + p.Status.Conditions = []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }} + return true, p, nil + }) +} + +// TestStage_HappyPath: orphan scan (empty) → create → wait ready +// → stream → delete. The fake clientset + fake executor let us +// verify the full orchestration without a real cluster. +func TestStage_HappyPath(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{} + var out bytes.Buffer + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err = Stage(ctx, StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "cats_dogs", + Out: &out, + }) + if err != nil { + t.Fatalf("Stage: %v\nout:\n%s", err, out.String()) + } + + // Diagnostic output: customer should see the lifecycle + // breadcrumbs. Pin a few key phrases so a regression that + // silences them is caught. + for _, want := range []string{"Created stage Pod", "Waiting for stage Pod", "Streaming", "Staged"} { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q:\n%s", want, out.String()) + } + } + + // Executor must have been called with the right ns + a tar + // stream on stdin (assert by inspecting captured tar). + if fe.gotNS != "tracebloc" { + t.Errorf("executor ns = %q, want tracebloc", fe.gotNS) + } + if len(fe.gotStdin) == 0 { + t.Error("executor stdin was empty; expected tar archive bytes") + } +} + +// TestStage_CleanupRunsOnError: deferred DeleteStagePod must fire +// even when StreamLayout fails. Otherwise every failed push leaks +// an orphan Pod that can't be reused for retry (the random suffix +// avoids name collisions, but it's still cluster pollution). +func TestStage_CleanupRunsOnError(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{ + errToReturn: errors.New("simulated stream failure"), + drainBeforeReturn: true, + } + var out bytes.Buffer + + err = Stage(context.Background(), StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "t1", + Out: &out, + }) + if err == nil { + t.Fatal("Stage returned nil on stream failure") + } + + // Cleanup contract: NO stage Pods should remain after Stage + // returns. The list-by-label query is the same one orphan.go + // uses, so if any leaks it's user-visible in the next push. + pods, listErr := cs.CoreV1().Pods("tracebloc").List(context.Background(), + metav1.ListOptions{LabelSelector: StagePodManagedByLabel + "=" + StagePodManagedByValue}) + if listErr != nil { + t.Fatalf("post-Stage list: %v", listErr) + } + if len(pods.Items) != 0 { + var names []string + for _, p := range pods.Items { + names = append(names, p.Name) + } + t.Errorf("Stage leaked %d Pod(s) after stream failure: %v", len(pods.Items), names) + } +} + +// TestStage_OrphanWarningSurfaces: stale stage Pods from a +// previous crash get surfaced as a warning at the start of the +// next push. Validates the integration: orphan scan → format → +// print to Out. +func TestStage_OrphanWarningSurfaces(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset(stagePodWithAge("stale-from-crash", "cats_dogs", 30*time.Minute)) + readyOnNextGet(cs) + fe := &fakeExecutor{} + var out bytes.Buffer + + err = Stage(context.Background(), StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "new_push", + Out: &out, + }) + if err != nil { + t.Fatalf("Stage: %v\nout:\n%s", err, out.String()) + } + + if !strings.Contains(out.String(), "stale-from-crash") { + t.Errorf("orphan warning didn't surface; output:\n%s", out.String()) + } + if !strings.Contains(out.String(), "kubectl delete pod") { + t.Errorf("orphan warning missing actionable hint; output:\n%s", out.String()) + } +} + +// TestStage_IngestorSANameFlowsToPod: the --ingestor-sa override +// MUST land on the stage Pod's ServiceAccountName — otherwise the +// flag is silently ignored and customers who renamed the chart's +// ingestor SA get pods running as the default SA (no PVC write +// access). Pin the integration here at the Stage layer since the +// flag wiring's effect is only observable end-to-end. Bugbot +// flagged the missing test coverage on PR-a. +func TestStage_IngestorSANameFlowsToPod(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{} + var out bytes.Buffer + + const customSA = "my-renamed-ingestor" + _ = Stage(context.Background(), StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: customSA, + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "t1", + Out: &out, + }) + + // List the stage Pods created (we deleted them in Stage's + // defer, so we need to inspect the tracker BEFORE the test + // process finishes — but the cleanup happens within Stage). + // Workaround: assert via the fake clientset's action log — + // look at the Create action's object directly. + var seenSA string + for _, action := range cs.Actions() { + if action.GetVerb() == "create" && action.GetResource().Resource == "pods" { + pod := action.(k8stesting.CreateAction).GetObject().(*corev1.Pod) + seenSA = pod.Spec.ServiceAccountName + break + } + } + if seenSA != customSA { + t.Errorf("created Pod's ServiceAccountName = %q, want %q (the --ingestor-sa override)", + seenSA, customSA) + } +} + +// TestStage_CancelledContext_StillCleansUp: when the parent ctx +// is cancelled (SIGINT scenario), the deferred cleanup MUST still +// run — using its own context with a fresh deadline. Without +// this, every Ctrl-C leaves an orphan Pod. +func TestStage_CancelledContext_StillCleansUp(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{ + // Simulate the SPDY stream returning ctx.Err() because the + // caller cancelled mid-stream. + errToReturn: context.Canceled, + } + var out bytes.Buffer + + // Use a context we cancel immediately. Stage's create still + // succeeds (fake clientset doesn't honor ctx on Create), but + // the executor sees Canceled. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _ = Stage(ctx, StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "interrupted", + Out: &out, + }) + + // Whether Stage returns an error or not, the cluster MUST be + // clean. This is the SIGINT-safety contract. + pods, _ := cs.CoreV1().Pods("tracebloc").List(context.Background(), + metav1.ListOptions{LabelSelector: StagePodManagedByLabel + "=" + StagePodManagedByValue}) + if len(pods.Items) != 0 { + t.Errorf("Stage leaked %d Pod(s) on context-cancel; SIGINT cleanup contract broken", + len(pods.Items)) + } +} diff --git a/internal/push/stream.go b/internal/push/stream.go new file mode 100644 index 0000000..baba3cb --- /dev/null +++ b/internal/push/stream.go @@ -0,0 +1,462 @@ +package push + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" +) + +// Executor abstracts "exec a command in a Pod with stdin/stdout/ +// stderr." The interface exists for one reason: client-go's +// fake.Clientset (the cornerstone of every test in this package) +// doesn't support the exec subresource. Without an interface +// boundary here, Stream's lifecycle couldn't be unit-tested — +// every assertion would need a real cluster. +// +// Production uses SPDYExecutor (this file). Tests construct a +// FakeExecutor (stream_test.go) that captures stdin into a buffer +// + records the command + can return a synthetic error. +type Executor interface { + // Exec runs cmd in the named container of ns/pod, wiring + // stdin/stdout/stderr to the provided streams. Returns when + // the remote command terminates or the context is cancelled. + // + // stdin, stdout, stderr may be nil — Exec must tolerate that + // (a stage Pod with no stderr to capture is a legitimate + // case for the "just delete me" command after the tar is + // done). + Exec(ctx context.Context, ns, pod, container string, + cmd []string, stdin io.Reader, stdout, stderr io.Writer) error +} + +// SPDYExecutor is the production Executor. Wraps client-go's +// remotecommand.NewSPDYExecutor — the same machinery `kubectl exec` +// uses internally. +// +// Holds both Config (needed by NewSPDYExecutor for auth/transport) +// and Client (for building the exec URL via the REST client). Both +// come from cluster.Load + cluster.NewClientset. +type SPDYExecutor struct { + Config *rest.Config + Client kubernetes.Interface +} + +func (e *SPDYExecutor) Exec( + ctx context.Context, + ns, pod, container string, + cmd []string, + stdin io.Reader, stdout, stderr io.Writer, +) error { + // Build the exec URL the way kubectl does — POST against the + // /pods//exec subresource, parametrized via PodExecOptions + // encoded through the scheme's ParameterCodec. + req := e.Client.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod). + Namespace(ns). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: cmd, + // Toggle the streams that the caller actually wants. + // PodExecOptions ignores the unused ones, but being + // explicit here matches kubectl's behavior and means + // "stdin requested but caller passed nil" surfaces as + // a clearer error from the executor. + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(e.Config, "POST", req.URL()) + if err != nil { + return fmt.Errorf("creating SPDY executor for %s/%s: %w", ns, pod, err) + } + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + // Tty=false: we're not running an interactive shell, we're + // piping bytes. With Tty=true the tar stream would get + // line-buffered/CR-translated/etc, which corrupts the + // archive. + Tty: false, + }) + if err != nil { + return fmt.Errorf("exec stream against %s/%s: %w", ns, pod, err) + } + return nil +} + +// StreamLayout pipes a tar of the layout's files into the stage +// Pod's `tar -xf - -C /data/shared/
` so the files land at +// /data/shared/
/{labels.csv, images/...}. +// +// On success, the stage Pod's PVC mount now contains the files +// jobs-manager's ingestor will read in Phase 4. +// +// On failure, the partial-write state on the PVC is whatever tar +// managed before the error — possibly a few image files but +// likely not all. Phase 4's idempotency_key handles "retry the +// whole push"; for v0.1, customers re-run the command and the +// shell's `mkdir -p` is idempotent so re-creating the destination +// is fine. Truly-recovering a failed mid-stream push (resume from +// last-written file) is a v0.2 optimization. +// +// Why `tar` (rather than a more modern protocol like a custom REST +// upload endpoint on jobs-manager): tar-over-exec works against +// any cluster running the existing chart, with zero server-side +// changes. A custom upload endpoint would require a coordinated +// jobs-manager release before the CLI could ship. +func StreamLayout( + ctx context.Context, + exec Executor, + namespace, podName, containerName string, + layout *LocalLayout, + table string, + progress Progress, +) error { + if progress == nil { + progress = NoOpProgress{} + } + defer progress.Finish() + + // Generate the staging-dir suffix + compose the remote command + // BEFORE spawning the tar goroutine. Earlier versions did the + // suffix after, which meant a crypto/rand failure would return + // from this function without closing the pipe or waiting on + // the goroutine — the goroutine would block once the ~64 KB + // pipe buffer filled, leaking forever. Bugbot flagged on r9. + // + // Staging dir gets a fresh 8-hex-char suffix per invocation so + // two concurrent `dataset push` runs for the same table don't + // race on the same .staging path (r8). Without this, push A's + // `rm -rf .staging` could wipe push B's in-progress tar + // extraction (or vice versa), producing an interleaved + // corrupt dataset on the PVC. + stagingSuffix, err := randomSuffix(4) // 4 bytes → 8 hex chars + if err != nil { + return fmt.Errorf("generating staging-dir suffix: %w", err) + } + dest := StagedPrefix(table) + staging := dest + ".staging-" + stagingSuffix + + // Backup path for the swap. Same unique suffix as staging so + // two parallel pushes don't collide on each other's .old-, + // AND so the find pattern below catches both .staging-* and + // .old-* siblings as orphan candidates. + backup := dest + ".old-" + stagingSuffix + + parentDir := path.Dir(dest) + tableBase := path.Base(dest) + + // Remote command pipeline. Crosses three semantic guarantees: + // + // - HERMETIC (r5): old files don't survive into a re-push + // - TRANSACTIONAL (r7+r10): the previously-staged dataset + // stays intact if THIS push fails mid-transfer; backup- + // and-rollback restores on mv failure + // - PARALLEL-SAFE (r8): unique .staging-/.old- + // suffix per invocation so concurrent pushes don't + // interleave each other's tar extraction + // - EXIT-FAITHFUL (r11): set -e propagates any non-zero + // status; the prior `&&-chain ; find || true` shape + // could silently return success on actual failures + // + // Pipeline steps: + // 1. set -e — any unhandled non-zero status aborts the script + // 2. rm -rf $STAGING (clean prior failed attempt) + // 3. mkdir + tar (extract to staging) + // 4. If $DEST exists, mv $DEST → $DEST.old- (backup) + // 5. mv $STAGING → $DEST (commit) + // On failure: restore backup, exit 1 + // 6. rm -rf $BACKUP (success) + // 7. find ... orphan cleanup (best-effort, || true) + // + // The earlier `&&-chained ... ; find ... || true` shape had a + // hidden correctness bug (Bugbot r11): POSIX `;` runs the + // next command regardless of prior status, and `|| true` then + // forces exit 0. A failed tar mid-script would silently + // return success to the exec subprocess, and the CLI would + // report "Staged N files" on what was actually a failed push. + // + // set -e fixes that: any unguarded non-zero exits the script + // with that status. The find's `|| true` is still fine + // because under set -e a `cmd || fallback` is treated as + // "cmd may fail" and doesn't trigger the immediate exit. + // + // Failure modes (recap): + // - tar fails: set -e aborts; $DEST untouched (transactional from r7) + // - backup mv fails: same — aborts before any change to $DEST + // - main mv fails: explicit if/then rolls back the backup + // then `exit 1` (set -e doesn't preempt the explicit + // handler because of the `||`) + // - final rm fails: set -e exits, but the customer's new + // data is already in place — the leftover .old- + // gets picked up by the find sweep on next invocation + // - find fails: `|| true` suppresses (cleanup is best-effort) + remoteCmd := []string{ + "/bin/sh", "-c", + fmt.Sprintf( + "set -e\n"+ + "rm -rf %q\n"+ + "mkdir -p %q\n"+ + "/bin/tar -xf - -C %q\n"+ + "if [ -e %q ]; then mv %q %q; fi\n"+ + "if ! mv %q %q; then\n"+ + " if [ -e %q ]; then mv %q %q; fi\n"+ + " exit 1\n"+ + "fi\n"+ + "rm -rf %q\n"+ + "find %q -maxdepth 1 \\( -name %q -o -name %q \\) -mmin +60 -exec rm -rf {} + 2>/dev/null || true\n", + staging, // rm + staging, // mkdir + staging, // tar -C + dest, dest, backup, // if exists, backup + staging, dest, // main mv + backup, backup, dest, // rollback + backup, // success cleanup + parentDir, tableBase+".staging-*", tableBase+".old-*"), // find + } + + // Use a synchronous pipe so the tar Writer's writes block on + // the exec stream's reads — no in-memory buffering of the + // whole archive. For a 1 GiB dataset this is the difference + // between using 1 GiB of laptop RAM and using a few KB. + pr, pw := io.Pipe() + + // Wire the progress sink on the WRITE side so every byte the + // tar Writer emits (headers + bodies) counts toward the bar. + countedPW := &progressWriter{w: pw, p: progress} + + // Capture stderr from the in-cluster tar so failures surface + // with the actual remote message. + var stderrBuf bytes.Buffer + + // Kick off the tar build in a goroutine. The Pipe.Writer MUST + // be closed when we're done — without that, the Pipe.Reader + // side blocks forever waiting for more bytes. + // + // CloseWithError vs Close: CloseWithError preserves the tar + // error across the pipe so the exec side sees the cause. + tarErrCh := make(chan error, 1) + go func() { + err := writeLayoutTar(countedPW, layout) + if err != nil { + _ = pw.CloseWithError(err) + } else { + _ = pw.Close() + } + tarErrCh <- err + }() + + streamErr := exec.Exec(ctx, namespace, podName, containerName, + remoteCmd, pr, nil, &stderrBuf) + + // Close the reader side. If the executor returned without + // fully draining stdin (early exit, ctx cancellation, remote + // tar dying), the tar-write goroutine is blocked on its next + // pipe.Write — closing the reader unblocks it with + // io.ErrClosedPipe so it can finish and report on tarErrCh. + // Without this, every error path deadlocks. + _ = pr.Close() + + // Drain the tar goroutine — even on stream error, it must + // finish or we leak. Buffered channel size 1 means this + // receive won't block (the goroutine already sent or will + // send imminently after CloseWithError or the pr.Close above). + tarErr := <-tarErrCh + + // Order matters here. Bugbot r11 caught the previous logic: + // when the LOCAL tar build fails (e.g. r8's stream-time size + // cap recheck rejecting a file that grew on disk), the + // CloseWithError on the pipe causes exec to see "broken pipe" + // and return a generic stream error. If we report streamErr + // first, the customer sees "streaming failed" instead of the + // actually-actionable "dataset exceeded v0.1 cap" diagnostic + // from the tar side. + // + // Check tarErr first — when both are non-nil, the tar side + // is usually the upstream cause. streamErr-only (no tarErr) + // is the genuine network/RBAC/remote-tar-failed case where + // the exec wording is the right surface. + if tarErr != nil { + return fmt.Errorf("building tar archive: %w", tarErr) + } + if streamErr != nil { + hint := "" + if remote := stderrBuf.String(); remote != "" { + hint = fmt.Sprintf(" (remote tar stderr: %s)", remote) + } + return fmt.Errorf("streaming files to %s/%s: %w%s", namespace, podName, streamErr, hint) + } + return nil +} + +// writeLayoutTar packages the layout's files into the writer as a +// flat tar archive with this structure: +// +// labels.csv (file) +// images/ (file, per layout.Images) +// +// The destination tar is unpacked with `tar -xf - -C /data/shared/
`, +// so file paths are RELATIVE to that root — that's why we write +// "labels.csv" not "/data/shared/
/labels.csv". +// +// Mode is 0644 on all files (read-only for the ingestor SA). Tar's +// type flag is TypeReg (regular file). No symlinks, no hard links, +// no extended attrs — the layout we accept doesn't have any. +// +// Uses a named return + deferred Close so a tar trailer-write error +// (truncated archive — GNU tar refuses to extract these) propagates +// even on the happy path. errcheck would otherwise flag the bare +// `defer tw.Close()`, and rightly: silently dropping that error +// means a truncated stream looks identical to a successful one +// from this function's caller. +func writeLayoutTar(w io.Writer, layout *LocalLayout) (err error) { + tw := tar.NewWriter(w) + defer func() { + // If we already have an error, preserve it; the close error + // is less useful than whatever caused the early return. + // Otherwise surface the close error so a failed trailer + // write doesn't silently corrupt the archive. + if cerr := tw.Close(); cerr != nil && err == nil { + err = fmt.Errorf("closing tar writer: %w", cerr) + } + }() + + // Running total enforces the v0.1 MaxTotalBytes cap at STREAM + // time, not just at Discover time. A file that grew between + // pre-flight and now (TOCTOU) — or just a sum miscount — would + // otherwise sneak past the cap silently. Bugbot flagged on + // PR-b r8 as Medium. + var totalBytes int64 + + // labels.csv first (small, sanity-checks the stream quickly). + n, err := writeTarFile(tw, layout.LabelsCSV, "labels.csv") + if err != nil { + return fmt.Errorf("packaging labels.csv: %w", err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s during stream "+ + "(labels.csv alone is %s; pre-flight likely raced with "+ + "a file growing on disk)", + HumanBytes(MaxTotalBytes), HumanBytes(totalBytes)) + } + + // Then each image. We write them in the order Discover returned + // (filesystem-walk order, not sorted) — sorting would make the + // stream deterministic across runs but adds zero customer value. + for _, abs := range layout.Images { + // The destination filename inside images/ is the file's + // basename — strips the customer's local path so a push + // from /home/alice/datasets/cats_dogs/images/001.jpg + // becomes images/001.jpg in the tar (and on the PVC). + // + // Use path.Join (always forward-slash) NOT filepath.Join + // (OS-native separator) for the tar HEADER name. USTAR/POSIX + // tar requires forward slashes; a Windows-built CLI using + // filepath.Join would emit `images\001.jpg` headers that the + // Linux stage Pod's `tar -xf` either rejects or extracts as + // a flat-named file. Bugbot flagged on PR-b round 6. + dst := path.Join("images", filepath.Base(abs)) + n, err := writeTarFile(tw, abs, dst) + if err != nil { + return fmt.Errorf("packaging %s: %w", abs, err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s after streaming %s "+ + "(reached %s; file growth between pre-flight and stream)", + HumanBytes(MaxTotalBytes), dst, HumanBytes(totalBytes)) + } + } + + // tw.Close() in the defer above writes the tar footer + // (two zero blocks). Without that, GNU tar treats the archive + // as truncated and refuses to extract. + return nil +} + +// writeTarFile writes one file from `src` into tw under the +// archive-relative name `dst`. Streams the file body — no full- +// read into memory — so a single 500 MiB image doesn't balloon +// the CLI's memory. +// +// Returns the file's declared size (from os.Lstat) so the caller +// can maintain a running total against MaxTotalBytes — closes the +// stream-time half of the TOCTOU window Bugbot flagged on r8. +// +// Uses os.Lstat (not Stat) + an explicit symlink reject as a +// defense-in-depth second layer behind walk.go's rejectSymlink +// (Bugbot r4). Also re-checks the single-file size cap at stream +// time: a file that grew between Discover and now would otherwise +// upload past the advertised 500 MiB cap (Bugbot r8). +func writeTarFile(tw *tar.Writer, src, dst string) (int64, error) { + st, err := os.Lstat(src) + if err != nil { + return 0, err + } + if st.Mode()&os.ModeSymlink != 0 { + // Reaching here means Discover let a symlink through — + // shouldn't happen in production, but the explicit error + // keeps the security property locally enforceable. + return 0, fmt.Errorf("refusing to stream symbolic link %q (defense-in-depth; should have been rejected by Discover)", src) + } + if st.Size() > MaxSingleFileBytes { + // Stream-time recheck of the single-file cap. Discover + // caught this in pre-flight; if we see it again here, the + // file grew between then and now. + return 0, sizeError(dst, st.Size(), MaxSingleFileBytes) + } + hdr := &tar.Header{ + Name: dst, + Size: st.Size(), + Mode: 0o644, + Typeflag: tar.TypeReg, + // We don't carry ModTime forward — the customer's local + // mtime has no useful semantic on the PVC, and zeroing it + // keeps the tar bit-for-bit reproducible across runs + // (helps tests, makes future content-hash checks easier). + } + if err := tw.WriteHeader(hdr); err != nil { + return 0, err + } + f, err := os.Open(src) + if err != nil { + return 0, err + } + // Read-only file Close errors are not meaningful — there's + // nothing the caller can do, and the underlying file descriptor + // gets returned to the OS regardless. Explicit-discard pattern, + // same as the Fprintf writers elsewhere. + defer func() { _ = f.Close() }() + // io.CopyN caps the actual stream at the declared header size. + // Without the cap, a file that grew between Lstat above and + // the Copy here would overflow the header — the tar would be + // corrupted (header says N bytes, body has > N). Cap-and-trust + // is the safe choice: if the file shrank, the tar trailer + // surfaces the mismatch; if the file grew, we just stop at + // the declared size and let the new bytes get re-pushed next + // invocation. Closes the body-side half of the TOCTOU window. + if _, err := io.CopyN(tw, f, st.Size()); err != nil && !errors.Is(err, io.EOF) { + return 0, err + } + return st.Size(), nil +} diff --git a/internal/push/stream_test.go b/internal/push/stream_test.go new file mode 100644 index 0000000..62bde45 --- /dev/null +++ b/internal/push/stream_test.go @@ -0,0 +1,428 @@ +package push + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "regexp" + "sort" + "strings" + "testing" +) + +// fakeExecutor is the stand-in for SPDYExecutor in tests. Captures +// every Exec call's parameters into fields the test can assert on, +// and reads the stdin stream into a buffer (so we can parse the tar +// archive back out and verify its contents). +type fakeExecutor struct { + // Captured call parameters. + gotNS, gotPod, gotContainer string + gotCmd []string + + // Captured stdin (the tar archive). + gotStdin []byte + + // What to write back to the caller as stderr (simulates a + // "tar: no space left on device" style remote-side failure). + stderrToReturn []byte + + // What to return as the exec error (simulates a 4xx/5xx from + // the apiserver, a network drop, etc.). + errToReturn error + + // drainBeforeReturn: if false, errToReturn fires BEFORE the + // stdin pipe is drained — simulates a fast failure where the + // remote tar dies on the first byte. Test infrastructure for + // broken-pipe coverage. + drainBeforeReturn bool +} + +func (f *fakeExecutor) Exec( + _ context.Context, + ns, pod, container string, + cmd []string, + stdin io.Reader, _ io.Writer, stderr io.Writer, +) error { + f.gotNS, f.gotPod, f.gotContainer = ns, pod, container + f.gotCmd = cmd + + if stdin != nil && (f.drainBeforeReturn || f.errToReturn == nil) { + b, _ := io.ReadAll(stdin) + f.gotStdin = b + } + if len(f.stderrToReturn) > 0 && stderr != nil { + _, _ = stderr.Write(f.stderrToReturn) + } + return f.errToReturn +} + +// TestStreamLayout_TarPathsAreForwardSlash pins the cross-platform +// tar header convention. On Windows, filepath.Join uses '\' as +// separator — using it for the tar HEADER name would produce +// archive entries the Linux stage Pod's `tar -xf` either rejects +// or extracts as flat-named files (collapsing the images/ subdir). +// The fix is path.Join (always forward-slash); this test asserts +// the portable property regardless of which OS the test runs on. +// Bugbot flagged the Windows bug on PR-b round 6. +func TestStreamLayout_TarPathsAreForwardSlash(t *testing.T) { + root := imgcDir(t, "a.jpg", "b.png") + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{} + if err := StreamLayout(context.Background(), fe, "tracebloc", "p", "stage", + layout, "t", NoOpProgress{}); err != nil { + t.Fatalf("StreamLayout: %v", err) + } + + tr := tar.NewReader(bytes.NewReader(fe.gotStdin)) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("reading tar: %v", err) + } + if strings.Contains(hdr.Name, `\`) { + t.Errorf("tar entry %q contains backslash; "+ + "Linux stage Pod's `tar -xf` requires forward-slash paths", hdr.Name) + } + } +} + +// TestStreamLayout_TarContents end-to-end: the bytes the executor +// sees on stdin MUST be a valid tar with the layout's files at the +// expected paths (labels.csv at the root, images/ under +// images/). If this drifts, the in-cluster `tar -xf - -C +// /data/shared/
/` lands files in the wrong place and +// jobs-manager's ingestor in Phase 4 silently sees zero rows. +func TestStreamLayout_TarContents(t *testing.T) { + root := imgcDir(t, "a.jpg", "b.png") + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + fe := &fakeExecutor{} + err = StreamLayout(context.Background(), fe, "tracebloc", "pod-x", "stage", + layout, "cats_dogs", NoOpProgress{}) + if err != nil { + t.Fatalf("StreamLayout: %v", err) + } + + // Parse the captured tar back out and collect (name, size). + type entry struct { + name string + size int64 + } + var got []entry + tr := tar.NewReader(bytes.NewReader(fe.gotStdin)) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("reading tar: %v", err) + } + got = append(got, entry{name: hdr.Name, size: hdr.Size}) + } + + // Sort for deterministic comparison — the producer walks + // images/ in OS order which isn't stable across filesystems. + sort.Slice(got, func(i, j int) bool { return got[i].name < got[j].name }) + + // Three entries: labels.csv + 2 images. + if len(got) != 3 { + t.Fatalf("tar entry count = %d, want 3 (labels.csv + 2 images); got=%v", len(got), got) + } + want := []entry{ + {name: "images/a.jpg", size: 100}, // imgcDir writes 100-byte images + {name: "images/b.png", size: 100}, + {name: "labels.csv", size: 39}, // "image_id,label\n001.jpg,cat\n002.jpg,dog\n" + } + for i := range want { + if got[i] != want[i] { + t.Errorf("entry[%d] = %v, want %v", i, got[i], want[i]) + } + } +} + +// TestStreamLayout_RemoteCommand pins the in-cluster command. If +// this ever drifts (e.g. someone changes the destination path +// helper), the tar gets extracted to the wrong PVC subdirectory +// and Phase 4 silently fails. The mkdir -p is what makes re-push +// overwrite semantics work. +func TestStreamLayout_RemoteCommand(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{} + _ = StreamLayout(context.Background(), fe, "tracebloc", "pod-x", "stage", + layout, "my_table", NoOpProgress{}) + + if got, want := len(fe.gotCmd), 3; got != want { + t.Fatalf("len(cmd) = %d, want %d", got, want) + } + if fe.gotCmd[0] != "/bin/sh" || fe.gotCmd[1] != "-c" { + t.Errorf("cmd[:2] = %v, want [/bin/sh -c]", fe.gotCmd[:2]) + } + script := fe.gotCmd[2] + // The remote command must be: + // 1. HERMETIC (Bugbot r5): old files don't survive a re-push + // 2. TRANSACTIONAL (Bugbot r7): tar failure preserves prev + // dataset + // 3. RACE-SAFE under parallel-push (Bugbot r8): unique + // .staging- suffix per invocation so concurrent pushes + // don't interleave each other's tar extraction + // + // Pattern: extract to .staging-, then on tar SUCCESS + // rm the old dest + mv staging → dest. The order matters: + // + // - tar BEFORE any rm of $DEST (preserves on tar failure) + // - mv AFTER tar succeeds (atomic-ish swap) + // + // Extract the staging path with a regex so the random hex + // suffix doesn't pin us to a specific invocation's bytes. + stagingRE := regexp.MustCompile(`/data/shared/my_table\.staging-[0-9a-f]{8}`) + stagingPaths := stagingRE.FindAllString(script, -1) + if len(stagingPaths) == 0 { + t.Fatalf("remote script has no /data/shared/my_table.staging-<8hex> path (race-safety regression):\n%s", script) + } + // All staging mentions must refer to the SAME suffix in a single + // invocation. If we see two distinct suffixes that's a bug: + // rm/mkdir/tar/mv would target different dirs. + for i := 1; i < len(stagingPaths); i++ { + if stagingPaths[i] != stagingPaths[0] { + t.Errorf("remote script mixes staging suffixes %q and %q:\n%s", + stagingPaths[0], stagingPaths[i], script) + } + } + staging := stagingPaths[0] + + // Pin the four key operations. + for _, want := range []string{ + `mkdir -p "` + staging + `"`, + `tar -xf - -C "` + staging + `"`, + `mv "` + staging + `" "/data/shared/my_table"`, + } { + if !strings.Contains(script, want) { + t.Errorf("remote script missing %q: %s", want, script) + } + } + // Transactional property: tar must come BEFORE any mutation + // of the real destination. r5+r7 used `rm $DEST` for the + // pre-mv step; r10 swapped that to `mv $DEST $DEST.old-` + // (backup-and-swap, so a mv failure can be rolled back). + // The contract is the same: tar runs while $DEST is intact. + tarIdx := strings.Index(script, `tar -xf - -C "`+staging+`"`) + destBackupMvIdx := strings.Index(script, `mv "/data/shared/my_table" "`) + if tarIdx < 0 || destBackupMvIdx < 0 { + t.Fatalf("remote script missing tar or destination-backup mv: %s", script) + } + if tarIdx >= destBackupMvIdx { + t.Errorf("remote script touches $DEST BEFORE tar succeeds — partial-transfer could destroy previous data:\n%s", script) + } + // Single-segment guarantee: both rm targets must end with + // /my_table or /my_table.staging-* — never just /data/shared. + if strings.Contains(script, `rm -rf "/data/shared"`) || + strings.Contains(script, `rm -rf "/data/shared/"`) { + t.Errorf("remote script rm-rfs the parent /data/shared (would nuke sibling tables):\n%s", script) + } + + // Bugbot r9 + r10: orphan cleanup for previously-failed pushes + // must catch BOTH .staging-* and .old-* siblings. The .old-* + // dirs are created by the round-10 backup-and-rollback swap; + // without including them in the find pattern, failed-mid-swap + // pushes would leak .old-* dirs. + for _, wantName := range []string{`-name "my_table.staging-*"`, `-name "my_table.old-*"`} { + if !strings.Contains(script, wantName) { + t.Errorf("remote script missing %s in orphan cleanup:\n%s", wantName, script) + } + } + if !strings.Contains(script, `-mmin +60 -exec rm -rf {} +`) { + t.Errorf("remote script missing -mmin+60 orphan window:\n%s", script) + } + + // Bugbot r10: backup-and-swap pattern. The previous dataset + // must be backed up to .old- BEFORE the new dataset + // arrives, and restored if the main mv fails. Pin the key + // shape pieces — backup mv, primary mv, rollback mv, cleanup. + backupRE := regexp.MustCompile(`/data/shared/my_table\.old-[0-9a-f]{8}`) + backupPaths := backupRE.FindAllString(script, -1) + if len(backupPaths) == 0 { + t.Fatalf("remote script has no .old- backup path (r10 rollback regression):\n%s", script) + } + // All .old- mentions in a single invocation must agree — + // otherwise the rollback would target a different path than + // the backup created. + for i := 1; i < len(backupPaths); i++ { + if backupPaths[i] != backupPaths[0] { + t.Errorf("remote script mixes backup suffixes %q vs %q:\n%s", + backupPaths[0], backupPaths[i], script) + } + } + backup := backupPaths[0] + // Backup must use the SAME suffix as staging — different + // suffixes would defeat the "find -name ...staging-* ...old-*" + // orphan-cleanup symmetry, AND would risk collision with a + // concurrent push's .old-. + if strings.TrimPrefix(backup, "/data/shared/my_table.old-") != + strings.TrimPrefix(staging, "/data/shared/my_table.staging-") { + t.Errorf("backup and staging suffixes diverge: %q vs %q", backup, staging) + } + // Backup mv (DEST → .old) must appear BEFORE primary mv + // (.staging → DEST), or rollback wouldn't have anything to + // restore. + backupMvIdx := strings.Index(script, `mv "/data/shared/my_table" "`+backup+`"`) + primaryMvIdx := strings.Index(script, `mv "`+staging+`" "/data/shared/my_table"`) + if backupMvIdx < 0 || primaryMvIdx < 0 { + t.Fatalf("remote script missing backup or primary mv:\n%s", script) + } + if backupMvIdx >= primaryMvIdx { + t.Errorf("backup mv (DEST→.old) must come BEFORE primary mv (.staging→DEST); rollback contract broken:\n%s", script) + } +} + +// TestStreamLayout_StagingSuffixIsUniquePerInvocation: two back-to- +// back StreamLayout calls for the same table MUST produce different +// staging-dir suffixes. Without this, concurrent `dataset push` runs +// for the same table would race on the same .staging path and +// corrupt each other's tar extraction. Bugbot r8 fix. +func TestStreamLayout_StagingSuffixIsUniquePerInvocation(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + stagingRE := regexp.MustCompile(`/data/shared/t\.staging-[0-9a-f]{8}`) + + collect := func() string { + fe := &fakeExecutor{} + if err := StreamLayout(context.Background(), fe, "tracebloc", "p", "stage", + layout, "t", NoOpProgress{}); err != nil { + t.Fatalf("StreamLayout: %v", err) + } + m := stagingRE.FindString(fe.gotCmd[2]) + if m == "" { + t.Fatalf("no staging path in: %s", fe.gotCmd[2]) + } + return m + } + a, b := collect(), collect() + if a == b { + t.Errorf("back-to-back StreamLayout produced identical staging path %q; race-safety regression", a) + } +} + +// TestStreamLayout_TargetingFromArgs pins that namespace + pod name +// flow through to the executor unchanged. Catches a refactor that +// hardcodes the namespace or builds the pod name in the wrong layer. +func TestStreamLayout_TargetingFromArgs(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{} + _ = StreamLayout(context.Background(), fe, "custom-ns", "weird-pod-name", "container-x", + layout, "t", NoOpProgress{}) + if fe.gotNS != "custom-ns" { + t.Errorf("namespace = %q, want custom-ns", fe.gotNS) + } + if fe.gotPod != "weird-pod-name" { + t.Errorf("pod = %q, want weird-pod-name", fe.gotPod) + } + if fe.gotContainer != "container-x" { + t.Errorf("container = %q, want container-x", fe.gotContainer) + } +} + +// TestStreamLayout_ExecErrorWithRemoteStderr: when the in-cluster +// tar fails (e.g. read-only FS, no-space), the remote stderr must +// surface in the customer-facing error. This is the actionable +// signal — without it, customers see "exec stream failed" with no +// way to diagnose what went wrong server-side. +func TestStreamLayout_ExecErrorWithRemoteStderr(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{ + stderrToReturn: []byte("tar: write error: No space left on device"), + errToReturn: errors.New("command terminated with exit code 1"), + drainBeforeReturn: true, + } + err = StreamLayout(context.Background(), fe, "tracebloc", "p", "stage", + layout, "t", NoOpProgress{}) + if err == nil { + t.Fatal("StreamLayout returned nil on exec error") + } + for _, want := range []string{"streaming files", "No space left on device", "exit code 1"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %v", want, err) + } + } +} + +// TestStreamLayout_ProgressBytes: the Progress sink receives byte +// counts during the stream. This is the contract the schollz bar +// keys off — if the wrapper ever stops calling Add(), the bar would +// freeze at 0%. +func TestStreamLayout_ProgressBytes(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fp := &fakeProgress{} + err = StreamLayout(context.Background(), &fakeExecutor{}, "tracebloc", "p", "stage", + layout, "t", fp) + if err != nil { + t.Fatalf("StreamLayout: %v", err) + } + // At minimum we expect the file bodies (200 + 39 = 239) plus + // tar headers (~512 per file = ~1536 bytes overhead minimum). + // Assert "some bytes flowed" rather than an exact count + // because the tar overhead is implementation-defined. + if fp.added < 239 { + t.Errorf("progress.Add total = %d, want at least 239 (file body bytes)", fp.added) + } + if !fp.finished { + t.Errorf("progress.Finish() never called") + } +} + +// TestWriteLayoutTar_LabelsCSVFirst: ordering matters for fail-fast +// diagnostic — labels.csv at the start means a corrupt tar is more +// likely to surface before we've shipped a hundred image bytes. +func TestWriteLayoutTar_LabelsCSVFirst(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + var buf bytes.Buffer + if err := writeLayoutTar(&buf, layout); err != nil { + t.Fatalf("writeLayoutTar: %v", err) + } + tr := tar.NewReader(&buf) + hdr, err := tr.Next() + if err != nil { + t.Fatalf("reading first entry: %v", err) + } + if hdr.Name != "labels.csv" { + t.Errorf("first entry name = %q, want labels.csv (ordering pins this)", hdr.Name) + } +} + +// fakeProgress: a Progress that records total bytes added and +// whether Finish was called. +type fakeProgress struct { + added int + finished bool +} + +func (p *fakeProgress) Add(n int) { p.added += n } +func (p *fakeProgress) Finish() { p.finished = true } diff --git a/internal/push/walk.go b/internal/push/walk.go index dd95571..bb7b15f 100644 --- a/internal/push/walk.go +++ b/internal/push/walk.go @@ -106,9 +106,17 @@ func Discover(rootDir string) (*LocalLayout, error) { layout := &LocalLayout{Root: abs} - // labels.csv (required). + // labels.csv (required). Use Lstat — NOT Stat — so a symlink + // shows up as a symlink (mode includes ModeSymlink) rather + // than being silently followed. v0.1 rejects symlinks entirely + // (see rejectSymlink); without Lstat the size cap below would + // see the symlink's own ~100-byte size while writeTarFile + // (which uses os.Stat → follows symlinks) would happily stream + // the target's full contents — a size-cap bypass and an + // arbitrary-local-file disclosure to the cluster PVC. Bugbot + // flagged this as Medium-severity security on PR-b round 4. labelsPath := filepath.Join(abs, "labels.csv") - labelsStat, err := os.Stat(labelsPath) + labelsStat, err := os.Lstat(labelsPath) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf( @@ -120,6 +128,9 @@ func Discover(rootDir string) (*LocalLayout, error) { } return nil, fmt.Errorf("stat labels.csv: %w", err) } + if err := rejectSymlink(labelsStat, "labels.csv"); err != nil { + return nil, err + } if labelsStat.IsDir() { // A directory literally named "labels.csv" passes the // os.Stat above — without this check the pre-flight would @@ -138,9 +149,14 @@ func Discover(rootDir string) (*LocalLayout, error) { layout.TotalBytes += labelsStat.Size() // images/ subdir (required, must contain at least one - // image-extension file). + // image-extension file). Use os.Lstat — NOT Stat — so a + // symlinked-directory case (e.g. `ln -s /etc images`) gets + // caught here, not silently followed into the linked path. + // Without Lstat, the symlink-image fixes from the previous + // commit don't matter: the whole directory could be a link. + // Bugbot flagged the missing dir-level Lstat on PR-b round 5. imagesDir := filepath.Join(abs, "images") - imagesStat, err := os.Stat(imagesDir) + imagesStat, err := os.Lstat(imagesDir) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, fmt.Errorf( @@ -150,6 +166,9 @@ func Discover(rootDir string) (*LocalLayout, error) { } return nil, fmt.Errorf("stat images/: %w", err) } + if err := rejectSymlink(imagesStat, "images"); err != nil { + return nil, err + } if !imagesStat.IsDir() { return nil, fmt.Errorf("%q exists but is not a directory", imagesDir) } @@ -175,10 +194,19 @@ func Discover(rootDir string) (*LocalLayout, error) { if _, ok := imageExtensions[ext]; !ok { continue } + // entry.Info() returns Lstat-like metadata for the + // directory entry (the symlink's own mode if it's a + // symlink, not the target's). That's exactly what we + // want here — combined with rejectSymlink it closes the + // symlink-bypass-size-caps hole Bugbot flagged on PR-b + // round 4. info, err := entry.Info() if err != nil { return nil, fmt.Errorf("stat %q: %w", entry.Name(), err) } + if err := rejectSymlink(info, filepath.Join("images", entry.Name())); err != nil { + return nil, err + } if info.Size() > MaxSingleFileBytes { return nil, sizeError(filepath.Join("images", entry.Name()), info.Size(), MaxSingleFileBytes) @@ -207,6 +235,54 @@ func Discover(rootDir string) (*LocalLayout, error) { return layout, nil } +// rejectSymlink returns a non-nil error if info describes a symlink. +// v0.1 refuses symlinks under /{labels.csv, images, images/*} +// entirely because: +// +// - SECURITY: writeTarFile uses os.Open (which follows symlinks). +// Discover sized entries via DirEntry.Info() (which does not). +// A symlink whose target is a multi-GB local file would pass +// Discover's size cap (the symlink itself is ~100 bytes) yet +// stream the target's full contents to the cluster PVC — a +// size-cap bypass and arbitrary-local-file disclosure to the +// cluster admin. Bugbot caught this on PR-b round 4. +// - UX: legitimate image_classification datasets don't use +// symlinks. A clear "symlinks not supported" error is better +// than the alternative fixes (resolve + re-stat the target, +// blanket Stat() everywhere) — both of which would let the +// security hole creep back in on a future refactor. +// +// Customers with symlink-heavy layouts (rare; usually means their +// data lives on another filesystem) can `cp -L` to materialize +// the files before pushing. +// +// Known limitation: HARD LINKS are NOT rejected. The filesystem +// doesn't expose a Mode bit for hard links the way ModeSymlink +// distinguishes symlinks, and a high-Nlink check has too many +// false positives (legitimate hard-linked datasets where the +// dataset dir is the only entry point). The implicit security +// boundary is the CLI's process-level read permissions: a +// customer can only hard-link files they already have read +// access to, so the cluster admin reading a hard-linked +// /etc/shadow via the PVC isn't a privilege escalation — they'd +// need the CLI to be running as root for that to be exploitable, +// and the docs already recommend running as a low-privileged +// user. v0.2 may add openat(O_NOFOLLOW)-based sandboxing if a +// real customer needs harder isolation; tracked alongside the +// cloud-source story (#147 non-goals). +func rejectSymlink(info os.FileInfo, relPath string) error { + if info.Mode()&os.ModeSymlink == 0 { + return nil + } + return fmt.Errorf( + "%q is a symbolic link, which v0.1 does not allow in the dataset "+ + "layout (security: a symlink could escape the dataset tree or "+ + "bypass size caps). Materialize the link target (e.g. `cp -L`) "+ + "and re-run, or wait for v0.2's cloud-source story if the data "+ + "lives elsewhere.", + relPath) +} + // sizeError builds the over-the-single-file-cap error with the same // human-readable framing as the total-cap branch above. Centralized // so the message stays consistent if we tune the wording later. diff --git a/internal/push/walk_test.go b/internal/push/walk_test.go index 8095415..441400c 100644 --- a/internal/push/walk_test.go +++ b/internal/push/walk_test.go @@ -3,6 +3,7 @@ package push import ( "os" "path/filepath" + "runtime" "strings" "testing" ) @@ -191,6 +192,124 @@ func TestDiscover_LabelsCSVIsDirectory(t *testing.T) { } } +// TestDiscover_RejectsSymlinkedImage is the security regression pin +// for the Bugbot-Medium finding on PR-b round 4. A symlink in +// images/ whose target points outside the dataset tree (or at a +// big local file) used to pass Discover's size cap (because +// DirEntry.Info() returns the link's own ~100-byte size) yet +// stream the target's full contents into the cluster PVC via +// writeTarFile's os.Stat+os.Open path. That's both a size-cap +// bypass AND an arbitrary-local-file disclosure to the cluster +// admin. v0.1 rejects symlinks outright. +func TestDiscover_RejectsSymlinkedImage(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require admin privileges on Windows") + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + // Create a real image file outside the dataset tree to be the + // symlink target. The symlink inside images/ points at it. + target := filepath.Join(t.TempDir(), "outside.jpg") + if err := os.WriteFile(target, make([]byte, 100), 0o644); err != nil { + t.Fatalf("write outside target: %v", err) + } + link := filepath.Join(imagesDir, "evil.jpg") + if err := os.Symlink(target, link); err != nil { + t.Fatalf("symlink: %v", err) + } + + _, err := Discover(root) + if err == nil { + t.Fatal("Discover accepted a symlinked image — security regression; v0.1 must reject") + } + for _, want := range []string{"symbolic link", "security", "cp -L"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q in: %v", want, err) + } + } +} + +// TestDiscover_RejectsSymlinkedImagesDir is the round-5 follow-up +// to the symlinked-FILE fix from round 4. The DIRECTORY itself +// could also be a symlink (e.g. `ln -s /etc images`), which the +// previous fix didn't catch because it only Lstat'd the entries +// INSIDE images/. Bugbot flagged it; this test pins the +// dir-level Lstat. +func TestDiscover_RejectsSymlinkedImagesDir(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require admin privileges on Windows") + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + // Create a real images dir somewhere ELSE that the symlink + // will point at. The symlink in `root` is what should be + // rejected. + realDir := filepath.Join(t.TempDir(), "real-images") + if err := os.MkdirAll(realDir, 0o755); err != nil { + t.Fatalf("mkdir real-images: %v", err) + } + if err := os.WriteFile(filepath.Join(realDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write img: %v", err) + } + if err := os.Symlink(realDir, filepath.Join(root, "images")); err != nil { + t.Fatalf("symlink images: %v", err) + } + + _, err := Discover(root) + if err == nil { + t.Fatal("Discover accepted a symlinked images/ directory — security regression") + } + if !strings.Contains(err.Error(), "symbolic link") { + t.Errorf("error missing symbolic-link rejection: %v", err) + } +} + +// TestDiscover_RejectsSymlinkedLabelsCSV: same hole, different +// file. The labels.csv path can also be a symlink (e.g. pointing +// at a huge local file). Lstat + reject covers both cases +// symmetrically. +func TestDiscover_RejectsSymlinkedLabelsCSV(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require admin privileges on Windows") + } + root := t.TempDir() + target := filepath.Join(t.TempDir(), "outside-labels.csv") + if err := os.WriteFile(target, []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write outside target: %v", err) + } + link := filepath.Join(root, "labels.csv") + if err := os.Symlink(target, link); err != nil { + t.Fatalf("symlink: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write img: %v", err) + } + + _, err := Discover(root) + if err == nil { + t.Fatal("Discover accepted a symlinked labels.csv — security regression") + } + if !strings.Contains(err.Error(), "symbolic link") { + t.Errorf("error missing symbolic-link rejection: %v", err) + } +} + func TestDiscover_NotADirectory(t *testing.T) { // Customer passes a path to a single file instead of a dir. // This is a common autocomplete-mistake (tab-completing From 4b25e6937b266073d71bdfe27d42ab1c5ccabfe4 Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Fri, 22 May 2026 22:06:45 +0500 Subject: [PATCH 05/17] =?UTF-8?q?feat(dataset):=20Phase=204=20=E2=80=94=20?= =?UTF-8?q?submit=20to=20jobs-manager=20+=20watch=20+=20summary=20(#152)?= =?UTF-8?q?=20(#10)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(dataset): Phase 4 — submit to jobs-manager + watch + summary (#152) Closes the v0.1 happy path. After Phase 3 stages files on the PVC, Phase 4 POSTs the ingest spec to jobs-manager, watches the spawned ingestor Job, and renders the parsed INGESTION SUMMARY panel. Customer sees a single end-to-end flow: $ tracebloc dataset push ./cats-dogs --table=cats_dogs_train ... Local dataset: ... Target cluster: ... Synthesized ingest spec: ... Created stage Pod tracebloc/tracebloc-stage-... Streaming N files (X MiB) to /data/shared/cats_dogs_train/... Staged N files to /data/shared/cats_dogs_train Submitted: jobs-manager spawned ingestor Job tracebloc/ingestor-... Streaming logs from Job tracebloc/ingestor-...: [ingestor's own log output streams here, verbatim] ┌─ Ingestion summary ──────────────────────────┐ │ Ingestor ID: run-abc │ │ Total records: 1,234 │ │ Inserted: 1,200 │ │ ... etc │ └──────────────────────────────────────────────┘ ## What lands internal/submit/ (5 new files): - body.go SubmitRequest struct (ingest_config + idempotency_key + optional image_digest); BuildRequest auto-generates a 16-byte hex idempotency key per invocation unless --idempotency-key overrides. - client.go HTTPSubmitter with bearer-token auth; 4xx/5xx surface jobs-manager's body verbatim via SubmitError; IsAuthError(err) for the 401/403 exit-code branch. - watch.go Poll-based watch (per the design discussion): wait for the Job's Pod to exist + Running, stream logs via GetLogs(Follow=true), terminal poll on Job conditions to distinguish Succeeded/Failed/Unknown. - summary.go Streaming parser for the 📊 INGESTION SUMMARY banner. Strips ANSI codes, accumulates fields by prefix match, finalizes on the closing ═-rule. RenderPanel formats the parsed Summary as a box-drawn panel. - submit.go Run() orchestrator: BuildRequest → POST → announce (Spawned vs Replayed) → either --detach exit or watch loop. SIGINT during watch produces Outcome=Detached with a kubectl-logs reconnect hint. internal/cli/dataset.go: wired after push.Stage. Mints a 1-hour SA token (vs cluster info's 10 min — the watch loop can run that long). Adds --detach, --idempotency-key, --image-digest flags. Maps Submit + Watch outcomes to exit codes 5/8/9. ## Design choices (per pre-build discussion) - Poll every 2s for Job status (not client-go Watch). Matches the Phase 3 WaitForStagePodReady pattern; simpler test surface, no long-lived connection to babysit. ~30 API calls/min is negligible. - SIGINT-after-201 → graceful detach. The cluster has already accepted the run; Ctrl-C just stops watching. Customer sees the kubectl logs reconnect hint and exit 0. Matches kubectl logs behavior. ## Exit codes (full set, v0.1) 0 — success (or --detach success: 201 came back clean) 2 — schema / v0.1-unsupported-category 3 — local-layout or kubeconfig 4 — cluster discovery (parent release or shared PVC missing) 5 — SA token couldn't be obtained, OR jobs-manager 401/403 7 — Phase 3 stage step failed 8 — jobs-manager 4xx/5xx other than auth 9 — ingestion Job exited non-zero, OR summary reports row failures ## Tests submit/ ≥85% line coverage across 5 files. Per-file: - body_test.go idempotency-key generation, override, uniqueness, JSON shape pinning (image_digest omitempty) - client_test.go httptest.Server: 201 happy path, replay path, 4xx body framing, 401/403 → IsAuthError, 5xx ≠ auth, missing-job_name, network drop, ctx-cancel - summary_test.go Real ingestor banner end-to-end, byte-by-byte streamed feed, no-banner path (early failure), post-banner lines ignored, ANSI stripping, panel rendering, comma formatting - watch_test.go Fake clientset: Running Pod surfaces, fast- completion path, Forbidden short-circuits, no-Pod timeout, Job Conditions to outcome, parserWriter contract - submit_test.go Run orchestrator: --detach happy path, replay, SubmitError propagation, request fields plumb through, nil-Out defaults to Discard, IsAuthError Local: vet, test -race -cover, gofmt -s, errcheck — all green. Phase 5 (#153: GitHub Releases, Homebrew tap, install.sh) is the last item on the v0.1 epic. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(submit): enforce JobWatchTimeout + flush parser + typed watch errors + Unknown exit code Four Bugbot findings on PR #10's initial commit, all real: ## Medium — JobWatchTimeout was declared but never enforced The constant existed and documented "absolute cap on a single dataset-push's watch phase," but WatchJob never used it. Per-step timeouts (PodReadyTimeout, finalJobStatus's 30s) bounded sub-loops, but the log-stream phase ran indefinitely against ctx.Done() with no overall cap. A stuck ingestor would hang the CLI forever. Fix: context.WithTimeout(ctx, JobWatchTimeout) at the top of WatchJob. Cancel via defer. Cap fires as ctx.DeadlineExceeded through the inner loops. ## Medium — Watch failures used the submit exit code submit.Run wrapped both POST errors AND watch errors in the same return path. cli/dataset.go's switch mapped IsAuthError → 5 and "everything else" → 8 (submit-failed). But waitForJobPod / streamPodLogsAndParse / finalJobStatus failures aren't submit failures — jobs-manager already accepted the run, the cluster is doing the work, the CLI just lost the ability to follow along. Exit 8 misled customers into thinking the submit was rejected. Fix: introduce a WatchError type wrapping watch-phase errors; add IsWatchError predicate. cli/dataset.go now routes: IsAuthError → 5 (token / auth) IsWatchError → 9 (ingest-side; cluster keeps running) else → 8 (submit-side rejection) Test TestIsWatchError pins the typing including wrapped errors. ## Low — Log parser didn't flush partial last lines streamPodLogsAndParse used bufio.Scanner for line-based stream display, but the SummaryParser's own internal buffer (for chunked-write line assembly) was never flushed at end-of-stream. If the Pod terminated without a trailing '\n' on its final stdout write — rare but possible if the container's stdout was killed mid-write — the final line would be lost, potentially including the closing ═-rule that finalizes the banner. Fix: parser.FlushLine() at end-of-stream AND on non-EOF scanner errors. ## Medium — Outcome=Unknown exited zero The exit-code switch only handled Failed + Succeeded. Outcome= Unknown (returned by finalJobStatus when the 30s post-stream poll doesn't see a terminal condition) fell through to the default `return nil` → exit 0. But "we don't know if it worked" is not success. Fix: add an explicit Unknown branch that exits 9 with a diagnostic pointing the customer at `kubectl get job ...` to check the actual outcome. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(submit): zero-metric finalize + namespace check + fresh ctx for final status Round 2 of Bugbot fixes on PR #10. Four findings, all real — three are second-order effects of the r1 fixes: ## Medium — Zero-metric banner never finalized The hasParsedAnyField check (which gates the closing ═-rule's finalization) required ANY metric counter to be > 0. But an ingestion that ran-then-died-early can legitimately produce a banner where ALL counts are 0 — the ingestor still prints the structure, just with zeroes. The parser would never finalize on that case, leaving the parser open to perturbation from post-banner shutdown logs. Fix: track a dedicated sawAnyField bool that latches on ANY successful field-line parse, regardless of the value. The finalization check now keys off "did we see any banner field" instead of "did we see a non-zero one." ## Medium — Missing submit response namespace not rejected client.go rejected 2xx responses with no job_name but happily accepted responses with no namespace. A malformed-but-parsing response would then drive the watch loop's k8s API calls at the empty namespace, plus produce a broken `kubectl logs` reconnect hint. Fix: parallel check for empty Namespace, same error shape as the job_name check. ## Medium — Watch timeout starved finalJobStatus R1 wrapped the WHOLE WatchJob in ctx.WithTimeout(JobWatchTimeout) to enforce the 1-hour cap. Side effect: finalJobStatus (which runs AFTER log streaming) inherited the SAME ctx — so a 59-minute log stream left finalJobStatus with 1 minute of budget, and a 60-minute stream left it with 0. Successful slow ingestions misreported as Unknown → exit 9. Fix: split the ctx hierarchy. - customerCtx — original input, carries SIGINT cancellation - watchCtx = WithTimeout(customerCtx, JobWatchTimeout) — caps the pod-wait + log-stream phases - finalCtx = WithTimeout(customerCtx, 30s) — fresh budget derived from customerCtx, not from watchCtx The customerCtx-vs-watchCtx distinction also fixes the detach detection: check customerCtx.Err() not ctx.Err() so a JobWatchTimeout expiry doesn't masquerade as a SIGINT-Detach. ## Medium — SIGINT after logs yielded exit 9 After log streaming ended cleanly, Ctrl-C during the brief finalJobStatus poll returned a watch error → exit 9. But the contract is "post-201 SIGINT = graceful detach, exit 0" — the cluster already has the work, the customer is just done watching. Fix: if customerCtx is Canceled inside the finalJobStatus error branch, return Outcome=Detached with the same reconnect-hint path the log-stream-cancel case uses. Consistent detach semantics across both watch sub-phases. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(submit): port-forward to jobs-manager from off-cluster (Bugbot r3) Major architectural finding from Bugbot's third PR #10 review: the CLI runs on the customer's laptop (or off-cluster CI), but the discovered JobsManagerService URL is a *.svc.cluster.local name that the laptop can't resolve. The submit POST would always fail. The original Phase 4 design quietly assumed in-cluster network reachability — wrong for the dominant v0.1 use case. The chart's existing flow doesn't have this problem because the helm hook runs as a Pod IN the cluster. ## Fix: in-process port-forward, same mechanism as kubectl pf New internal/submit/portforward.go: - PortForwardJobsManager opens an SPDY tunnel via the customer's kubeconfig-authenticated apiserver connection. - pickServicePod resolves the jobs-manager Service to a Running Pod backing it (Services aren't directly port-forwardable; client-go's portforward API speaks Pod names). - ForwardedConnection.Close tears it down on defer; idempotent so the orchestrator's defer-Close + any inner cleanup don't risk a double-close panic. Wiring in cli/dataset.go: - After token mint, open the port-forward to JobsManagerServiceName:JobsManagerPort (8080 today). - Defer Close — runs on every path, including the SIGINT-detach + error paths from submit.Run. - HTTPSubmitter targets http://localhost:; the in-cluster URL is no longer used at all from the CLI. cluster/discover.go: expose the bare Service name + port alongside the existing FQDN URL. Phase 2 still constructs the FQDN for the diagnostic output of `cluster info`, but Phase 4 needs the unqualified pieces for the port-forward setup. ## Tests internal/submit/portforward_test.go covers the pickServicePod resolution (5 cases: happy path, skip-non-Running, no-matching- Pod, missing-Service, selector-less). The full port-forward loop needs a real apiserver — covered by EKS smoke. internal/cluster/discover_test.go: updated the struct-literal comparison in TestDiscoverParentRelease_HappyPath to include the two new fields. Local: vet, test -race -cover, gofmt -s, errcheck — all green. ## What we got wrong in the initial design Designing Phase 4, I read the chart's existing flow (which runs in-cluster) + assumed the CLI could use the same URL. That's exactly the "cargo-cult the URL" mistake. The right framing would have been: "Phase 4 needs to reach jobs-manager from the laptop — what mechanism does kubectl have for that?" → port- forward. The fix is correct + well-tested where unit-testable. The full loop will get exercised on the EKS smoke test. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(submit): pick most-recent useful-phase Pod, not items[0] (Bugbot r4) Bugbot's fourth PR #10 finding: waitForJobPod picked pods.Items[0] from an unordered List API response. A Job with backoffLimit > 0 (or any scenario where jobs-manager re-spawned the Pod for retry, node drain, etc.) can have multiple Pods bearing the same `job-name=` label. Picking items[0] could grab the old Failed Pod from a prior retry while the current Running one waits — the CLI then streams stale logs and reports the prior-attempt outcome. Fix: iterate all Pods, prefer the most recent (by creationTimestamp) in a useful phase (Running | Succeeded | Failed). Pending Pods don't count — no log stream yet, so we keep polling rather than "return a Pending pod's name." Behaviorally: - Single Pod (common): same as before — that one Pod returns. - Old Failed + new Running: returns the Running. Correct. - Old Failed + new Pending: returns nothing yet; keeps polling until Pending → Running. Correct. - Multiple Running (parallelism > 1): returns the most recent — defensible choice; either would work for log streaming. Two new tests: - TestWaitForJobPod_PicksMostRecentNotFirst: seeds an older Failed + newer Running; asserts the Running's name returns. - TestWaitForJobPod_AllPendingKeepsPolling: seeds an only- Pending Pod; asserts DeadlineExceeded on a tight ctx (no early return with the Pending name). Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(submit): Pod-wait timeout = Detached (not failed); non-blocking errCh send Round 5 Bugbot findings on PR #10: ## Medium — Pod-wait timeout reported ingest failure waitForJobPod times out after PodReadyTimeout (5min) if the ingestor Pod doesn't reach Running — slow image pull, scheduling backlog, PSA still rejecting. The previous flow wrapped that as WatchError → exit 9 ("ingestion failed"). That's a false positive: jobs-manager already accepted the submit, the cluster will run the ingestion (eventually), the CLI just gave up observing within the timeout. Fix: distinguish errors.Is(err, context.DeadlineExceeded) from waitForJobPod and map to Outcome=Detached (same semantic as SIGINT-mid-watch). Customer sees the kubectl-logs reconnect hint and exits 0. The submit was successful; the ingestion runs. New test TestWatchJob_PodWaitTimeoutMapsToDetach pins the contract via a tight outer ctx that triggers DeadlineExceeded through the same code path. ## Medium — Defensive non-blocking errCh send in portforward.go Bugbot's read: "unbuffered handoff can block the goroutine indefinitely." False on the literal code (errCh is make(chan error, 1), buffered) — but the spirit of the concern is reasonable: a future refactor that adds a second send path, or changes the buffer size, could re-introduce the leak. Fix: keep the cap-1 buffer AND wrap the send in a non-blocking select with default. Two layers of safety, so the goroutine never blocks regardless of how the channel is used downstream. Closes out the finding structurally rather than relying on the code reader to spot that the buffer was already sized correctly. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(submit): JobWatchTimeout-during-stream = Detached, not exit 9 Bugbot PR #10 r6 caught the inconsistency r5 introduced: the PodReadyTimeout path was mapped to Detached (commit 0b02cd4), but the SAME JobWatchTimeout constant capping the log-stream phase still mapped to exit 9 ("ingestion failed") if it expired during streaming. Both should mean the same thing: "the cluster keeps running; the CLI just gave up observing." Fix: accept context.DeadlineExceeded as well as context.Canceled in the post-stream branch. If watchCtx (the 1-hour-capped one) expired but customerCtx (the SIGINT-aware parent) didn't, that's the JobWatchTimeout-hit case — map to Detached, same UX as the SIGINT path. Customer sees the kubectl-logs reconnect hint and exits 0. The watchCtx-vs-customerCtx distinction (from r3) is what lets us tell SIGINT from JobWatchTimeout at this branch point — without it, the only way to detect timeout-vs-cancel would be the inner error. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(submit): per-reason detach message (Bugbot r7) R5 + r6 expanded "Detached" to cover PodReadyTimeout and JobWatchTimeout in addition to the original SIGINT case, but the orchestrator's diagnostic still said "Detached on signal" for all three. Customers who hit the 5-min Pod-ready cap or 1-hour watch cap aren't pressing Ctrl-C; the framing was misleading. Fix: WatchResult gains a DetachReason field (DetachReasonSignal, DetachReasonPodWaitTimeout, DetachReasonWatchCap, or None). Every detach branch in WatchJob sets the appropriate reason. submit.Run's print-detach block now switches on the reason: Signal → "Detached on signal." PodWaitTimeout → "Detached: the ingestor Pod didn't reach Ready within the observation window..." WatchCap → "Detached: the watch window (1 hour) elapsed while the ingestion was still running." Common reconnect hint (`kubectl logs -f`) follows in all cases. The customerCtx-vs-watchCtx distinction (introduced in r3) makes the reason classification trivially correct: customerCtx.Canceled takes precedence over watchCtx.DeadlineExceeded if both fired, so a customer hitting Ctrl-C exactly at the 1-hour cap still gets the "on signal" message. Local: vet, test -race -cover, gofmt -s, errcheck — all green. Co-Authored-By: Claude Opus 4.7 (1M context) --------- Co-authored-by: Claude Opus 4.7 (1M context) --- internal/cli/dataset.go | 142 +++++++- internal/cluster/discover.go | 18 +- internal/cluster/discover_test.go | 14 +- internal/submit/body.go | 107 ++++++ internal/submit/body_test.go | 100 ++++++ internal/submit/client.go | 194 +++++++++++ internal/submit/client_test.go | 234 +++++++++++++ internal/submit/portforward.go | 238 ++++++++++++++ internal/submit/portforward_test.go | 142 ++++++++ internal/submit/submit.go | 206 ++++++++++++ internal/submit/submit_test.go | 231 +++++++++++++ internal/submit/summary.go | 388 ++++++++++++++++++++++ internal/submit/summary_test.go | 240 ++++++++++++++ internal/submit/watch.go | 491 ++++++++++++++++++++++++++++ internal/submit/watch_test.go | 299 +++++++++++++++++ 15 files changed, 3027 insertions(+), 17 deletions(-) create mode 100644 internal/submit/body.go create mode 100644 internal/submit/body_test.go create mode 100644 internal/submit/client.go create mode 100644 internal/submit/client_test.go create mode 100644 internal/submit/portforward.go create mode 100644 internal/submit/portforward_test.go create mode 100644 internal/submit/submit.go create mode 100644 internal/submit/submit_test.go create mode 100644 internal/submit/summary.go create mode 100644 internal/submit/summary_test.go create mode 100644 internal/submit/watch.go create mode 100644 internal/submit/watch_test.go diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go index 6034e03..5e76e0f 100644 --- a/internal/cli/dataset.go +++ b/internal/cli/dataset.go @@ -12,6 +12,7 @@ import ( "github.com/tracebloc/cli/internal/cluster" "github.com/tracebloc/cli/internal/push" "github.com/tracebloc/cli/internal/schema" + "github.com/tracebloc/cli/internal/submit" ) // newDatasetCmd wires the `tracebloc dataset` subtree. The dominant @@ -89,6 +90,16 @@ func newDatasetPushCmd() *cobra.Command { // Pin by digest in your override too — tag-only references // drift silently and break "all my pushes worked yesterday." stagePodImage string + + // Phase 4 flags. --detach exits immediately after the 201 + // from jobs-manager; --idempotency-key plumbs through to + // the submit body for retry-safety across CLI invocations + // (default: fresh per call); --image-digest pins the + // ingestor image (default: jobs-manager picks the + // cluster-configured one). + detach bool + idempotencyKey string + imageDigest string ) cmd := &cobra.Command{ @@ -113,13 +124,18 @@ datasets need the v0.2 cloud-source story (S3/GCS/HTTPS sources) — see tracebloc/client#147 non-goals. Exit codes: - 0 files staged successfully (Phase 4 will add: submitted + completed) + 0 files staged + ingested successfully (or --detach: just staged + submitted) 2 schema validation failed (synthesized spec rejected) or v0.1-unsupported category passed 3 local-layout or kubeconfig error 4 cluster reachable but parent release / shared PVC missing + 5 ingestor SA token couldn't be obtained, or jobs-manager + rejected the token (401/403) 7 pre-flight succeeded but staging the files failed - (Pod creation, image pull, exec stream, or remote tar error)`, + (Pod creation, image pull, exec stream, or remote tar error) + 8 jobs-manager rejected the submit (4xx/5xx other than auth) + 9 ingestion Job exited non-zero, or completed with row-level + failures the summary panel reports`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { return runDatasetPush(cmd.Context(), cmd.OutOrStdout(), cmd.ErrOrStderr(), @@ -132,6 +148,9 @@ Exit codes: DryRun: dryRun, IngestorSAName: ingestorSAName, StagePodImage: stagePodImage, + Detach: detach, + IdempotencyKey: idempotencyKey, + ImageDigest: imageDigest, }) }, } @@ -166,6 +185,17 @@ Exit codes: "override the ephemeral stage Pod's image (default: digest-pinned alpine 3.20 baked into the CLI). "+ "Pin by digest in your override too — tag-only refs drift silently.") + cmd.Flags().BoolVar(&detach, "detach", false, + "exit immediately after jobs-manager accepts the run (no log streaming, no summary panel). "+ + "Use for CI scenarios; reconnect later with `kubectl logs -f -n job/`.") + cmd.Flags().StringVar(&idempotencyKey, "idempotency-key", "", + "reuse this idempotency key across retry attempts (default: fresh per invocation). "+ + "jobs-manager treats a duplicate key as a replay and attaches to the existing Job "+ + "rather than spawning a new one — useful for at-most-once-across-attempts semantics.") + cmd.Flags().StringVar(&imageDigest, "image-digest", "", + "pin the ingestor container image to a specific digest (default: jobs-manager picks the "+ + "cluster-configured `images.ingestor.digest`). Format: sha256:.") + return cmd } @@ -182,6 +212,12 @@ type runDatasetPushArgs struct { DryRun bool IngestorSAName string StagePodImage string + + // Phase 4 (#152) fields. See the flag declarations for the + // per-knob rationale; all three are optional. + Detach bool + IdempotencyKey string + ImageDigest string } // runDatasetPush is the full Phase 3 implementation: pre-flight @@ -350,14 +386,102 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush return &exitError{code: 7, err: stageErr} } - // 10. Phase 4 (submit to jobs-manager + watch + summary) hooks - // in here — tracebloc/client#152. For now PR-b leaves the - // dataset staged on the PVC and exits 0, which the customer - // can then chase manually via `helm install ingestor ...` if - // they need the ingestion to actually run today. + // 10. Mint the SA token Phase 4 uses to authenticate the POST + // to jobs-manager. Expiry is 1 hour (vs cluster info's 10 + // min) because the full Phase 4 lifecycle — submit + watch + // + log stream — can run that long for large ingestions. + // The chart's helm flow uses the same token-mint code path. _, _ = fmt.Fprintln(out) - _, _ = fmt.Fprintln(out, "Dataset staged. Submission to jobs-manager arrives in Phase 4 (#152);") - _, _ = fmt.Fprintln(out, "in the meantime, the existing helm ingestor flow can pick up the staged files.") + tok, err := cluster.MintIngestorToken(ctx, cs, resolved.Namespace, + release.IngestorSAName, 3600, nil) + if err != nil { + return &exitError{code: 5, err: err} + } + + // 11. Open a port-forward to a Pod backing the jobs-manager + // Service. The CLI runs off-cluster (on a laptop, in CI + // runners outside the cluster network), so the discovered + // *.svc.cluster.local URL isn't reachable — we tunnel + // through the kubeconfig-authenticated apiserver, same as + // `kubectl port-forward`. Bugbot PR #10 r3 caught the + // original broken-by-design direct-URL POST. + _, _ = fmt.Fprintln(out, "Opening port-forward to jobs-manager...") + pf, err := submit.PortForwardJobsManager(ctx, cs, resolved.RestConfig, + resolved.Namespace, release.JobsManagerServiceName, release.JobsManagerPort) + if err != nil { + return &exitError{code: 8, err: fmt.Errorf("setting up jobs-manager port-forward: %w", err)} + } + defer pf.Close() + + // 12. Phase 4: POST to jobs-manager via the local port, + // watch the spawned ingestor Job, render the parsed + // INGESTION SUMMARY panel. + // + // Exit-code mapping: + // SubmitError 401/403 → 5 (auth — same bucket as + // token-mint, shared + // "your SA can't do this" + // diagnostic class) + // SubmitError other 4xx/5xx → 8 (submit failed) + // WatchResult Failed → 9 (ingest failed) + // WatchResult Succeeded + + // summary.HasFailures() → 9 (some rows failed + // even though Job exited 0; + // the ingestor surfaces + // partial-failure summaries) + // WatchResult Detached → 0 (cluster keeps running) + // WatchResult Succeeded clean → 0 + localEndpoint := fmt.Sprintf("http://localhost:%d", pf.LocalPort) + submitRes, err := submit.Run(ctx, submit.Options{ + Submitter: submit.NewHTTPSubmitter(localEndpoint, tok.Token), + Client: cs, + IngestConfigYAML: string(specBytes), + IdempotencyKey: a.IdempotencyKey, + ImageDigest: a.ImageDigest, + Detach: a.Detach, + Out: out, + }) + if err != nil { + switch { + case submit.IsAuthError(err): + return &exitError{code: 5, err: err} + case submit.IsWatchError(err): + // Watch-phase failure: jobs-manager already accepted + // the run, the cluster is doing the work, the CLI + // just couldn't follow along. Exit 9 (ingest-side) + // not 8 (submit-side). Bugbot flagged the + // previously-undifferentiated mapping on PR #10. + return &exitError{code: 9, err: err} + default: + return &exitError{code: 8, err: err} + } + } + + // Detach paths (--detach flag OR SIGINT-mid-watch) are + // success — cluster keeps running; the orchestrator already + // printed the reconnect hint. + if submitRes.Watch == nil || submitRes.Watch.Outcome == submit.JobOutcomeDetached { + return nil + } + + // Watch outcomes. Both Failed and Unknown route to exit 9 + // (Unknown = finalJobStatus timed out without seeing a + // terminal condition, which we can't claim as success). + // Bugbot flagged the prior switch's missing Unknown branch + // on PR #10. + switch submitRes.Watch.Outcome { + case submit.JobOutcomeFailed: + return &exitError{code: 9, err: errors.New("ingestion Job exited non-zero — see logs above")} + case submit.JobOutcomeUnknown: + return &exitError{code: 9, err: errors.New( + "ingestion Job's final status couldn't be determined within the watch window — " + + "check `kubectl get job -n " + submitRes.Submit.Namespace + " " + submitRes.Submit.JobName + "` for the outcome")} + case submit.JobOutcomeSucceeded: + if submitRes.Watch.Summary != nil && submitRes.Watch.Summary.HasFailures() { + return &exitError{code: 9, err: errors.New( + "ingestion Job completed but the summary reports failures — see panel above")} + } + } return nil } diff --git a/internal/cluster/discover.go b/internal/cluster/discover.go index 7ee34c7..b1718f9 100644 --- a/internal/cluster/discover.go +++ b/internal/cluster/discover.go @@ -35,9 +35,20 @@ type ParentRelease struct { // JobsManagerService is the in-cluster DNS name of the // jobs-manager Service, e.g. // "-jobs-manager..svc.cluster.local:8080". - // Used as the POST target for ingestion submissions. + // Used as the POST target for ingestion submissions WHEN + // the CLI runs in-cluster (e.g. CI inside the same cluster). + // For laptop / off-cluster use, the orchestrator port-forwards + // to JobsManagerServiceName + JobsManagerPort instead. JobsManagerService string + // JobsManagerServiceName + JobsManagerPort are the bare Service + // reference for off-cluster port-forwarding (Bugbot PR #10 r3). + // The FQDN-based JobsManagerService URL above doesn't resolve + // from a laptop; the port-forward path uses these to set up a + // localhost tunnel via the kubeconfig API server. + JobsManagerServiceName string + JobsManagerPort int + // IngestorSAName is the name of the ServiceAccount the chart's // hook pods run as. Today this is always the chart's default // "ingestor". Customers who set `ingestionAuthz.serviceAccountName` @@ -143,7 +154,10 @@ func DiscoverParentRelease(ctx context.Context, cs kubernetes.Interface, namespa // release-prefixed form. Customers can always override via the // ingestor subchart's `jobsManager.endpoint` value. svc := pickJobsManagerService(ctx, cs, namespace, release.ReleaseName) - release.JobsManagerService = fmt.Sprintf("http://%s.%s.svc.cluster.local:8080", svc, namespace) + const jobsManagerPort = 8080 // chart's well-known port for /internal/submit-ingestion-run + release.JobsManagerService = fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", svc, namespace, jobsManagerPort) + release.JobsManagerServiceName = svc + release.JobsManagerPort = jobsManagerPort // Read INGESTOR_IMAGE_DIGEST from jobs-manager's pod-spec env. // The chart pipes images.ingestor.digest through to here. diff --git a/internal/cluster/discover_test.go b/internal/cluster/discover_test.go index 34f4433..5c0921c 100644 --- a/internal/cluster/discover_test.go +++ b/internal/cluster/discover_test.go @@ -76,12 +76,14 @@ func TestDiscoverParentRelease_HappyPath(t *testing.T) { } want := ParentRelease{ - ReleaseName: "tracebloc", - ChartVersion: "1.3.5", - AppVersion: "1.3.5", - JobsManagerService: "http://jobs-manager." + ns + ".svc.cluster.local:8080", - IngestorSAName: "ingestor", - IngestorImageDigest: "sha256:463e236748708a5e3564569eec9173ea8cb3bcf515992d4939c5b610f3807a4a", + ReleaseName: "tracebloc", + ChartVersion: "1.3.5", + AppVersion: "1.3.5", + JobsManagerService: "http://jobs-manager." + ns + ".svc.cluster.local:8080", + JobsManagerServiceName: "jobs-manager", + JobsManagerPort: 8080, + IngestorSAName: "ingestor", + IngestorImageDigest: "sha256:463e236748708a5e3564569eec9173ea8cb3bcf515992d4939c5b610f3807a4a", } if *release != want { t.Errorf("mismatch.\ngot: %+v\nwant: %+v", *release, want) diff --git a/internal/submit/body.go b/internal/submit/body.go new file mode 100644 index 0000000..80b641e --- /dev/null +++ b/internal/submit/body.go @@ -0,0 +1,107 @@ +// Package submit owns the `tracebloc dataset push` Phase 4 step: +// POST the synthesized ingest spec to jobs-manager's +// /internal/submit-ingestion-run endpoint, then watch the ingestor +// Job the cluster spawns in response. +// +// Phase 4 sits between Phase 3's stage Pod (which lays files on the +// PVC) and Phase 5's release distribution. The protocol is the same +// one tracebloc/client's ingestor subchart's post-install hook uses +// today — see ingestor/templates/configmap-ingest-config.yaml. +// Keeping the protocol identical means the CLI and the helm flow +// are interchangeable at the cluster's API surface, and the chart +// stays a fully-supported alternative for ops folks who prefer it. +// +// The package is split into: +// - body.go (this file): synthesize the POST body +// - client.go: HTTP client + bearer token + 4xx framing +// - watch.go: poll Job + stream Pod logs +// - summary.go: parse the 📊 INGESTION SUMMARY banner +// - submit.go: top-level orchestrator +package submit + +import ( + "crypto/rand" + "encoding/hex" + "fmt" +) + +// SubmitRequest is the wire shape POSTed to jobs-manager's +// /internal/submit-ingestion-run. Field names mirror the chart's +// ingestor/templates/configmap-ingest-config.yaml body.json key +// for-key so the chart and the CLI are interchangeable on the +// server side. +// +// json struct tags are explicit-snake so a future re-import via +// encoding/json doesn't silently produce mixedCase keys. +type SubmitRequest struct { + // IngestConfig is the customer's ingest spec as a verbatim + // YAML string. jobs-manager re-parses + revalidates this + // server-side. We don't re-marshal — the CLI's Phase 3 spec + // synthesis already produced canonical YAML. + IngestConfig string `json:"ingest_config"` + + // IdempotencyKey is the per-invocation replay token. + // jobs-manager records this in its idempotency-key table; a + // second POST with the same key returns the SAME job_name as + // the first (with replay=true) instead of spawning a new Job. + // + // Default is a fresh UUID-ish 16-byte hex string per + // invocation; `--idempotency-key ` overrides for the + // at-most-once-across-attempts case where the customer + // genuinely wants retry-safety across multiple CLI runs. + IdempotencyKey string `json:"idempotency_key"` + + // ImageDigest optionally pins the ingestor container image. + // Empty = let jobs-manager use the cluster's configured + // default (set by the parent client chart's + // `images.ingestor.digest`, kept current by the auto-upgrade + // cronjob). Setting it locks the run to a specific image, + // matching the chart's --set image.digest=... override path. + // + // `omitempty` on the JSON tag means jobs-manager sees no + // image_digest key at all when this is empty, which is the + // well-tested default-image code path on the server side. + ImageDigest string `json:"image_digest,omitempty"` +} + +// BuildRequest is the constructor used by the orchestrator. Both +// IngestConfig (the YAML the CLI already synthesized in Phase 3) +// and the optional ImageDigest flow through unchanged; the +// idempotency key is the only non-trivial bit. +// +// If override is empty, a fresh 16-byte hex string is generated +// from crypto/rand. UUID-shaped without the dashes — the chart's +// own helper does the same (ingestor.idempotencyKey in +// _helpers.tpl), so server-side hash-table lookups are uniform +// across both flows. +func BuildRequest(ingestYAML string, idempotencyKeyOverride, imageDigest string) (*SubmitRequest, error) { + key := idempotencyKeyOverride + if key == "" { + raw := make([]byte, 16) + if _, err := rand.Read(raw); err != nil { + return nil, fmt.Errorf("generating idempotency key: %w", err) + } + key = hex.EncodeToString(raw) + } + return &SubmitRequest{ + IngestConfig: ingestYAML, + IdempotencyKey: key, + ImageDigest: imageDigest, + }, nil +} + +// SubmitResponse is jobs-manager's 201 reply. job_name is the +// ingestor Job watch.go will poll; namespace is the resolved API +// namespace (usually the same one the CLI POSTed to, but +// jobs-manager can in principle redirect cross-namespace). +// +// Replay distinguishes "we just spawned this Job" (replay=false) +// from "we already have a Job for this idempotency key, here it +// is" (replay=true). The CLI prints a different lifecycle banner +// for each — replay means "another invocation already kicked +// this off; we're attaching to it." +type SubmitResponse struct { + JobName string `json:"job_name"` + Namespace string `json:"namespace"` + Replay bool `json:"replay"` +} diff --git a/internal/submit/body_test.go b/internal/submit/body_test.go new file mode 100644 index 0000000..dd6e459 --- /dev/null +++ b/internal/submit/body_test.go @@ -0,0 +1,100 @@ +package submit + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestBuildRequest_GeneratesIdempotencyKey: when no override is +// provided, BuildRequest produces a fresh hex-encoded 16-byte key. +// Pins both "non-empty" + "looks-like-hex" so a future change to +// random source can't silently produce a malformed key. +func TestBuildRequest_GeneratesIdempotencyKey(t *testing.T) { + req, err := BuildRequest("yaml content", "", "") + if err != nil { + t.Fatalf("BuildRequest: %v", err) + } + if req.IdempotencyKey == "" { + t.Fatal("generated idempotency key is empty") + } + if len(req.IdempotencyKey) != 32 { + t.Errorf("generated key length = %d, want 32 (16 bytes hex)", len(req.IdempotencyKey)) + } + for i, c := range req.IdempotencyKey { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + t.Errorf("key[%d] = %c, want lowercase hex", i, c) + break + } + } +} + +// TestBuildRequest_IdempotencyKeyOverride: --idempotency-key flag +// path. The override is plumbed verbatim — no hashing, no munging — +// so a customer using the same key across retries gets the chart's +// replay semantics. +func TestBuildRequest_IdempotencyKeyOverride(t *testing.T) { + req, err := BuildRequest("yaml", "my-fixed-key-abc123", "") + if err != nil { + t.Fatalf("BuildRequest: %v", err) + } + if req.IdempotencyKey != "my-fixed-key-abc123" { + t.Errorf("IdempotencyKey = %q, want override value", req.IdempotencyKey) + } +} + +// TestBuildRequest_KeysAreUniquePerCall: two back-to-back +// BuildRequest calls with no override produce distinct keys. +// Critical for jobs-manager's idempotency table — same key means +// "this is a retry, replay the previous run." +func TestBuildRequest_KeysAreUniquePerCall(t *testing.T) { + a, err := BuildRequest("yaml", "", "") + if err != nil { + t.Fatalf("BuildRequest a: %v", err) + } + b, err := BuildRequest("yaml", "", "") + if err != nil { + t.Fatalf("BuildRequest b: %v", err) + } + if a.IdempotencyKey == b.IdempotencyKey { + t.Errorf("back-to-back BuildRequest produced identical keys %q", a.IdempotencyKey) + } +} + +// TestBuildRequest_JSONShape: the wire format jobs-manager expects. +// Field names + omitempty behavior are the contract; if any of +// these drift, the server-side handler fails to parse. +func TestBuildRequest_JSONShape(t *testing.T) { + t.Run("with image digest", func(t *testing.T) { + req, _ := BuildRequest("yaml-content", "key123", "sha256:abc") + b, err := json.Marshal(req) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + s := string(b) + for _, want := range []string{ + `"ingest_config":"yaml-content"`, + `"idempotency_key":"key123"`, + `"image_digest":"sha256:abc"`, + } { + if !strings.Contains(s, want) { + t.Errorf("JSON missing %q in: %s", want, s) + } + } + }) + t.Run("omits empty image digest", func(t *testing.T) { + req, _ := BuildRequest("yaml", "key", "") + b, err := json.Marshal(req) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + // jobs-manager's no-image-digest code path is the + // well-tested default; passing an empty string would + // route through the override path on the server. The + // omitempty tag is the contract that keeps the default + // path engaged. + if strings.Contains(string(b), "image_digest") { + t.Errorf("JSON includes image_digest when empty: %s", b) + } + }) +} diff --git a/internal/submit/client.go b/internal/submit/client.go new file mode 100644 index 0000000..972d91b --- /dev/null +++ b/internal/submit/client.go @@ -0,0 +1,194 @@ +package submit + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// SubmitTimeout caps how long we wait for jobs-manager to respond to +// the POST. jobs-manager validates synchronously (schema re-check, +// idempotency lookup, Job creation) — the chart's hook bounds this +// at 30s and we mirror that. Beyond 30s, something genuinely wrong +// is happening server-side, and the customer wants the diagnostic +// rather than a longer wait. +const SubmitTimeout = 30 * time.Second + +// Submitter is the narrow surface the orchestrator (submit.go) uses +// for the POST. Real impl is *HTTPSubmitter; tests use a fake that +// records the request and returns a synthetic response. +type Submitter interface { + Submit(ctx context.Context, req *SubmitRequest) (*SubmitResponse, error) +} + +// HTTPSubmitter is the production implementation. Wraps net/http +// with a fixed jobs-manager URL + a SA token from Phase 2's mint. +// +// Endpoint comes from Phase 2's cluster.DiscoverParentRelease +// (`http://-jobs-manager..svc.cluster.local:8080`), +// SubmitPath is hardcoded per the chart's published contract. +type HTTPSubmitter struct { + // Endpoint is the full jobs-manager URL, e.g. + // "http://release-jobs-manager.tracebloc.svc.cluster.local:8080". + // No trailing slash — the submitter appends SubmitPath. + Endpoint string + + // Token is the bearer token to send in the Authorization + // header. Comes from Phase 2's cluster.MintIngestorToken. + Token string + + // Client is the underlying *http.Client. Set in NewHTTPSubmitter + // with a sensible timeout. Exposed for tests that need to point + // at an httptest.Server with a custom RoundTripper. + Client *http.Client +} + +// SubmitPath is the well-known URL path on jobs-manager. Pinned +// here as a constant rather than a knob because the chart's +// post-install hook also pins it; if jobs-manager ever moves the +// endpoint, both have to bump together (which is a coordinated +// release across tracebloc/client + tracebloc/cli, exactly what +// you want for a protocol-level change). +const SubmitPath = "/internal/submit-ingestion-run" + +// NewHTTPSubmitter returns a Submitter wired with a sensible +// timeout + a transport that DOESN'T do TLS verification against +// the in-cluster CA. The jobs-manager endpoint is HTTP-only inside +// the cluster (kube-proxy handles all the rest), so TLS isn't in +// the picture at all today. If a future jobs-manager exposes +// HTTPS, the InsecureSkipVerify path is the right v0.1 default +// because the customer's kubeconfig has already authenticated them +// to the cluster — the cluster-internal jobs-manager doesn't have +// a CA the laptop would recognize anyway. +func NewHTTPSubmitter(endpoint, token string) *HTTPSubmitter { + return &HTTPSubmitter{ + Endpoint: strings.TrimRight(endpoint, "/"), + Token: token, + Client: &http.Client{ + Timeout: SubmitTimeout, + Transport: &http.Transport{ + // See doc on NewHTTPSubmitter for the + // InsecureSkipVerify rationale. + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + }, + }, + } +} + +// Submit POSTs the request body to jobs-manager and decodes the +// 201 response into a SubmitResponse. On non-201 status codes, the +// remote body is surfaced verbatim so the customer sees whatever +// jobs-manager said (typically a JSON {error, detail} from the +// fastapi handler) rather than just "HTTP 422". +// +// Replays (idempotency-key already seen → 200 with replay=true) +// are reported in the response struct; this method treats them as +// success because the upstream behavior IS "your run is already +// in progress / already done." The orchestrator handles the +// replay branch in its own diagnostic output. +func (s *HTTPSubmitter) Submit(ctx context.Context, req *SubmitRequest) (*SubmitResponse, error) { + body, err := json.Marshal(req) + if err != nil { + // Marshaling a struct of strings shouldn't fail at all; + // surfacing the error means a future struct change broke + // the wire format. + return nil, fmt.Errorf("marshaling submit request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, + s.Endpoint+SubmitPath, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("building submit request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+s.Token) + + httpResp, err := s.Client.Do(httpReq) + if err != nil { + // Network errors (DNS, connection refused, TLS handshake + // failure, ctx cancellation). The error wraps net.OpError + // already; just frame it with our endpoint so the + // customer knows what was being attempted. + return nil, fmt.Errorf("POST %s%s: %w", s.Endpoint, SubmitPath, err) + } + defer func() { _ = httpResp.Body.Close() }() + + respBody, err := io.ReadAll(httpResp.Body) + if err != nil { + // Short read on a 201 body — extremely rare; the + // connection dropped between header and body. + return nil, fmt.Errorf("reading submit response body: %w", err) + } + + // 2xx (typically 201 Created; also 200 for replays) is success. + // Everything else surfaces the remote body verbatim so the + // customer sees jobs-manager's actual diagnostic (HTTP 4xx + // schema-rejection, HTTP 5xx kube-apiserver-failure, etc.) + // rather than just a status code. + if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { + return nil, &SubmitError{ + StatusCode: httpResp.StatusCode, + Body: string(respBody), + Endpoint: s.Endpoint + SubmitPath, + } + } + + var parsed SubmitResponse + if err := json.Unmarshal(respBody, &parsed); err != nil { + // 2xx but the body isn't the expected shape. Almost + // certainly a protocol mismatch (jobs-manager version + // drift). Include the raw body so the customer can pin + // the version they're on. + return nil, fmt.Errorf("decoding submit response (got body %q): %w", string(respBody), err) + } + if parsed.JobName == "" { + // Defensive: a malformed-but-parsing response with no + // job_name would silently break Phase 4's watch step. + return nil, fmt.Errorf("submit response missing job_name (got body %q)", string(respBody)) + } + if parsed.Namespace == "" { + // Same shape as the job_name check: a missing namespace + // would route subsequent k8s API calls (watch the Pod, + // stream logs) at the empty string — kubelet returns + // confusing errors, and the kubectl-logs reconnect hint + // printed in --detach output would be malformed. Bugbot + // PR #10 r2 flagged the gap. + return nil, fmt.Errorf("submit response missing namespace (got body %q)", string(respBody)) + } + return &parsed, nil +} + +// SubmitError is the typed non-2xx response. Pulled into a struct +// (rather than an opaque string) so the orchestrator can branch on +// StatusCode for the exit-code mapping: 401/403 → auth exit code, +// 4xx other → submit-validation exit code, 5xx → submit-server. +// +// Implements `error` + an Is for errors.Is detection in tests. +type SubmitError struct { + StatusCode int + Body string + Endpoint string +} + +func (e *SubmitError) Error() string { + // Compact framing: the customer sees status + body. The body + // is jobs-manager's actual diagnostic (e.g. fastapi's + // {"detail": "..."}) which is the actionable part. + return fmt.Sprintf("jobs-manager %s returned HTTP %d: %s", + e.Endpoint, e.StatusCode, strings.TrimSpace(e.Body)) +} + +// IsSubmitError reports whether err is a *SubmitError. Convenience +// for the orchestrator's exit-code mapping; errors.As would also +// work but this reads cleaner at the branch site. +func IsSubmitError(err error) bool { + var se *SubmitError + return errors.As(err, &se) +} diff --git a/internal/submit/client_test.go b/internal/submit/client_test.go new file mode 100644 index 0000000..7519c14 --- /dev/null +++ b/internal/submit/client_test.go @@ -0,0 +1,234 @@ +package submit + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestHTTPSubmitter_HappyPath: jobs-manager returns 201 with the +// canonical body shape; client decodes correctly + surfaces all +// three response fields. +func TestHTTPSubmitter_HappyPath(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Pin the wire format jobs-manager actually expects. + if r.Method != http.MethodPost { + t.Errorf("got method %s, want POST", r.Method) + } + if r.URL.Path != SubmitPath { + t.Errorf("got path %s, want %s", r.URL.Path, SubmitPath) + } + if got := r.Header.Get("Authorization"); got != "Bearer fake-token-deadbeef" { + t.Errorf("Authorization = %q, want Bearer fake-token-deadbeef", got) + } + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q, want application/json", got) + } + body, _ := io.ReadAll(r.Body) + // Light shape check — body_test.go pins the full JSON shape. + if !strings.Contains(string(body), `"ingest_config"`) { + t.Errorf("body missing ingest_config: %s", body) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "job_name": "ingestor-abc123", + "namespace": "tracebloc", + "replay": false, + }) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "fake-token-deadbeef") + req, _ := BuildRequest("yaml-content", "key1", "") + + resp, err := s.Submit(context.Background(), req) + if err != nil { + t.Fatalf("Submit: %v", err) + } + if resp.JobName != "ingestor-abc123" { + t.Errorf("JobName = %q, want ingestor-abc123", resp.JobName) + } + if resp.Namespace != "tracebloc" { + t.Errorf("Namespace = %q, want tracebloc", resp.Namespace) + } + if resp.Replay { + t.Errorf("Replay = true, want false") + } +} + +// TestHTTPSubmitter_ReplayResponse: replay=true is also a success +// path. The orchestrator distinguishes the two via the response +// flag, not via HTTP status — both come back 2xx. +func TestHTTPSubmitter_ReplayResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) // jobs-manager returns 200 for replays per source + _ = json.NewEncoder(w).Encode(map[string]any{ + "job_name": "existing-job", + "namespace": "tracebloc", + "replay": true, + }) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "same-key", "") + resp, err := s.Submit(context.Background(), req) + if err != nil { + t.Fatalf("Submit on replay: %v", err) + } + if !resp.Replay { + t.Errorf("Replay = false, want true") + } +} + +// TestHTTPSubmitter_4xxSurfacesBody: 4xx from jobs-manager +// surfaces the verbatim body so the customer sees jobs-manager's +// actual diagnostic (typically {"detail": "..."}), not just +// "HTTP 422". +func TestHTTPSubmitter_4xxSurfacesBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte(`{"detail":"ingest_config schema rejected: missing required field 'intent'"}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on 4xx") + } + if !IsSubmitError(err) { + t.Errorf("err is not *SubmitError: %T", err) + } + for _, want := range []string{"HTTP 422", "missing required field 'intent'"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %v", want, err) + } + } +} + +// TestHTTPSubmitter_401IsAuthError: distinguishes the auth case +// (401/403) from generic 4xx for the orchestrator's exit-code +// mapping. Used by the CLI to return "your SA token doesn't +// work" vs "your spec was rejected." +func TestHTTPSubmitter_401IsAuthError(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + t.Run(http.StatusText(status), func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"detail":"token expired"}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on auth error") + } + if !IsAuthError(err) { + t.Errorf("IsAuthError(%v) = false, want true (status=%d)", err, status) + } + }) + } +} + +// TestHTTPSubmitter_5xxNotAuthError: 5xx is server-side trouble +// (kube-apiserver flake, jobs-manager bug), NOT an auth issue. +// IsAuthError should return false so the orchestrator routes to +// the right exit-code bucket. +func TestHTTPSubmitter_5xxNotAuthError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"detail":"internal"}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on 500") + } + if IsAuthError(err) { + t.Errorf("IsAuthError(500) = true, want false") + } + if !IsSubmitError(err) { + t.Errorf("err is not *SubmitError: %T", err) + } +} + +// TestHTTPSubmitter_2xxMissingJobName: a malformed 2xx response +// (server bug or version drift) without job_name is a hard error, +// not a silent success — the orchestrator wouldn't know what to +// watch. +func TestHTTPSubmitter_2xxMissingJobName(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"namespace":"tracebloc","replay":false}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on missing job_name") + } + if !strings.Contains(err.Error(), "missing job_name") { + t.Errorf("error missing 'missing job_name' framing: %v", err) + } +} + +// TestHTTPSubmitter_NetworkError: unreachable server (closed +// httptest.Server) surfaces as a wrapped net error with the +// endpoint in the message so customers know what was attempted. +func TestHTTPSubmitter_NetworkError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + })) + endpoint := srv.URL + srv.Close() // kill the server + + s := NewHTTPSubmitter(endpoint, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on unreachable server") + } + if !strings.Contains(err.Error(), endpoint) { + t.Errorf("error missing endpoint URL %q: %v", endpoint, err) + } +} + +// TestHTTPSubmitter_RespectsContext: ctx cancellation aborts the +// in-flight POST. Critical for the SIGINT path (main.go's +// signal.NotifyContext). +func TestHTTPSubmitter_RespectsContext(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Wait longer than the test's ctx allows. + <-r.Context().Done() + w.WriteHeader(http.StatusGatewayTimeout) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // already cancelled + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(ctx, req) + if err == nil { + t.Fatal("Submit returned nil on cancelled ctx") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("error doesn't wrap context.Canceled: %v", err) + } +} diff --git a/internal/submit/portforward.go b/internal/submit/portforward.go new file mode 100644 index 0000000..20c6155 --- /dev/null +++ b/internal/submit/portforward.go @@ -0,0 +1,238 @@ +package submit + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +// ForwardedConnection is a live port-forward to an in-cluster +// Service. LocalPort is the random port the kernel picked on the +// CLI host; the Submitter POSTs to http://localhost:LocalPort. +// +// Close MUST be called when the caller is done — otherwise the +// goroutine running the SPDY tunnel leaks for the lifetime of the +// process. +type ForwardedConnection struct { + LocalPort int + + stopCh chan struct{} + done chan struct{} +} + +// Close tears down the port-forward. Safe to call multiple times. +func (f *ForwardedConnection) Close() { + select { + case <-f.stopCh: + return // already closed + default: + } + close(f.stopCh) + // Wait briefly for the goroutine to drain — without this, a + // fast Close-then-process-exit could race the SPDY teardown + // and leave a half-open connection on the apiserver side. + select { + case <-f.done: + case <-time.After(2 * time.Second): + } +} + +// PortForwardJobsManager opens a port-forward to a Pod backing the +// jobs-manager Service in `namespace`. The customer's CLI runs +// off-cluster (on a laptop, in a CI runner outside the cluster +// network); the discovered jobs-manager URL is a +// *.svc.cluster.local name that's NOT resolvable from there. The +// port-forward routes traffic through the kubeconfig-authenticated +// kube-apiserver connection — same machinery `kubectl port-forward` +// uses internally. +// +// Returns a ForwardedConnection whose LocalPort the caller targets +// for HTTP. Bugbot PR #10 r3 caught the broken-by-design assumption +// in the initial Phase 4 implementation. +// +// Lifecycle: caller MUST defer Close(). The port-forward stays open +// for the entire submit + watch sequence (submit needs a single +// POST, watch only uses the kubeconfig API server connection, so +// strictly speaking we could close right after the POST — but +// keeping it open is simpler and the resource cost is one idle +// goroutine). +func PortForwardJobsManager( + ctx context.Context, + cs kubernetes.Interface, + restConfig *rest.Config, + namespace, serviceName string, + targetPort int, +) (*ForwardedConnection, error) { + // 1. Find a Running Pod backing the Service. client-go's + // port-forward API speaks Pods, not Services — even though + // kubectl port-forward accepts both, it resolves the Service + // to a Pod internally. + pod, err := pickServicePod(ctx, cs, namespace, serviceName) + if err != nil { + return nil, fmt.Errorf("resolving Service %s/%s to a Pod: %w", + namespace, serviceName, err) + } + + // 2. Build the SPDY transport. client-go bundles helpers in + // transport/spdy for exactly this; the round-tripper handles + // the SPDY upgrade negotiation the apiserver expects on the + // /portforward subresource. + transport, upgrader, err := spdy.RoundTripperFor(restConfig) + if err != nil { + return nil, fmt.Errorf("building SPDY transport: %w", err) + } + + // 3. Construct the portforward URL on the Pod. The path is + // /api/v1/namespaces//pods//portforward; we build + // it via the REST client so kubeconfig's authentication + + // TLS config flow through automatically. + req := cs.CoreV1().RESTClient().Post(). + Resource("pods"). + Namespace(namespace). + Name(pod.Name). + SubResource("portforward") + + dialer := spdy.NewDialer(upgrader, + &http.Client{Transport: transport}, + "POST", req.URL()) + + // 4. Create the port-forwarder. "0:" means "pick + // any free local port; map it to in the Pod." + // The kernel allocates the local port at goroutine start; + // we read it back after readyCh fires. + stopCh := make(chan struct{}) + readyCh := make(chan struct{}) + pf, err := portforward.New(dialer, + []string{fmt.Sprintf("0:%d", targetPort)}, + stopCh, readyCh, + io.Discard, // forwarder's stdout — verbose listener logs we don't want + io.Discard, // forwarder's stderr + ) + if err != nil { + return nil, fmt.Errorf("creating port-forwarder: %w", err) + } + + // 5. Launch the forward goroutine. ForwardPorts blocks until + // stopCh closes (or it errors). The select below waits for + // EITHER ready (success) or done (early failure). + // + // errCh is buffered (capacity 1) AND the send is wrapped in + // a non-blocking select with a default — two layers of + // safety against the "goroutine leaks waiting to send" + // pattern. ForwardPorts only ever sends once, so the buffer + // is sufficient; the non-blocking select is paranoid + // defensive against a future refactor that adds a second + // send path. Bugbot PR #10 r5 flagged the (already- + // buffered) channel as a potential leak — the false alarm + // is worth closing out structurally. + done := make(chan struct{}) + errCh := make(chan error, 1) + go func() { + defer close(done) + // ForwardPorts is the long-running call. Its return value + // is meaningful only on early failure — on normal close, + // it returns nil after stopCh fires. + err := pf.ForwardPorts() + if err == nil { + return + } + select { + case errCh <- err: + default: + // Buffer full = receiver already saw an earlier error + // (impossible today — single sender, single error — + // but the default arm makes the goroutine drainable + // regardless of any future change). + } + }() + + select { + case <-readyCh: + // happy path — port allocated, tunnel up + case err := <-errCh: + return nil, fmt.Errorf("port-forward to %s/%s failed during startup: %w", + namespace, pod.Name, err) + case <-ctx.Done(): + close(stopCh) + return nil, ctx.Err() + } + + ports, err := pf.GetPorts() + if err != nil { + close(stopCh) + return nil, fmt.Errorf("reading allocated port: %w", err) + } + if len(ports) == 0 { + close(stopCh) + return nil, fmt.Errorf("port-forward allocated zero ports") + } + + return &ForwardedConnection{ + LocalPort: int(ports[0].Local), + stopCh: stopCh, + done: done, + }, nil +} + +// pickServicePod resolves a Service to a Running Pod backing it. +// Uses the Service's own selector (read from the Service spec) to +// match Pods — same mechanism the cluster's own kube-proxy uses. +// +// Picks the first Running Pod found; in the common case there's +// only one (jobs-manager is single-replica by chart default). For +// multi-replica deployments, picking any Running Pod is fine — +// they're load-balanced equivalents. +func pickServicePod(ctx context.Context, cs kubernetes.Interface, namespace, serviceName string) (*corev1.Pod, error) { + svc, err := cs.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("reading service %s/%s: %w", namespace, serviceName, err) + } + if len(svc.Spec.Selector) == 0 { + // A Service with no selector is one whose Endpoints are + // hand-managed (e.g. ExternalName). The chart's jobs- + // manager is a normal selector-based Service so this + // shouldn't happen in production; the error makes the + // debugging path obvious if it ever does. + return nil, fmt.Errorf( + "service %s/%s has no selector — can't resolve to a Pod for port-forwarding", + namespace, serviceName) + } + + // Build a label selector from the Service's spec.selector map. + // strings.Join keeps the order deterministic for readable + // error output; the order doesn't affect the actual Pods + // returned. + parts := make([]string, 0, len(svc.Spec.Selector)) + for k, v := range svc.Spec.Selector { + parts = append(parts, k+"="+v) + } + selector := strings.Join(parts, ",") + + pods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return nil, fmt.Errorf("listing Pods for service %s/%s: %w", + namespace, serviceName, err) + } + for i := range pods.Items { + p := &pods.Items[i] + if p.Status.Phase == corev1.PodRunning { + return p, nil + } + } + return nil, fmt.Errorf( + "no Running Pod backing service %s/%s (found %d Pod(s); "+ + "check `kubectl get pods -n %s -l %s`)", + namespace, serviceName, len(pods.Items), namespace, selector) +} diff --git a/internal/submit/portforward_test.go b/internal/submit/portforward_test.go new file mode 100644 index 0000000..933831c --- /dev/null +++ b/internal/submit/portforward_test.go @@ -0,0 +1,142 @@ +package submit + +import ( + "context" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// The full port-forward (PortForwardJobsManager) requires a real +// apiserver + SPDY upgrade, so it's out of scope for unit tests — +// covered by the EKS smoke. What IS testable: pickServicePod's +// Service→Pod resolution, which is the only client-go-only logic +// in the file. + +// svc constructs a Service with the given selector. Used to seed +// the fake clientset. +func svc(name string, selector map[string]string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tracebloc"}, + Spec: corev1.ServiceSpec{Selector: selector}, + } +} + +// podForSvc constructs a Pod with labels matching the selector. +func podForSvc(name string, labels map[string]string, phase corev1.PodPhase) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "tracebloc", + Labels: labels, + }, + Status: corev1.PodStatus{Phase: phase}, + } +} + +// TestPickServicePod_HappyPath: a Service with one matching Running +// Pod resolves to that Pod's name. +func TestPickServicePod_HappyPath(t *testing.T) { + sel := map[string]string{"app": "jobs-manager"} + cs := fake.NewClientset( + svc("jobs-manager", sel), + podForSvc("jobs-manager-abc", sel, corev1.PodRunning), + ) + p, err := pickServicePod(context.Background(), cs, "tracebloc", "jobs-manager") + if err != nil { + t.Fatalf("pickServicePod: %v", err) + } + if p.Name != "jobs-manager-abc" { + t.Errorf("Pod name = %q, want jobs-manager-abc", p.Name) + } +} + +// TestPickServicePod_SkipsNonRunning: Pending / Failed Pods backing +// the same Service are filtered out — the port-forward only works +// against a Running Pod. +func TestPickServicePod_SkipsNonRunning(t *testing.T) { + sel := map[string]string{"app": "jobs-manager"} + cs := fake.NewClientset( + svc("jobs-manager", sel), + podForSvc("crashed", sel, corev1.PodFailed), + podForSvc("pending", sel, corev1.PodPending), + podForSvc("running", sel, corev1.PodRunning), + ) + p, err := pickServicePod(context.Background(), cs, "tracebloc", "jobs-manager") + if err != nil { + t.Fatalf("pickServicePod: %v", err) + } + if p.Name != "running" { + t.Errorf("Pod name = %q, want running", p.Name) + } +} + +// TestPickServicePod_NoMatchingPod: a Service whose Pods are all +// non-Running (or absent) surfaces a clear error pointing at the +// kubectl command to debug. +func TestPickServicePod_NoMatchingPod(t *testing.T) { + sel := map[string]string{"app": "jobs-manager"} + cs := fake.NewClientset( + svc("jobs-manager", sel), + podForSvc("crashed", sel, corev1.PodFailed), + ) + _, err := pickServicePod(context.Background(), cs, "tracebloc", "jobs-manager") + if err == nil { + t.Fatal("pickServicePod returned nil on no-Running-Pod") + } + for _, want := range []string{ + "no Running Pod", + "jobs-manager", + "kubectl get pods", + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %v", want, err) + } + } +} + +// TestPickServicePod_ServiceMissing: trying to port-forward to a +// non-existent Service surfaces with the Service name in the error. +func TestPickServicePod_ServiceMissing(t *testing.T) { + cs := fake.NewClientset() // empty + _, err := pickServicePod(context.Background(), cs, "tracebloc", "missing-svc") + if err == nil { + t.Fatal("pickServicePod returned nil on missing Service") + } + if !strings.Contains(err.Error(), "missing-svc") { + t.Errorf("error missing service name: %v", err) + } +} + +// TestPickServicePod_NoSelector: ExternalName Services and other +// selector-less shapes can't be port-forwarded by Pod lookup. +// Surface a clear error rather than silently picking nothing. +func TestPickServicePod_NoSelector(t *testing.T) { + cs := fake.NewClientset(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "externalname-svc", Namespace: "tracebloc"}, + // no Spec.Selector + }) + _, err := pickServicePod(context.Background(), cs, "tracebloc", "externalname-svc") + if err == nil { + t.Fatal("pickServicePod returned nil on selector-less Service") + } + if !strings.Contains(err.Error(), "no selector") { + t.Errorf("error missing selector-less framing: %v", err) + } +} + +// TestForwardedConnection_CloseIdempotent: Close is safe to call +// multiple times. defer-Close patterns at multiple levels of the +// orchestrator shouldn't risk a double-close panic. +func TestForwardedConnection_CloseIdempotent(t *testing.T) { + stopCh := make(chan struct{}) + done := make(chan struct{}) + close(done) // simulate goroutine already finished + f := &ForwardedConnection{LocalPort: 12345, stopCh: stopCh, done: done} + f.Close() + f.Close() // must not panic + f.Close() +} diff --git a/internal/submit/submit.go b/internal/submit/submit.go new file mode 100644 index 0000000..d4ccab3 --- /dev/null +++ b/internal/submit/submit.go @@ -0,0 +1,206 @@ +package submit + +import ( + "context" + "errors" + "fmt" + "io" + + "k8s.io/client-go/kubernetes" +) + +// Options bundles every dependency Run needs. The CLI builds one +// from the resolved Phase 2/3 state (kubeconfig clientset, SA +// token, jobs-manager endpoint) + the flags from Phase 4 +// (--detach, --idempotency-key, --image-digest). +type Options struct { + // Submitter is how the POST reaches jobs-manager. Production + // uses NewHTTPSubmitter(endpoint, token); tests inject a + // fake that captures the request + returns a canned response. + Submitter Submitter + + // Client is the kubernetes.Interface for the watch loop's + // Job + Pod polls. Same clientset Phase 3 used. + Client kubernetes.Interface + + // IngestConfigYAML is the synthesized YAML body to POST. + // Phase 3 already produced this in canonical form + // (push.SpecArgs.Build → yaml.Marshal) so we don't re-marshal + // here. + IngestConfigYAML string + + // IdempotencyKey overrides the auto-generated random key. + // Empty = let BuildRequest generate a fresh one. Used by + // the --idempotency-key flag for retry-safety across + // invocations. + IdempotencyKey string + + // ImageDigest optionally pins the ingestor image. Empty = + // jobs-manager uses the cluster's configured default. + ImageDigest string + + // Detach exits immediately after the 201 — no watch, no log + // streaming, no summary. Used by CI scenarios where the + // customer just wants the Job name in stdout and the run + // proceeds asynchronously in the cluster. + Detach bool + + // Out is the customer-facing log stream. Submit writes the + // 201 announcement here, then either streams the Pod's logs + // to it (live watch) or prints the Job name (detach). The + // rendered summary panel also goes here. + Out io.Writer +} + +// Result is what Run reports back to the CLI orchestrator. +// Outcome drives the exit-code mapping (see cli/dataset.go's +// Phase 4 wiring); JobName + PodName are echoed back so the CLI +// can build "reconnect with kubectl logs -n " hints. +type Result struct { + // Submit is the 201 response from jobs-manager. Non-nil on + // any path that got past the POST (including --detach + + // failed watches). nil only if the POST itself failed. + Submit *SubmitResponse + + // Watch is the result of the watch loop. nil on --detach + // (we never started watching). nil on early POST failure. + Watch *WatchResult +} + +// Run is the Phase 4 top-level entrypoint. Steps: +// +// 1. BuildRequest from the YAML + flags +// 2. POST via opts.Submitter; surface SubmitError verbatim +// 3. Print the 201 announcement (job_name / namespace / replay flag) +// 4. If --detach, exit +// 5. WatchJob until Pod terminates or ctx cancels +// 6. Render the parsed Summary panel +// +// Returns the Result + an error. Errors come from steps 1-2-5; on +// steps 3 + 4 + 6, success means "we got far enough" and the +// outcome of the actual ingestion is in Result.Watch.Outcome. +func Run(ctx context.Context, opts Options) (*Result, error) { + if opts.Out == nil { + opts.Out = io.Discard + } + + req, err := BuildRequest(opts.IngestConfigYAML, opts.IdempotencyKey, opts.ImageDigest) + if err != nil { + return nil, fmt.Errorf("building submit request: %w", err) + } + + resp, err := opts.Submitter.Submit(ctx, req) + if err != nil { + return nil, err + } + + // 201 announcement. Customer sees this whether --detach is + // set or not, so they have the Job name for kubectl-poke + // follow-up. + if resp.Replay { + _, _ = fmt.Fprintf(opts.Out, + "Replayed: idempotency key matches a previous run; attaching to existing Job %s/%s\n", + resp.Namespace, resp.JobName) + } else { + _, _ = fmt.Fprintf(opts.Out, + "Submitted: jobs-manager spawned ingestor Job %s/%s\n", + resp.Namespace, resp.JobName) + } + + if opts.Detach { + // --detach: print the reconnect hint and bail. The + // cluster continues without us; the customer can come + // back with `kubectl logs -f -n job/`. + _, _ = fmt.Fprintf(opts.Out, + "Detached (no log streaming). Reconnect with: kubectl logs -f -n %s job/%s\n", + resp.Namespace, resp.JobName) + return &Result{Submit: resp}, nil + } + + // Watch loop. ctx propagates SIGINT cancellation (main.go's + // signal.NotifyContext); a Ctrl-C during the watch produces + // Outcome=Detached + the reconnect hint below. + _, _ = fmt.Fprintf(opts.Out, "Streaming logs from Job %s/%s:\n", resp.Namespace, resp.JobName) + wr, err := WatchJob(ctx, opts.Client, resp.Namespace, resp.JobName, opts.Out) + if err != nil { + // Tag as WatchError so the orchestrator picks the + // ingest-flavored exit code (9), not the submit-flavored + // one (8). The cluster has already accepted the run by + // this point — the CLI just failed to follow it. + return &Result{Submit: resp}, &WatchError{Err: fmt.Errorf("watching ingestor Job: %w", err)} + } + + // Detach paths print a per-reason diagnostic + the same + // kubectl-logs reconnect hint. Bugbot PR #10 r7 caught the + // previous "on signal" framing being misleading for the + // timeout-detach cases — customers who hit the 5-min + // PodReadyTimeout or 1-hour JobWatchTimeout aren't pressing + // Ctrl-C, so attributing it to "signal" was wrong. + if wr.Outcome == JobOutcomeDetached { + _, _ = fmt.Fprintln(opts.Out) + switch wr.DetachReason { + case DetachReasonSignal: + _, _ = fmt.Fprintln(opts.Out, "Detached on signal.") + case DetachReasonPodWaitTimeout: + _, _ = fmt.Fprintln(opts.Out, + "Detached: the ingestor Pod didn't reach Ready within the observation window. "+ + "This usually means slow image pull or scheduling backlog — the run is in "+ + "jobs-manager's queue and will execute when the Pod starts.") + case DetachReasonWatchCap: + _, _ = fmt.Fprintln(opts.Out, + "Detached: the watch window (1 hour) elapsed while the ingestion was still running.") + default: + _, _ = fmt.Fprintln(opts.Out, "Detached.") + } + _, _ = fmt.Fprintf(opts.Out, + "Ingestion continues in the cluster. Reconnect with: kubectl logs -f -n %s job/%s\n", + resp.Namespace, resp.JobName) + return &Result{Submit: resp, Watch: wr}, nil + } + + // Render the summary panel if the ingestor produced one. + // Both Succeeded and Failed paths print it — on Failed, the + // banner tells the customer what got partially through. + if wr.Summary != nil { + _, _ = fmt.Fprintln(opts.Out) + _, _ = fmt.Fprint(opts.Out, RenderPanel(wr.Summary)) + } + + return &Result{Submit: resp, Watch: wr}, nil +} + +// IsAuthError reports whether the error is the auth-flavored case +// (401/403 from jobs-manager). The orchestrator's exit-code +// mapping uses this to distinguish "your SA token doesn't work" +// from "your spec was rejected." +func IsAuthError(err error) bool { + var se *SubmitError + if !errors.As(err, &se) { + return false + } + return se.StatusCode == 401 || se.StatusCode == 403 +} + +// WatchError wraps errors that originated in the watch phase +// (waitForJobPod, log streaming, finalJobStatus). The +// orchestrator distinguishes these from submit-phase errors so +// the exit-code mapping is correct: jobs-manager accepted the +// run already, the cluster is doing the work, the CLI just +// failed to follow along. Maps to exit code 9 (ingest-side +// problem), not 8 (submit-side problem). Bugbot flagged the +// previous "everything that wasn't auth → exit 8" version on +// PR #10. +type WatchError struct { + Err error +} + +func (e *WatchError) Error() string { return e.Err.Error() } +func (e *WatchError) Unwrap() error { return e.Err } + +// IsWatchError reports whether err originated in the watch phase +// rather than the submit phase. The orchestrator's exit-code +// branch uses this directly. +func IsWatchError(err error) bool { + var we *WatchError + return errors.As(err, &we) +} diff --git a/internal/submit/submit_test.go b/internal/submit/submit_test.go new file mode 100644 index 0000000..cf34126 --- /dev/null +++ b/internal/submit/submit_test.go @@ -0,0 +1,231 @@ +package submit + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "k8s.io/client-go/kubernetes/fake" +) + +// fakeSubmitter captures the request + returns a canned response. +// Used by Run() tests to exercise the orchestrator without needing +// an httptest.Server (covered by client_test.go). +type fakeSubmitter struct { + gotRequest *SubmitRequest + resp *SubmitResponse + err error +} + +func (f *fakeSubmitter) Submit(_ context.Context, req *SubmitRequest) (*SubmitResponse, error) { + f.gotRequest = req + return f.resp, f.err +} + +// TestRun_DetachPath_HappyPath: --detach exits immediately after +// the 201 with the reconnect hint. No watch loop, no log streaming. +func TestRun_DetachPath_HappyPath(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{ + JobName: "ingestor-abc", + Namespace: "tracebloc", + Replay: false, + }, + } + var out bytes.Buffer + + res, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "apiVersion: tracebloc.io/v1\n", + Detach: true, + Out: &out, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Submit == nil || res.Submit.JobName != "ingestor-abc" { + t.Errorf("Result.Submit lost: %+v", res.Submit) + } + if res.Watch != nil { + t.Errorf("Result.Watch = %+v, want nil (detach skips watch)", res.Watch) + } + for _, want := range []string{ + "Submitted: jobs-manager spawned ingestor Job tracebloc/ingestor-abc", + "Detached (no log streaming)", + "kubectl logs -f -n tracebloc job/ingestor-abc", + } { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q in:\n%s", want, out.String()) + } + } +} + +// TestRun_ReplayPath: replay=true changes the announcement +// wording — "attaching to existing Job" instead of "spawned" — +// because the cluster is already doing the work. +func TestRun_ReplayPath(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{ + JobName: "ingestor-existing", + Namespace: "tracebloc", + Replay: true, + }, + } + var out bytes.Buffer + + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml", + Detach: true, // skip the watch for this test + Out: &out, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !strings.Contains(out.String(), "Replayed:") { + t.Errorf("output missing Replayed framing:\n%s", out.String()) + } + if !strings.Contains(out.String(), "attaching to existing Job") { + t.Errorf("output missing replay-specific wording:\n%s", out.String()) + } +} + +// TestRun_SubmitErrorPropagates: a non-2xx from jobs-manager +// stops Run before any watching happens. The error surfaces with +// jobs-manager's body framing (the client.go path). +func TestRun_SubmitErrorPropagates(t *testing.T) { + sub := &fakeSubmitter{ + err: &SubmitError{ + StatusCode: 422, + Body: `{"detail":"bad spec"}`, + Endpoint: "http://jm/internal/submit-ingestion-run", + }, + } + var out bytes.Buffer + + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml", + Out: &out, + }) + if err == nil { + t.Fatal("Run returned nil on submit error") + } + if !IsSubmitError(err) { + t.Errorf("err is not *SubmitError: %T", err) + } +} + +// TestRun_BuildRequestErrorPropagates: a crypto/rand failure in +// BuildRequest stops Run before the submitter even gets called. +// We can't easily mock crypto/rand, but we can verify the error +// path is wired by checking that any failure here doesn't reach +// the submitter. This test is more about the contract than the +// trigger. +func TestRun_PassesRequestFieldsThrough(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{JobName: "j", Namespace: "ns"}, + } + var out bytes.Buffer + + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml-content-verbatim", + IdempotencyKey: "my-key-override", + ImageDigest: "sha256:abc", + Detach: true, + Out: &out, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if sub.gotRequest == nil { + t.Fatal("submitter never called") + } + if sub.gotRequest.IngestConfig != "yaml-content-verbatim" { + t.Errorf("IngestConfig = %q, want yaml-content-verbatim", + sub.gotRequest.IngestConfig) + } + if sub.gotRequest.IdempotencyKey != "my-key-override" { + t.Errorf("IdempotencyKey = %q, want override value", + sub.gotRequest.IdempotencyKey) + } + if sub.gotRequest.ImageDigest != "sha256:abc" { + t.Errorf("ImageDigest = %q, want sha256:abc", + sub.gotRequest.ImageDigest) + } +} + +// TestRun_NilOutDefaultsToDiscard: callers passing nil Out +// shouldn't panic. The orchestrator silently discards output. +func TestRun_NilOutDefaultsToDiscard(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{JobName: "j", Namespace: "ns"}, + } + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml", + Detach: true, + Out: nil, + }) + if err != nil { + t.Fatalf("Run with nil Out panicked or errored: %v", err) + } +} + +// TestIsWatchError: pin the contract that the orchestrator uses +// to distinguish watch-phase failures (exit 9) from submit-phase +// failures (exit 8). Bugbot r1 found the missing distinction; +// this test guards against a regression that drops the typing. +func TestIsWatchError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"plain error", errors.New("plain"), false}, + {"submit error", &SubmitError{StatusCode: 422}, false}, + {"watch error", &WatchError{Err: errors.New("inner")}, true}, + {"wrapped watch error", fmt.Errorf("outer: %w", &WatchError{Err: errors.New("inner")}), true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsWatchError(c.err); got != c.want { + t.Errorf("IsWatchError = %v, want %v", got, c.want) + } + }) + } +} + +// TestIsAuthError: helper smoke test. Pin the contract used by +// the CLI's exit-code mapping. +func TestIsAuthError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"non-submit error", errors.New("network"), false}, + {"submit 401", &SubmitError{StatusCode: 401}, true}, + {"submit 403", &SubmitError{StatusCode: 403}, true}, + {"submit 422", &SubmitError{StatusCode: 422}, false}, + {"submit 500", &SubmitError{StatusCode: 500}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsAuthError(c.err); got != c.want { + t.Errorf("IsAuthError = %v, want %v", got, c.want) + } + }) + } +} diff --git a/internal/submit/summary.go b/internal/submit/summary.go new file mode 100644 index 0000000..23fd375 --- /dev/null +++ b/internal/submit/summary.go @@ -0,0 +1,388 @@ +package submit + +import ( + "bufio" + "bytes" + "fmt" + "io" + "regexp" + "strconv" + "strings" +) + +// Summary is the parsed contents of the ingestor's 📊 INGESTION +// SUMMARY 📊 banner. Fields mirror what +// tracebloc_ingestor/ingestors/base.py:624+ prints. Zero values +// are valid — an early failure may produce a summary with most +// counts at 0 (the ingestor still prints the banner so operators +// can see what got through). +// +// All counts are int64 to fit row counts well past int32's 2.1B +// ceiling — a customer ingesting a few-billion-row table would +// silently truncate with int. +type Summary struct { + // IngestorID is the run identifier the ingestor logs at the + // top of the banner. Useful in the customer-facing panel as + // "you can grep cluster logs for this ID." + IngestorID string + + // TotalRecords is the row count the ingestor saw in the + // source data. Includes every row regardless of outcome. + TotalRecords int64 + + // ProcessedRecords is the row count that made it through + // validation (passed FileTypeValidator, ImageResolutionValidator, + // etc.). Excludes invalid rows. + ProcessedRecords int64 + + // InsertedRecords is the row count that landed in the + // cluster-internal MySQL. The "I actually have this data + // staged" metric — this is what matters for downstream + // training jobs. + InsertedRecords int64 + + // APISentRecords is the row count that synced metadata to + // the central tracebloc backend. Only the row count + label + // is sent, not the raw data; this is the "central catalog + // knows about this dataset" metric. + APISentRecords int64 + + // SkippedRecords is the row count rejected by validators + // (wrong dimensions, missing image file, etc.). Non-fatal + // for the run but worth surfacing — a customer with 50% + // skipped wants to see that. + SkippedRecords int64 + + // FileTransferFailures is the count of files (NOT rows) that + // failed to transfer to the requests-proxy. Distinct from + // FailedRecords because file transfer is a separate stage + // from DB insertion. Non-zero here is the dominant "your + // network is flaky" signal. + FileTransferFailures int64 + + // FailedRecords is the row count that errored at the + // DB-insert stage (constraint violation, type mismatch, + // connection drop). The catch-all "something went wrong + // at the storage layer" bucket. + FailedRecords int64 +} + +// HasFailures returns true if any failure-class counter is non-zero. +// Used by the orchestrator to decide which exit code to return +// (success: 0, ingest-with-failures: non-zero) and how to color +// the rendered panel. +func (s *Summary) HasFailures() bool { + if s == nil { + return false + } + return s.FileTransferFailures > 0 || s.FailedRecords > 0 +} + +// SuccessRate returns a 0-100 percentage for the panel header. +// Defined as ProcessedRecords / TotalRecords; returns 0 when +// TotalRecords is 0 to avoid divide-by-zero in early-failure +// banners. +func (s *Summary) SuccessRate() float64 { + if s == nil || s.TotalRecords == 0 { + return 0 + } + return float64(s.ProcessedRecords) / float64(s.TotalRecords) * 100 +} + +// ansiCodeRE matches the ANSI SGR (Select Graphic Rendition) +// escape sequences the ingestor uses for its color output — +// `\x1b[1m` (bold), `\x1b[36m` (cyan), `\x1b[0m` (reset), etc. +// The ingestor prints these inline in the summary text; we strip +// them before parsing so a future palette change doesn't break +// the parser. +var ansiCodeRE = regexp.MustCompile(`\x1b\[[0-9;]*m`) + +// stripANSI removes SGR codes from a single line, returning the +// printable text. Implementation note: this is a hot path for the +// log stream so we avoid an allocation when no codes are present. +func stripANSI(line string) string { + if !strings.Contains(line, "\x1b[") { + return line + } + return ansiCodeRE.ReplaceAllString(line, "") +} + +// bannerStartMarker is the ingestor's literal banner-header line +// (with the BOLD + CYAN ANSI prefix stripped by stripANSI before +// matching). When we see this, we flip to "inside-banner" mode. +const bannerStartMarker = "📊 INGESTION SUMMARY 📊" + +// bannerEndMarker is the equals-rule the ingestor prints AFTER +// the last metric line (a second `═`x60 line). We use it as the +// terminator so a parser at end-of-stream emits a complete +// Summary even if the Pod was killed mid-log-flush. +const bannerEndMarker = "════════════════════════════════════════════════════════════" + +// fieldPatterns are the regex patterns for each Summary field, +// keyed by the human-readable label the ingestor prints. The +// values are populated by NewSummaryParser via Compile-once-at- +// init. Each pattern matches a `Label: ` shape with +// optional spacing and an optional `,`-separated digit format +// (e.g. "1,234,567"). +// +// Maintained as a parallel slice of {label, pointer-target} +// rather than a map so the order of parsing matches the order +// the ingestor prints them — useful if a future ingestor adds a +// new line, the parser doesn't have to re-scan the existing ones. +var fieldPatterns = []struct { + prefix string + apply func(s *Summary, n int64) +}{ + {"📈 Total Records Found:", func(s *Summary, n int64) { s.TotalRecords = n }}, + {"✅ Successfully Processed:", func(s *Summary, n int64) { s.ProcessedRecords = n }}, + {"💾 Inserted to Database:", func(s *Summary, n int64) { s.InsertedRecords = n }}, + {"🚀 Sent to API:", func(s *Summary, n int64) { s.APISentRecords = n }}, + {"⏭️ Skipped Records:", func(s *Summary, n int64) { s.SkippedRecords = n }}, + {"📁 File Transfer Failures:", func(s *Summary, n int64) { s.FileTransferFailures = n }}, + {"❌ Failed DB Insertion:", func(s *Summary, n int64) { s.FailedRecords = n }}, +} + +// numberRE captures the trailing digit-group on a metric line. +// Allows optional thousands-separator commas the ingestor's +// `f"{count:,}"` formatting emits. +var numberRE = regexp.MustCompile(`([0-9][0-9,]*)\s*$`) + +// ingestorIDRE matches the ingestor-ID line specifically; the +// value is a UUID-ish string, not a number, so it gets its own +// pattern. +var ingestorIDRE = regexp.MustCompile(`Ingestor ID:\s*(.+?)\s*$`) + +// SummaryParser is a streaming parser for the 📊 banner. Feed it +// log lines as they arrive (any chunk size, any line splitting); +// Result() returns the accumulated Summary at any point. The +// banner-end marker latches the result so post-banner log lines +// don't perturb it. +// +// The parser is stateful but not thread-safe — the watch loop +// uses it from a single goroutine (the log-streaming TeeReader), +// so no synchronization needed. +type SummaryParser struct { + // buf accumulates partial-line input across Feed calls. The + // log stream from the API server arrives in TCP-sized chunks + // that may split lines; we buffer until we see a '\n' to + // finalize each line. + buf bytes.Buffer + + // summary is the accumulator. nil until we see the banner + // header — Result returns nil if the run never produced one. + summary *Summary + + // finalized latches when we see the banner-end marker. After + // that, additional Feed calls don't modify summary (the + // ingestor may keep logging after the banner, e.g. shutdown + // messages; those shouldn't perturb the result). + finalized bool + + // insideBanner is true between bannerStartMarker and + // bannerEndMarker. Outside this window, lines are ignored + // (so e.g. a stray emoji in earlier log output doesn't + // trigger spurious parsing). + insideBanner bool + + // sawAnyField latches once we successfully parse a + // fieldPatterns line (regardless of whether the count is + // zero). Used by feedLine to distinguish the opening ═-rule + // (no fields yet → ignore) from the closing one (fields + // already parsed → finalize). The earlier "any field + // non-zero" check failed on banners where every metric was + // genuinely 0 (early failure case) — Bugbot caught this on + // PR #10 round 2. + sawAnyField bool +} + +// NewSummaryParser returns an initialized parser. Caller's +// goroutine owns it for the duration of the watch loop. +func NewSummaryParser() *SummaryParser { + return &SummaryParser{} +} + +// Feed accepts arbitrary log bytes; the parser buffers and splits +// internally. Safe to call with partial lines, multiple lines, or +// empty input. +func (p *SummaryParser) Feed(b []byte) { + if p.finalized { + return + } + _, _ = p.buf.Write(b) + for { + idx := bytes.IndexByte(p.buf.Bytes(), '\n') + if idx < 0 { + // No complete line yet — wait for more input. + return + } + line := p.buf.Next(idx + 1) // consume up to and including '\n' + p.feedLine(string(bytes.TrimRight(line, "\n"))) + } +} + +// FlushLine forces parsing of any buffered partial-line content. +// Called at end-of-stream by the watch loop in case the Pod +// terminated without a final '\n' (rare but possible if the +// container's stdout was killed mid-write). +func (p *SummaryParser) FlushLine() { + if p.buf.Len() > 0 && !p.finalized { + p.feedLine(p.buf.String()) + p.buf.Reset() + } +} + +// feedLine parses a single line, ANSI-stripped. The state machine +// has three regions: +// +// - Pre-banner: skip until we see the start marker +// - Inside banner: match each line against fieldPatterns + the +// Ingestor ID line +// - End marker: latch and stop processing +func (p *SummaryParser) feedLine(rawLine string) { + line := stripANSI(rawLine) + if strings.Contains(line, bannerStartMarker) { + p.summary = &Summary{} + p.insideBanner = true + return + } + if !p.insideBanner { + return + } + // Banner-end check: a long row of '═'. Only count when we've + // already crossed the start marker (the start banner ALSO + // has a '═' rule before the header, which we want to ignore). + if strings.Contains(line, bannerEndMarker) { + // Two ═-rules in the banner: one immediately after the + // header, one at the very end. We use a simple counter + // to distinguish — first one we see while insideBanner + // is the post-header rule (skip), second is the + // post-metrics rule (finalize). + // + // Actually the simpler approach: check whether we've + // parsed any field yet. If so, this ═ is the closing + // rule; if not, it's the opening one. The ingestor + // always prints fields between the two rules. + if p.summary != nil && p.hasParsedAnyField() { + p.finalized = true + } + return + } + + // Ingestor ID line is the first content line in the banner. + if m := ingestorIDRE.FindStringSubmatch(line); m != nil { + p.summary.IngestorID = strings.TrimSpace(m[1]) + return + } + + // Otherwise: try each field pattern. The prefix match is + // linear over a 7-element slice — microscopic overhead per + // line, and the fixed order matches the ingestor's print + // order. + for _, fp := range fieldPatterns { + if !strings.Contains(line, fp.prefix) { + continue + } + m := numberRE.FindStringSubmatch(line) + if m == nil { + return + } + // Strip thousands-separator commas. + raw := strings.ReplaceAll(m[1], ",", "") + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return // malformed; ignore this line + } + fp.apply(p.summary, n) + // Latch regardless of value. An all-zero banner is a + // real shape (early failure: the ingestor still prints + // the summary structure with all counts at 0). Bugbot + // PR #10 r2 flagged the "non-zero" hasParsedAnyField + // check as a finalization hole for this case. + p.sawAnyField = true + return + } +} + +// hasParsedAnyField reports whether the parser has seen and +// applied at least one fieldPatterns line. Used to disambiguate +// the two ═-rules in the banner (see feedLine). +func (p *SummaryParser) hasParsedAnyField() bool { + return p.sawAnyField +} + +// Result returns the accumulated Summary. nil if the parser never +// saw a banner header (early failure, OOM before the ingestor got +// to print its results). Safe to call at any point — the parser +// returns the in-progress accumulator if not yet finalized. +func (p *SummaryParser) Result() *Summary { + return p.summary +} + +// RenderPanel returns a multi-line, customer-facing rendering of +// the Summary for display in the orchestrator's success/failure +// message. Format: +// +// ┌─ Ingestion summary ──────────────────────────┐ +// │ Ingestor ID: │ +// │ Total records: 1,234 │ +// │ Inserted: 1,200 │ +// │ Skipped: 4 │ +// │ File transfer failures: 0 │ +// │ DB-insert failures: 30 │ +// │ Success rate: 97.2% │ +// └──────────────────────────────────────────────┘ +// +// Uses box-drawing characters for visual structure. Plain ASCII +// fallback could be added in v0.2 for terminals that don't render +// Unicode (rare on modern OS X/Linux/Windows-Terminal). +func RenderPanel(s *Summary) string { + if s == nil { + return "" + } + const labelWidth = 26 + var b strings.Builder + b.WriteString("┌─ Ingestion summary ──────────────────────────┐\n") + row := func(label, value string) { + fmt.Fprintf(&b, "│ %-*s %s\n", labelWidth, label, value) + } + if s.IngestorID != "" { + row("Ingestor ID:", s.IngestorID) + } + row("Total records:", commaSep(s.TotalRecords)) + row("Inserted:", commaSep(s.InsertedRecords)) + row("Sent to API:", commaSep(s.APISentRecords)) + row("Skipped:", commaSep(s.SkippedRecords)) + row("File transfer failures:", commaSep(s.FileTransferFailures)) + row("DB-insert failures:", commaSep(s.FailedRecords)) + row("Success rate:", fmt.Sprintf("%.1f%%", s.SuccessRate())) + b.WriteString("└──────────────────────────────────────────────┘\n") + return b.String() +} + +// commaSep formats an int64 with thousands-separator commas to +// match the ingestor's own banner format. Pure Go, no x/text. +func commaSep(n int64) string { + s := strconv.FormatInt(n, 10) + if len(s) <= 3 { + return s + } + // Insert commas every 3 digits from the right. Handle the + // optional leading '-' by carving it off first. + neg := "" + if s[0] == '-' { + neg = "-" + s = s[1:] + } + var out []byte + for i, c := range []byte(s) { + if i > 0 && (len(s)-i)%3 == 0 { + out = append(out, ',') + } + out = append(out, c) + } + return neg + string(out) +} + +// (compile-time test: bufio + io are referenced via Feed's buffer) +var _ = bufio.ScanLines +var _ io.Writer = parserWriter{} // ensures parserWriter satisfies io.Writer diff --git a/internal/submit/summary_test.go b/internal/submit/summary_test.go new file mode 100644 index 0000000..5bfe950 --- /dev/null +++ b/internal/submit/summary_test.go @@ -0,0 +1,240 @@ +package submit + +import ( + "strings" + "testing" +) + +// realIngestorBanner mirrors what +// tracebloc_ingestor/ingestors/base.py:624+ actually prints, +// ANSI codes and all. The parser strips ANSI before matching so +// the test is bit-exact with the production logs. +// +// The leading "preamble" line + trailing "post-banner" lines +// simulate what real ingestor logs look like — the parser must +// ignore non-banner content + finalize on the closing rule. +var realIngestorBanner = "starting up...\n" + + "loaded 1234 rows from labels.csv\n" + + "\n" + + "\x1b[36m" + strings.Repeat("═", 60) + "\x1b[0m\n" + + "\x1b[1m\x1b[36m📊 INGESTION SUMMARY 📊\x1b[0m\n" + + "\x1b[36m" + strings.Repeat("═", 60) + "\x1b[0m\n" + + "\x1b[1mIngestor ID:\x1b[0m \x1b[34mrun-abc-123\x1b[0m\n" + + "\x1b[1m📈 Total Records Found:\x1b[0m \x1b[34m1,234\x1b[0m\n" + + "\x1b[1m✅ Successfully Processed:\x1b[0m \x1b[32m1,200\x1b[0m\n" + + "\x1b[1m💾 Inserted to Database:\x1b[0m \x1b[32m1,200\x1b[0m\n" + + "\x1b[1m🚀 Sent to API:\x1b[0m \x1b[32m1,150\x1b[0m\n" + + "\x1b[1m⏭️ Skipped Records:\x1b[0m \x1b[33m4\x1b[0m\n" + + "\x1b[1m📁 File Transfer Failures:\x1b[0m \x1b[32m0\x1b[0m\n" + + "\x1b[1m❌ Failed DB Insertion:\x1b[0m \x1b[31m30\x1b[0m\n" + + "\x1b[36m" + strings.Repeat("═", 60) + "\x1b[0m\n" + + "ingestor exiting cleanly\n" + +// TestSummaryParser_RealBannerEndToEnd pins the parser against +// the actual ingestor's output format. If a regression breaks +// any field's extraction, this test fails with a clear "got X +// want Y" for that specific counter. +func TestSummaryParser_RealBannerEndToEnd(t *testing.T) { + p := NewSummaryParser() + p.Feed([]byte(realIngestorBanner)) + + got := p.Result() + if got == nil { + t.Fatal("parser returned nil Result; expected populated Summary") + } + + cases := []struct { + name string + got any + want any + }{ + {"IngestorID", got.IngestorID, "run-abc-123"}, + {"TotalRecords", got.TotalRecords, int64(1234)}, + {"ProcessedRecords", got.ProcessedRecords, int64(1200)}, + {"InsertedRecords", got.InsertedRecords, int64(1200)}, + {"APISentRecords", got.APISentRecords, int64(1150)}, + {"SkippedRecords", got.SkippedRecords, int64(4)}, + {"FileTransferFailures", got.FileTransferFailures, int64(0)}, + {"FailedRecords", got.FailedRecords, int64(30)}, + } + for _, c := range cases { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.name, c.got, c.want) + } + } +} + +// TestSummaryParser_HasFailures pins the failure-detection logic +// that the orchestrator uses to choose between success exit code +// (0) and ingest-failure exit code (9). +func TestSummaryParser_HasFailures(t *testing.T) { + cases := []struct { + name string + s *Summary + want bool + }{ + {"nil", nil, false}, + {"all zero", &Summary{TotalRecords: 100, ProcessedRecords: 100}, false}, + {"file transfer failures", &Summary{FileTransferFailures: 1}, true}, + {"failed records", &Summary{FailedRecords: 1}, true}, + {"both", &Summary{FileTransferFailures: 1, FailedRecords: 1}, true}, + // Skipped records are NOT failures — they're rows that + // validators rejected. The customer wants to see the + // count but it doesn't change the exit code. + {"skipped is not failure", &Summary{SkippedRecords: 100}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.s.HasFailures(); got != c.want { + t.Errorf("HasFailures = %v, want %v", got, c.want) + } + }) + } +} + +// TestSummaryParser_SuccessRate pins the math that feeds the +// rendered panel's "Success rate: XX%" line. Divide-by-zero on +// empty banner is the critical edge case. +func TestSummaryParser_SuccessRate(t *testing.T) { + cases := []struct { + name string + s *Summary + want float64 + }{ + {"nil", nil, 0}, + {"empty banner", &Summary{}, 0}, + {"100%", &Summary{TotalRecords: 100, ProcessedRecords: 100}, 100}, + {"50%", &Summary{TotalRecords: 100, ProcessedRecords: 50}, 50}, + {"all failed", &Summary{TotalRecords: 100}, 0}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.s.SuccessRate(); got != c.want { + t.Errorf("SuccessRate = %v, want %v", got, c.want) + } + }) + } +} + +// TestSummaryParser_StreamedInChunks: real log streams arrive in +// arbitrary-sized chunks. Feed the banner one byte at a time to +// pin that the parser correctly buffers and finalizes despite +// partial-line input. +func TestSummaryParser_StreamedInChunks(t *testing.T) { + p := NewSummaryParser() + for i := 0; i < len(realIngestorBanner); i++ { + p.Feed([]byte{realIngestorBanner[i]}) + } + got := p.Result() + if got == nil { + t.Fatal("byte-by-byte feed produced nil Result") + } + if got.TotalRecords != 1234 || got.FailedRecords != 30 { + t.Errorf("chunked feed produced different result than monolithic: TotalRecords=%d FailedRecords=%d", + got.TotalRecords, got.FailedRecords) + } +} + +// TestSummaryParser_NoBanner: if the run died before producing a +// banner (image crashloop, OOM at startup), Result returns nil +// rather than an empty Summary. The orchestrator uses this +// nil-check to decide whether to render the panel at all. +func TestSummaryParser_NoBanner(t *testing.T) { + p := NewSummaryParser() + p.Feed([]byte("starting up...\nerror: connection refused\n")) + if got := p.Result(); got != nil { + t.Errorf("Result on no-banner log = %+v, want nil", got) + } +} + +// TestSummaryParser_PostBannerLogsIgnored: lines after the closing +// ═-rule are ignored. The ingestor may print shutdown messages +// after the summary; those shouldn't perturb the parsed counts +// (e.g. a regex-misfire interpreting "1234" in an unrelated log +// line as TotalRecords). +func TestSummaryParser_PostBannerLogsIgnored(t *testing.T) { + p := NewSummaryParser() + p.Feed([]byte(realIngestorBanner)) + pre := *p.Result() + p.Feed([]byte("📈 Total Records Found: 999999999\n")) // would alter TotalRecords if not finalized + post := *p.Result() + if pre != post { + t.Errorf("post-banner line altered Summary; pre=%+v post=%+v", pre, post) + } +} + +// TestStripANSI: the parser strips ANSI SGR codes from each line +// before matching. Validate the regex handles common shapes. +func TestStripANSI(t *testing.T) { + cases := []struct { + in, want string + }{ + {"plain text", "plain text"}, + {"\x1b[1mbold\x1b[0m", "bold"}, + {"\x1b[1;36mbold-cyan\x1b[0m", "bold-cyan"}, + {"prefix\x1b[31mred\x1b[0msuffix", "prefixredsuffix"}, + {"", ""}, + } + for _, c := range cases { + if got := stripANSI(c.in); got != c.want { + t.Errorf("stripANSI(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// TestRenderPanel_BasicShape: the panel rendering is what the +// customer sees on success; pin a few key lines so a refactor +// breaks the test rather than silently producing weird output. +func TestRenderPanel_BasicShape(t *testing.T) { + s := &Summary{ + IngestorID: "run-abc", + TotalRecords: 1234567, + InsertedRecords: 1200000, + APISentRecords: 1150000, + SkippedRecords: 4000, + FileTransferFailures: 30, + FailedRecords: 5, + } + got := RenderPanel(s) + for _, want := range []string{ + "Ingestion summary", + "run-abc", + "1,234,567", // commaSep formatting + "1,200,000", // commaSep formatting + "30", // file transfer failures + "DB-insert failures:", + } { + if !strings.Contains(got, want) { + t.Errorf("RenderPanel missing %q in:\n%s", want, got) + } + } +} + +// TestRenderPanel_Nil: nil summary returns empty string so the +// orchestrator can blind-print without a guard. +func TestRenderPanel_Nil(t *testing.T) { + if got := RenderPanel(nil); got != "" { + t.Errorf("RenderPanel(nil) = %q, want empty", got) + } +} + +// TestCommaSep: small helper test. Pin the boundary cases that +// would catch off-by-one in the comma-insertion loop. +func TestCommaSep(t *testing.T) { + cases := []struct { + in int64 + want string + }{ + {0, "0"}, + {999, "999"}, + {1000, "1,000"}, + {12345, "12,345"}, + {1234567, "1,234,567"}, + {-1234, "-1,234"}, + } + for _, c := range cases { + if got := commaSep(c.in); got != c.want { + t.Errorf("commaSep(%d) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/submit/watch.go b/internal/submit/watch.go new file mode 100644 index 0000000..3280e81 --- /dev/null +++ b/internal/submit/watch.go @@ -0,0 +1,491 @@ +package submit + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +// Watch-loop tunables. Both deliberately conservative — Phase 4's +// watching is the dominant time-spend of a typical push (the actual +// data move was done in Phase 3; what's left is the in-cluster +// ingestion which can take minutes-to-an-hour for large datasets). +const ( + // JobPollInterval is how often the watch loop re-Gets the Job. + // 2s is a sweet spot: human-perceptible enough that a + // 30-second ingestion has clean lifecycle output, light + // enough that an hour-long ingestion adds <2000 API calls + // (negligible at the apiserver's ~10k req/s ceiling). + JobPollInterval = 2 * time.Second + + // JobWatchTimeout is the absolute cap on a single + // dataset-push's watch phase. 1 hour is generous — typical + // image_classification ingestions finish in <10 min; cap + // exists to avoid infinite hangs if the cluster goes weird + // (kubelet stops reporting status, etc.). Customers running + // hours-long ingestions should use --detach. + JobWatchTimeout = 1 * time.Hour + + // PodPollInterval is how often we look for the ingestor Job's + // Pod once we have the Job name. Same 2s as the Job-level + // poll; same rationale. + PodPollInterval = 2 * time.Second + + // PodReadyTimeout caps how long the Job's Pod has to be + // schedulable + Running before we give up looking for it. + // 5 min covers image pull on a slow registry; beyond that + // the ingestion isn't going to start at all and the customer + // wants the diagnostic. + PodReadyTimeout = 5 * time.Minute +) + +// JobOutcome enumerates the terminal states the watch loop reports. +// The orchestrator (submit.go) maps these to exit codes. +type JobOutcome int + +const ( + // JobOutcomeUnknown is the zero value — never returned, used + // only as a switch-default sentinel. + JobOutcomeUnknown JobOutcome = iota + + // JobOutcomeSucceeded means the ingestor Job's Pod exited 0. + // Maps to exit code 0 in the orchestrator. + JobOutcomeSucceeded + + // JobOutcomeFailed means the ingestor Job's Pod exited + // non-zero (any cause: ingestion runtime error, OOM, image + // crashloop). Maps to the "ingest" exit code (9) in the + // orchestrator. The Pod-side summary banner may or may not + // have been printed depending on how far the run got — the + // orchestrator parses what it can. + JobOutcomeFailed + + // JobOutcomeDetached means the customer Ctrl-C'd mid-watch. + // jobs-manager already accepted the run; the cluster will + // continue without us. Maps to exit code 0 with a "reconnect + // with kubectl logs" hint in the orchestrator output. + JobOutcomeDetached +) + +func (o JobOutcome) String() string { + switch o { + case JobOutcomeSucceeded: + return "Succeeded" + case JobOutcomeFailed: + return "Failed" + case JobOutcomeDetached: + return "Detached" + default: + return "Unknown" + } +} + +// WatchResult bundles everything the orchestrator wants from the +// watch loop. Outcome drives the exit code; PodName is what the +// detach-hint prints; Summary is the structured INGESTION SUMMARY +// (nil if the run didn't produce one — early failure, OOM at +// startup, etc.). +type WatchResult struct { + Outcome JobOutcome + PodName string + + // Summary is the parsed 📊 banner, nil on early failure. + // The orchestrator decides whether to render the panel + // (success path) or include it in the failure framing + // (failed-after-summary path). + Summary *Summary + + // DetachReason qualifies the Detached outcome — set only + // when Outcome == JobOutcomeDetached. Lets the orchestrator + // print accurate diagnostics ("Detached on signal" vs + // "Pod didn't become Ready within timeout" vs "Watch cap + // exceeded") instead of the previous one-size-fits-all + // message. Bugbot PR #10 r7 flagged the misleading "signal" + // framing for the timeout-detach paths. + DetachReason DetachReason +} + +// DetachReason enumerates the conditions that produce a Detached +// outcome. Used by the orchestrator's diagnostic output only — +// the exit-code mapping treats all detach reasons as success (0) +// because the cluster keeps running regardless of why we stopped +// watching. +type DetachReason int + +const ( + // DetachReasonNone is the zero value, used when Outcome is + // not Detached. + DetachReasonNone DetachReason = iota + + // DetachReasonSignal: customer pressed Ctrl-C (or a parent + // process sent SIGTERM). The original Detach semantic. + DetachReasonSignal + + // DetachReasonPodWaitTimeout: PodReadyTimeout (5 min) + // exhausted before the ingestor Pod reached a useful + // phase. Slow image pull, scheduling backlog, PSA rejection. + DetachReasonPodWaitTimeout + + // DetachReasonWatchCap: JobWatchTimeout (1 hour) exceeded + // during log streaming. Long-running ingestion that + // outlasted the observation window. + DetachReasonWatchCap +) + +// WatchJob is the top-level watch loop: poll the Job until it +// reaches a terminal phase, stream the Pod's logs while it's +// running, and return a WatchResult. +// +// SIGINT contract (Bugbot-r9 echo for the previous package): +// the caller (cli/main.go via signal.NotifyContext) is expected +// to cancel ctx on Ctrl-C. WatchJob detects ctx.Err() == Canceled +// and returns Outcome=Detached rather than treating it as a poll +// failure. The customer who Ctrl-C'd during the watch sees the +// "your ingestion is still running in the cluster; reconnect with +// kubectl logs " hint. +// +// out is the customer-facing log stream (typically os.Stdout). +// Logs are written verbatim — no prefix, no munging — so the +// stream looks identical to `kubectl logs -f `. +func WatchJob( + ctx context.Context, + cs kubernetes.Interface, + namespace, jobName string, + out io.Writer, +) (*WatchResult, error) { + // Keep the customer's original ctx separately so finalJobStatus + // can derive a FRESH 30s context from it (rather than inheriting + // a possibly-depleted JobWatchTimeout). Bugbot PR #10 r2: the + // previous "wrap everything in JobWatchTimeout" approach starved + // finalJobStatus's budget when streaming used most of the hour, + // so a successful slow ingestion misreported as Unknown → exit 9. + customerCtx := ctx + + // JobWatchTimeout caps the pod-wait + log-stream phases (the + // time-spend dominant parts of the watch). finalJobStatus gets + // its own ctx below. + watchCtx, cancel := context.WithTimeout(customerCtx, JobWatchTimeout) + defer cancel() + + // 1. Wait for the ingestor Job's Pod to exist + reach Running. + // jobs-manager creates the Job and Kubernetes spawns the + // Pod asynchronously, so the Pod usually isn't there the + // moment after the 201 comes back. + podName, err := waitForJobPod(watchCtx, cs, namespace, jobName) + if err != nil { + if errors.Is(err, context.Canceled) { + // SIGINT before the Pod even appeared. jobs-manager + // has accepted the run, the cluster will run it, + // the CLI is just not watching anymore. + return &WatchResult{ + Outcome: JobOutcomeDetached, + DetachReason: DetachReasonSignal, + }, nil + } + // PodReadyTimeout (5min) exhausted = slow image pull / + // scheduling backlog / PSA still rejecting. The submit + // was accepted, the run will (eventually) execute in the + // cluster — the CLI just gave up observing within the + // timeout. Treat as Detached, not ingest-failed: bumping + // to exit 9 would falsely claim the ingestion failed. + // Bugbot PR #10 r5 flagged the false-positive exit code. + if errors.Is(err, context.DeadlineExceeded) { + return &WatchResult{ + Outcome: JobOutcomeDetached, + DetachReason: DetachReasonPodWaitTimeout, + }, nil + } + return nil, fmt.Errorf("waiting for ingestor Pod: %w", err) + } + + // 2. Stream Pod logs. This blocks until the Pod terminates + // or ctx is cancelled. We don't need a separate Job-status + // poll here because the Pod terminating drains the log + // stream — when GetLogs(Follow=true) returns EOF, the + // Pod has completed (success or failure). + // + // Any text the ingestor prints (including the 📊 banner + // at the end) flows verbatim through `out`. We also feed + // a side-channel to the summary parser so we end up with + // a structured representation of the banner without + // requiring a second log fetch post-completion. + summary, logErr := streamPodLogsAndParse(watchCtx, cs, namespace, podName, out) + // Filter out the two ctx-flavored errors — both are "observation + // gave up early," not "stream failed." They get classified below + // into Detached (customer SIGINT, JobWatchTimeout expiry). Any + // other error is a real streaming failure (network mid-stream, + // API server tantrum) and bubbles up as a watch error. + if logErr != nil && + !errors.Is(logErr, context.Canceled) && + !errors.Is(logErr, context.DeadlineExceeded) { + return nil, fmt.Errorf("streaming logs from Pod %s/%s: %w", namespace, podName, logErr) + } + + // 3. Detach branches: + // - customerCtx canceled = SIGINT + // - watchCtx expired (DeadlineExceeded) = JobWatchTimeout cap + // hit during streaming (1-hour observation window exceeded) + // + // Both are "the cluster keeps running; the CLI just gave up + // observing." Same UX as the PodReadyTimeout case from r5 + // above — exit 0 with the kubectl-logs reconnect hint. + // Bugbot PR #10 r6 flagged the inconsistency: r5 detached + // on PodReady timeout but the watch-cap exit still mapped + // to exit 9. + if errors.Is(customerCtx.Err(), context.Canceled) || errors.Is(watchCtx.Err(), context.DeadlineExceeded) { + reason := DetachReasonSignal + if errors.Is(watchCtx.Err(), context.DeadlineExceeded) && + !errors.Is(customerCtx.Err(), context.Canceled) { + // Pure watchCtx-only expiry = JobWatchTimeout. The + // customerCtx-canceled case takes precedence (if both + // fired, the customer's intent was SIGINT). + reason = DetachReasonWatchCap + } + return &WatchResult{ + Outcome: JobOutcomeDetached, + PodName: podName, + Summary: summary, // may be partial + DetachReason: reason, + }, nil + } + + // 4. Final status check with a FRESH 30s budget derived from + // the customer's ctx (not watchCtx, which may be near- + // expired after a long log stream). Bugbot PR #10 r2: + // inheriting watchCtx's depleted budget caused successful + // slow ingestions to misreport as Unknown. + // + // The fresh ctx still propagates SIGINT (parent is + // customerCtx, which carries signal.NotifyContext's + // cancel). If the customer Ctrl-C's during this 30s + // window, we fall into the detach branch below — same + // contract as during the log stream. + finalCtx, finalCancel := context.WithTimeout(customerCtx, 30*time.Second) + defer finalCancel() + outcome, err := finalJobStatus(finalCtx, cs, namespace, jobName) + if err != nil { + // Treat SIGINT during finalJobStatus as graceful detach + // (same as during the log stream — jobs-manager already + // accepted the run, the customer is just stopping the + // observation). Bugbot PR #10 r2 flagged the "exit 9 on + // post-stream SIGINT" inconsistency. + if errors.Is(customerCtx.Err(), context.Canceled) { + return &WatchResult{ + Outcome: JobOutcomeDetached, + PodName: podName, + Summary: summary, + DetachReason: DetachReasonSignal, + }, nil + } + return nil, fmt.Errorf("reading final Job status for %s/%s: %w", namespace, jobName, err) + } + return &WatchResult{ + Outcome: outcome, + PodName: podName, + Summary: summary, + }, nil +} + +// waitForJobPod polls until the Job has spawned its Pod and that +// Pod has reached Phase=Running. The selection key is the +// `job-name=` label that batch/v1 controllers attach to +// every Pod they create. +func waitForJobPod(ctx context.Context, cs kubernetes.Interface, namespace, jobName string) (string, error) { + var podName string + err := wait.PollUntilContextTimeout(ctx, PodPollInterval, PodReadyTimeout, true, + func(ctx context.Context) (bool, error) { + pods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "job-name=" + jobName, + }) + if err != nil { + // Terminal errors short-circuit (echoes the + // Phase-3 r3 fix in push/pod.go). + if apierrors.IsForbidden(err) { + return false, err + } + return false, nil // transient + } + if len(pods.Items) == 0 { + return false, nil // Pod hasn't been created yet + } + // Pick the MOST RECENT useful-phase Pod, not just + // items[0]. A Job with backoffLimit > 0 (or a Job + // where jobs-manager re-spawned the Pod for any + // reason) can have multiple Pods bearing the same + // `job-name=` label. The List API doesn't + // guarantee order, so items[0] could be the old + // Failed Pod from a prior retry instead of the + // current Running one. Bugbot PR #10 r4 caught this. + // + // "Useful phase" = Running (happy path) | Succeeded + // (fast-completing ingestion we missed) | Failed + // (terminated; we still want its logs). Pending Pods + // don't count — they have no logs to stream yet, so + // we keep polling until they either transition or + // become irrelevant. + var bestPod *corev1.Pod + for i := range pods.Items { + p := &pods.Items[i] + switch p.Status.Phase { + case corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed: + if bestPod == nil || + p.CreationTimestamp.After(bestPod.CreationTimestamp.Time) { + bestPod = p + } + } + } + if bestPod == nil { + return false, nil // all Pods still Pending + } + podName = bestPod.Name + return true, nil + }) + if err != nil { + return "", err + } + return podName, nil +} + +// streamPodLogsAndParse opens a streaming log read on the Pod and +// pipes it through (a) a TeeReader to `out` for verbatim display +// and (b) a Summary parser for the 📊 banner. Returns the parsed +// Summary (or nil if the Pod never produced one) + the underlying +// stream error. +// +// Streaming model: Follow=true means the API server keeps the +// connection open until the Pod terminates. When the Pod's +// container exits, the stream returns EOF and we drop out. This +// avoids the "poll twice" anti-pattern where Phase 4 would have to +// re-fetch logs after the Job is done to see the summary. +func streamPodLogsAndParse( + ctx context.Context, + cs kubernetes.Interface, + namespace, podName string, + out io.Writer, +) (*Summary, error) { + req := cs.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{ + Follow: true, + // Container omitted — Job Pods have exactly one container + // (the ingestor). If a future ingestor adds a sidecar + // (e.g. for metrics scrape), this needs to specify + // `Container: "ingestor"`. + }) + stream, err := req.Stream(ctx) + if err != nil { + return nil, err + } + defer func() { _ = stream.Close() }() + + // Wrap the stream in a TeeReader so each line flows through + // both customer-facing output AND the summary parser. The + // parser keeps a small ring buffer internally; it doesn't + // need to see the full stream in memory. + parser := NewSummaryParser() + tee := io.TeeReader(stream, parserWriter{parser: parser}) + + // Line-by-line copy so the customer sees output progressively. + // io.Copy would also work but would buffer chunks at the + // transport layer, making the output feel laggy on a fast + // ingestion. + scanner := bufio.NewScanner(tee) + // Default scanner buffer is 64 KB per line — fine for log + // lines but bump to 1 MB to handle the (rare) case where a + // single ingestion-error stacktrace has a multi-KB Python + // traceback line. + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := scanner.Bytes() + // Print the line + a newline (scanner strips the trailing + // '\n'). errcheck-friendly: we discard the writer error + // because the exit code is the customer-facing contract. + _, _ = out.Write(line) + _, _ = out.Write([]byte("\n")) + } + if err := scanner.Err(); err != nil { + // EOF is normal end-of-stream; other errors (network drop + // mid-stream, ctx cancel) get propagated. + if !errors.Is(err, io.EOF) { + // Flush any buffered partial line before returning so + // the parser sees content even on mid-line failure. + parser.FlushLine() + return parser.Result(), err + } + } + // Flush at the end of stream too. A Pod that exited without + // a trailing newline on its final stdout write would otherwise + // lose the final line — including potentially the closing + // ═-rule that finalizes the banner. Bugbot flagged on PR #10. + parser.FlushLine() + return parser.Result(), nil +} + +// parserWriter adapts a SummaryParser into an io.Writer for use +// with io.TeeReader. The TeeReader writes everything to its +// secondary sink as bytes flow through; the parser's Feed method +// accepts them line-by-line internally. +type parserWriter struct { + parser *SummaryParser +} + +func (pw parserWriter) Write(b []byte) (int, error) { + pw.parser.Feed(b) + return len(b), nil +} + +// finalJobStatus does a bounded poll on the Job's status to +// determine Succeeded vs Failed after log streaming ends. This is +// a separate step because the log-stream-end doesn't always race +// the Job-status-update; we need to wait briefly for the +// apiserver to post the terminal phase. +func finalJobStatus(ctx context.Context, cs kubernetes.Interface, namespace, jobName string) (JobOutcome, error) { + var outcome JobOutcome + err := wait.PollUntilContextTimeout(ctx, JobPollInterval, 30*time.Second, true, + func(ctx context.Context) (bool, error) { + job, err := cs.BatchV1().Jobs(namespace).Get(ctx, jobName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsForbidden(err) || apierrors.IsNotFound(err) { + return false, err + } + return false, nil + } + // batch/v1 Job conditions: Complete (success) or + // Failed. Poll until one is set; in practice this + // resolves within ~1s of log stream EOF. + for _, c := range job.Status.Conditions { + if c.Status != corev1.ConditionTrue { + continue + } + switch c.Type { + case batchv1.JobComplete: + outcome = JobOutcomeSucceeded + return true, nil + case batchv1.JobFailed: + outcome = JobOutcomeFailed + return true, nil + } + } + return false, nil + }) + if err != nil { + // If the poll timed out without seeing a terminal + // condition, the apiserver is being slow. Treat as + // Unknown rather than failing the whole push — the + // orchestrator can render a useful diagnostic from the + // streamed logs. + if errors.Is(err, context.DeadlineExceeded) { + return JobOutcomeUnknown, nil + } + return JobOutcomeUnknown, err + } + return outcome, nil +} diff --git a/internal/submit/watch_test.go b/internal/submit/watch_test.go new file mode 100644 index 0000000..2be8b05 --- /dev/null +++ b/internal/submit/watch_test.go @@ -0,0 +1,299 @@ +package submit + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// jobPod constructs a Pod owned by `jobName` via the standard +// batch/v1 job-name label. Used to seed the fake clientset for +// waitForJobPod tests. +func jobPod(name, jobName string, phase corev1.PodPhase) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "tracebloc", + Labels: map[string]string{"job-name": jobName}, + }, + Status: corev1.PodStatus{Phase: phase}, + } +} + +// TestWaitForJobPod_RunningPodSurfaces: a Pod with job-name label +// in Phase=Running is returned. Pin the label-selector contract + +// the happy-path return. +func TestWaitForJobPod_RunningPodSurfaces(t *testing.T) { + cs := fake.NewClientset(jobPod("ingestor-abc-xyz", "ingestor-abc", corev1.PodRunning)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + name, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor-abc") + if err != nil { + t.Fatalf("waitForJobPod: %v", err) + } + if name != "ingestor-abc-xyz" { + t.Errorf("name = %q, want ingestor-abc-xyz", name) + } +} + +// TestWaitForJobPod_PicksMostRecentNotFirst: Jobs with retries +// (or any multi-Pod scenario) produce multiple Pods with the +// same `job-name=` label. The List API doesn't guarantee +// order — picking items[0] could grab the old Failed Pod from a +// prior retry while the current Running one waits. Bugbot +// PR #10 r4 caught the missing tie-break. +func TestWaitForJobPod_PicksMostRecentNotFirst(t *testing.T) { + now := time.Now() + older := jobPod("ingestor-old-failed", "ingestor", corev1.PodFailed) + older.CreationTimestamp = metav1.NewTime(now.Add(-10 * time.Minute)) + + newer := jobPod("ingestor-new-running", "ingestor", corev1.PodRunning) + newer.CreationTimestamp = metav1.NewTime(now.Add(-1 * time.Minute)) + + cs := fake.NewClientset(older, newer) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + name, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor") + if err != nil { + t.Fatalf("waitForJobPod: %v", err) + } + if name != "ingestor-new-running" { + t.Errorf("name = %q, want ingestor-new-running "+ + "(most-recent useful-phase Pod, not items[0])", name) + } +} + +// TestWaitForJobPod_AllPendingKeepsPolling: if every Pod is still +// Pending (image pulling, scheduling), the function keeps polling +// rather than returning a Pending Pod's name (Pods with no log +// stream yet aren't useful to attach to). +func TestWaitForJobPod_AllPendingKeepsPolling(t *testing.T) { + cs := fake.NewClientset(jobPod("pending-1", "ingestor", corev1.PodPending)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor") + if err == nil { + t.Fatal("waitForJobPod returned nil on all-Pending; expected DeadlineExceeded") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("error doesn't wrap DeadlineExceeded: %v", err) + } +} + +// TestWaitForJobPod_FastCompletionPath: ingestions that finish +// faster than the poll interval might be in Phase=Succeeded by +// the time the watch loop checks. We still want the Pod's name so +// we can fetch its (post-mortem) logs. +func TestWaitForJobPod_FastCompletionPath(t *testing.T) { + cs := fake.NewClientset(jobPod("ingestor-fast", "ingestor", corev1.PodSucceeded)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + name, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor") + if err != nil { + t.Fatalf("waitForJobPod on Succeeded: %v", err) + } + if name != "ingestor-fast" { + t.Errorf("name = %q, want ingestor-fast", name) + } +} + +// TestWaitForJobPod_ForbiddenIsTerminal: an RBAC denial on +// `list pods` must short-circuit the wait — otherwise the +// customer sits through PodReadyTimeout for an error that +// doesn't change. +func TestWaitForJobPod_ForbiddenIsTerminal(t *testing.T) { + cs := fake.NewClientset() + cs.PrependReactor("list", "pods", + func(_ k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + corev1.Resource("pods"), "", + errors.New("user cannot list pods")) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := waitForJobPod(ctx, cs, "tracebloc", "j") + elapsed := time.Since(start) + if err == nil { + t.Fatal("waitForJobPod returned nil on Forbidden") + } + if elapsed > 3*time.Second { + t.Errorf("waitForJobPod waited %s on Forbidden; expected immediate return", elapsed) + } +} + +// TestWaitForJobPod_NoPodEverShows: ingestor Job that doesn't +// spawn its Pod (image pull stuck, scheduling impossible) hits +// the PodReadyTimeout. Bound the test's ctx so it doesn't wait +// the full 5min. +func TestWaitForJobPod_NoPodEverShows(t *testing.T) { + cs := fake.NewClientset() // empty + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := waitForJobPod(ctx, cs, "tracebloc", "missing-job") + if err == nil { + t.Fatal("waitForJobPod returned nil when no Pod ever appeared") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("error doesn't wrap DeadlineExceeded: %v", err) + } +} + +// jobWithCondition constructs a batch/v1 Job whose status reports +// the given terminal condition. Used to seed finalJobStatus tests. +func jobWithCondition(name string, cond batchv1.JobConditionType) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tracebloc"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{{ + Type: cond, + Status: corev1.ConditionTrue, + }}, + }, + } +} + +// TestFinalJobStatus_Complete: Job with Condition=Complete maps +// to JobOutcomeSucceeded. +func TestFinalJobStatus_Complete(t *testing.T) { + cs := fake.NewClientset(jobWithCondition("done", batchv1.JobComplete)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + out, err := finalJobStatus(ctx, cs, "tracebloc", "done") + if err != nil { + t.Fatalf("finalJobStatus: %v", err) + } + if out != JobOutcomeSucceeded { + t.Errorf("Outcome = %v, want Succeeded", out) + } +} + +// TestFinalJobStatus_Failed: Job with Condition=Failed maps to +// JobOutcomeFailed (drives the exit-9 "ingest failed" path). +func TestFinalJobStatus_Failed(t *testing.T) { + cs := fake.NewClientset(jobWithCondition("crashed", batchv1.JobFailed)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + out, err := finalJobStatus(ctx, cs, "tracebloc", "crashed") + if err != nil { + t.Fatalf("finalJobStatus: %v", err) + } + if out != JobOutcomeFailed { + t.Errorf("Outcome = %v, want Failed", out) + } +} + +// TestFinalJobStatus_TimeoutIsUnknown: if no terminal condition +// posts within 30s of the log stream ending, we return Unknown +// rather than blocking the customer forever. The orchestrator +// renders a useful diagnostic from the streamed logs. +func TestFinalJobStatus_TimeoutIsUnknown(t *testing.T) { + cs := fake.NewClientset(&batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "stuck", Namespace: "tracebloc"}, + // No conditions — Job in a weird mid-state. + }) + + // Tight ctx so the test doesn't take 30s. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + out, err := finalJobStatus(ctx, cs, "tracebloc", "stuck") + if err != nil { + t.Fatalf("finalJobStatus: %v", err) + } + if out != JobOutcomeUnknown { + t.Errorf("Outcome = %v, want Unknown", out) + } +} + +// TestWatchJob_PodWaitTimeoutMapsToDetach: when waitForJobPod +// exhausts its 5-min budget (slow image pull, PSA backlog), the +// submit has already been accepted by jobs-manager — the +// ingestion will run, the CLI just gave up watching within the +// timeout. WatchJob must return Outcome=Detached, not an error +// that bubbles up as exit 9 ("ingestion failed"). Bugbot PR #10 +// r5 caught the false-positive exit code. +func TestWatchJob_PodWaitTimeoutMapsToDetach(t *testing.T) { + cs := fake.NewClientset() // no Pod backing the job-name + + // Tight ctx so the test doesn't actually wait PodReadyTimeout (5m). + // The DeadlineExceeded from the parent ctx propagates the same + // way the inner PodReadyTimeout would, exercising the same + // errors.Is(DeadlineExceeded) branch in WatchJob. + ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) + defer cancel() + + var out bytes.Buffer + wr, err := WatchJob(ctx, cs, "tracebloc", "ingestor-stuck", &out) + if err != nil { + t.Fatalf("WatchJob returned error on Pod-wait timeout; want nil + Detached: %v", err) + } + if wr == nil { + t.Fatal("WatchJob returned nil result on Pod-wait timeout") + } + if wr.Outcome != JobOutcomeDetached { + t.Errorf("Outcome = %v, want Detached (the cluster keeps running; the CLI just gave up observing)", wr.Outcome) + } +} + +// TestJobOutcome_String: stringer pin so diagnostic output stays +// stable. +func TestJobOutcome_String(t *testing.T) { + cases := map[JobOutcome]string{ + JobOutcomeSucceeded: "Succeeded", + JobOutcomeFailed: "Failed", + JobOutcomeDetached: "Detached", + JobOutcomeUnknown: "Unknown", + } + for o, want := range cases { + if got := o.String(); got != want { + t.Errorf("%v.String() = %q, want %q", o, got, want) + } + } +} + +// TestParserWriter_FeedsParser: the io.Writer adapter that hooks +// the log-stream TeeReader to the SummaryParser. Pin that writes +// flow through to Feed correctly. +func TestParserWriter_FeedsParser(t *testing.T) { + p := NewSummaryParser() + pw := parserWriter{parser: p} + chunks := strings.Split(realIngestorBanner, "\n") + for _, line := range chunks { + _, _ = pw.Write([]byte(line + "\n")) + } + got := p.Result() + if got == nil { + t.Fatal("parserWriter didn't feed parser; Result is nil") + } + if got.TotalRecords != 1234 { + t.Errorf("TotalRecords = %d, want 1234 (via parserWriter)", got.TotalRecords) + } +} From 02cd3f355856ee19704acfbbeb2d4c33a8de7ab5 Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Mon, 25 May 2026 13:01:37 +0500 Subject: [PATCH 06/17] feat(release): GitHub Release workflow + install scripts + Homebrew formula (#153) (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(release): GitHub Release workflow, install scripts, Homebrew formula (#153) Closes the v0.1 epic (#147). Phase 5 is the distribution infrastructure that makes the CLI installable without `go install`. ## What lands in this PR ### .github/workflows/release.yml Triggered by `v*.*.*` tags (and workflow_dispatch for manual re-runs). Two-job pipeline: - `release` (matrix): cross-compiles linux/{amd64,arm64}, darwin/{amd64,arm64}, windows/amd64. Each binary gets a cosign keyless OIDC signature (.sig + .cert), per-platform SHA256, and a stamped version string via -ldflags. - `publish`: aggregates SHA256SUMS, stages the install scripts, creates the GitHub Release with auto-generated notes + every artifact attached. prerelease=true when the tag contains a hyphen (e.g. v0.1.0-rc1). A third `bump-homebrew-tap` job is wired but gated `if: false` until the tracebloc/homebrew-tap repo + HOMEBREW_TAP_TOKEN secret are set up. The path is: flip the gate, secret is added, formula gets updated automatically on each tag. RELEASE_CHECKLIST documents the one-time setup. ### scripts/install.sh POSIX-compatible (tested against dash, busybox sh, bash) — the customer's distro might not have bash. Detects OS+arch, resolves the latest tag via the /releases/latest redirect-trail (no rate-limited API call), downloads binary + SHA256SUMS, verifies the checksum, optionally verifies cosign signature if cosign is on PATH, installs to /usr/local/bin (falls back to ~/.local/bin with PATH advice if not writable). One-liner: `curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh` ### scripts/install.ps1 PowerShell 5.1+ (ships with Windows 10 21H1+). amd64 only for v0.1; arm64 errors with a clear "file an issue" pointer. Strict-mode + halt-on-error so half-installs don't happen. Same SHA256 + cosign verification surface as install.sh. Installs to $LOCALAPPDATA\Programs\tracebloc and PATH-adds at user scope. One-liner: `irm https://github.com/tracebloc/cli/releases/latest/download/install.ps1 | iex` ### scripts/homebrew-formula.rb.tmpl Formula template the bump-homebrew-tap job renders per release. Substitutes __VERSION__, __TAG__, and four per-platform __SHA_*__ placeholders via sed (no Ruby in the runner needed). Ships pre-built binaries rather than building from source — the k8s.io transitive dep tree is too heavy to build on customer laptops. ### scripts/RELEASE_CHECKLIST.md Per-release on-call doc. Distinguishes "what's automated" (everything in the workflow) from "what needs one-time setup" (homebrew-tap repo, eventually install.tracebloc.io DNS) from "what's manual per release" (the actual tag push + sanity-test of each install path). ## Hosting choice GitHub Release raw URLs (per user decision). Zero infrastructure needed; the bootstrap URL works the day v0.1.0 ships. install. tracebloc.io is a v0.2 follow-up via CNAME / Cloudflare worker. ## Out of this PR - Creating the tracebloc/homebrew-tap repo (separate; documented in RELEASE_CHECKLIST.md) - DNS for install.tracebloc.io (v0.2) - Tagging v0.1.0 (post-merge, after the Phase 6 dogfood) Local: vet, test -race -cover, gofmt -s, errcheck — all green. After this merges, the v0.1 epic (#147) is mechanically complete — `tag v0.1.0` produces the full release pipeline output. Co-Authored-By: Claude Opus 4.7 (1M context) * fix(install.sh): no sha256sum/shasum = hard error, not silent skip Bugbot PR #11 round 1: the previous flow printed "✓ checksum matches" unconditionally after the verify block, even when neither sha256sum nor shasum was on PATH (the no-tool branch set actual="" and the mismatch check guarded on -n "$actual", so it was silently skipped). Two failures in one: 1. Misleading log: claimed verification succeeded when it didn't run. 2. Security regression: an attacker could MITM the binary and the customer would never notice on a tool-missing host. Fix: refuse to install when no SHA256 tool is available. Error message lists the per-distro install commands so the customer knows the fix. Trade-off: a "warn-but-continue" mode would be friendlier on exotic hosts, but the whole point of this script is to verify a binary the customer pulls off the internet. "Friendly" defaults that skip the security check are exactly the install-script anti-pattern that's bitten the industry repeatedly. Hard error is the right call. `sh -n` syntax check clean. * fix(install): cosign-fail-installs (.ps1) + custom-prefix-ignored (.sh) Two Bugbot PR #11 r2 findings, both real: ## Medium — install.ps1: failed cosign verify treated as "skip" The previous flow's try-block contained BOTH the .sig/.cert download AND the cosign verify + Write-Error. With $ErrorActionPreference = 'Stop', Write-Error throws an exception caught by the SAME catch that handled missing-sig downloads — so a verified-failed binary went through the "couldn't download .sig/.cert — release may pre-date signing" branch and continued to install. Fix: separate the two concerns. Download in a try/catch (failures = "predates signing, skip"). Verify OUTSIDE the try via & cosign (external process, doesn't interact with $ErrorActionPreference); $LASTEXITCODE check + Write-Host + explicit exit 1 if non-zero. A failed signature now correctly refuses to install. ## Medium — install.sh: custom --prefix silently ignored if missing POSIX `test -w` returns false for nonexistent paths. The previous flow used `[ ! -w "$PREFIX" ]` to decide whether to fall back to ~/.local/bin — but that meant a legitimate `--prefix /opt/tracebloc` (a dir that doesn't exist yet) silently triggered the fallback, overriding the customer's explicit choice. Fix: try `mkdir -p "$PREFIX"` first; if THAT succeeds AND the resulting dir is -w, use it. Only fall back if mkdir fails (no write perms on parent) OR the existing dir is unwriteable. The dominant default-prefix case (`/usr/local/bin` without sudo) still routes to ~/.local/bin; custom prefixes get respected. Local: `sh -n scripts/install.sh` clean; full Go test suite + gofmt + errcheck green. * fix(install.ps1): Fail helper for clean errors + null-PATH guard (Bugbot r3) Two more PowerShell-specific Bugbot findings on PR #11: ## Medium — Write-Error + exit 1 = dead exit, ugly UX With $ErrorActionPreference = 'Stop', every Write-Error call throws a terminating error BEFORE any subsequent statement runs, so the `exit 1` lines after them were dead code. The result: a customer hitting an error saw PowerShell's full error record (stack trace + script line numbers + ErrorCategoryInfo) instead of a clean one-line message. The cosign branch was fixed in r2 with Write-Host + explicit exit. The earlier error paths (Get-Arch, Resolve-Tag, SHA256 mismatch, missing SHA256SUMS entry) still used the broken Write-Error pattern. Fix: add a Fail helper that does `Write-Host "Error: $msg" -ForegroundColor Red; exit 1` and replace every Write-Error call site with `Fail "..."`. DRY + consistent UX. ## Medium-Security — Null user PATH = leading semicolon = PATH-injection GetEnvironmentVariable('Path', 'User') returns $null on fresh Windows installs (or accounts that never set a user-scope PATH). The naive `"$userPath;$InstallPrefix"` interpolation then produced `";C:\Users\...\Programs\tracebloc"` — a leading semicolon = empty PATH entry, which Windows resolves as the CURRENT WORKING DIRECTORY. That's a well-known PATH-injection vector: any binary planted in the user's cwd runs ahead of real PATH entries. Fix: null-guard $userPath before concatenation. If $userPath is falsy (null or empty), use just $InstallPrefix as the new value. The `$existingEntries -notcontains` check now also handles the null case correctly via the `if ($userPath) { ... } else { @() }` fallback. Local: go test green, gofmt + errcheck clean. This is r3 on PR #11 — the install scripts have surfaced the most findings because they're shell+PowerShell where bugbot's language-specific coverage is sharpest. Each finding has been real. --------- Co-authored-by: Claude Opus 4.7 (1M context) --- .github/workflows/release.yml | 272 +++++++++++++++++++++++++++++++ scripts/RELEASE_CHECKLIST.md | 107 ++++++++++++ scripts/homebrew-formula.rb.tmpl | 63 +++++++ scripts/install.ps1 | 221 +++++++++++++++++++++++++ scripts/install.sh | 272 +++++++++++++++++++++++++++++++ 5 files changed, 935 insertions(+) create mode 100644 .github/workflows/release.yml create mode 100644 scripts/RELEASE_CHECKLIST.md create mode 100644 scripts/homebrew-formula.rb.tmpl create mode 100644 scripts/install.ps1 create mode 100755 scripts/install.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..bee6109 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,272 @@ +# Build, sign, and publish the official tracebloc CLI binaries. +# +# Triggered by pushing a vX.Y.Z tag (or manually via +# workflow_dispatch). Produces: +# +# - Cross-compiled binaries for linux/{amd64,arm64}, +# darwin/{amd64,arm64}, windows/amd64 +# - Cosign keyless signature for each binary (Sigstore Rekor; +# same machinery as tracebloc/data-ingestors' image release) +# - SHA256SUMS file covering every binary +# - install.sh + install.ps1 attached so the bootstrap URL +# `https://github.com/tracebloc/cli/releases/latest/download/install.sh` +# resolves to the right script for that release +# - GitHub Release with all of the above attached + release-notes +# body generated from CHANGELOG.md +# - Optional: Homebrew tap formula bump (gated on the +# HOMEBREW_TAP_TOKEN secret being present — uncommenting that +# secret in repo settings turns the bump on for future releases) +# +# Verification (one-liner customers run to check the signature): +# +# cosign verify-blob \ +# --certificate-identity-regexp \ +# 'https://github.com/tracebloc/cli/.github/workflows/release.yml@.*' \ +# --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ +# --certificate .cert \ +# --signature .sig \ +# + +name: Release + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + inputs: + ref: + description: 'Tag to build (e.g. v0.1.0). Must already exist on origin.' + required: true + type: string + +permissions: + contents: write # create / update the GitHub Release + id-token: write # cosign keyless OIDC + +jobs: + release: + name: Build + sign + publish + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: amd64 + - os: linux + arch: arm64 + - os: darwin + arch: amd64 + - os: darwin + arch: arm64 + - os: windows + arch: amd64 + ext: .exe + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + with: + cosign-release: 'v2.4.1' + + - name: Determine release version + id: version + run: | + # On a tag push: github.ref_name = "v0.1.0" + # On workflow_dispatch: inputs.ref = "v0.1.0" + REF="${{ inputs.ref || github.ref_name }}" + # Strip the leading v for use in -X main.version + VERSION="${REF#v}" + echo "tag=$REF" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Building $REF (version=$VERSION)" + + - name: Build binary + env: + GOOS: ${{ matrix.os }} + GOARCH: ${{ matrix.arch }} + CGO_ENABLED: '0' + run: | + mkdir -p dist + BIN_NAME="tracebloc-${{ steps.version.outputs.tag }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}" + go build \ + -trimpath \ + -ldflags "\ + -s -w \ + -X main.version=${{ steps.version.outputs.version }} \ + -X main.gitSHA=${GITHUB_SHA:0:12} \ + -X main.buildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -o "dist/$BIN_NAME" \ + ./cmd/tracebloc + echo "Binary built: dist/$BIN_NAME" + ls -lh "dist/$BIN_NAME" + + - name: Sign binary with cosign (keyless) + env: + # Required for keyless signing in GHA. The id-token: write + # permission above grants the workflow an OIDC token Sigstore + # validates against github.com's well-known issuer. + COSIGN_YES: 'true' + run: | + BIN_NAME="tracebloc-${{ steps.version.outputs.tag }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}" + cd dist + cosign sign-blob \ + --output-certificate "$BIN_NAME.cert" \ + --output-signature "$BIN_NAME.sig" \ + "$BIN_NAME" + echo "Signed: $BIN_NAME" + ls -lh "$BIN_NAME"* + + - name: Compute SHA256 + run: | + BIN_NAME="tracebloc-${{ steps.version.outputs.tag }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}" + cd dist + # Per-binary SHA256 file. The release step aggregates them + # into a single SHA256SUMS that the install.sh script reads. + sha256sum "$BIN_NAME" > "$BIN_NAME.sha256" + cat "$BIN_NAME.sha256" + + - name: Upload per-matrix artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.os }}-${{ matrix.arch }} + path: dist/ + retention-days: 7 + if-no-files-found: error + + publish: + name: Aggregate + create GitHub Release + runs-on: ubuntu-latest + needs: release + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Download all matrix artifacts + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: Aggregate SHA256SUMS + run: | + cd dist + # Combine every per-binary .sha256 into one canonical file. + # The install scripts download SHA256SUMS + their target binary, + # grep the line for that binary, and verify locally. + cat tracebloc-*.sha256 > SHA256SUMS + rm tracebloc-*.sha256 + echo "Aggregated SHA256SUMS:" + cat SHA256SUMS + + - name: Stage install scripts into the release + run: | + # The install.sh / install.ps1 ship inside the release so + # `releases/latest/download/install.sh` resolves stably (the + # raw scripts/ path would require knowing the tag). + cp scripts/install.sh dist/ + cp scripts/install.ps1 dist/ + + - name: Determine release tag + id: tag + run: | + REF="${{ inputs.ref || github.ref_name }}" + echo "tag=$REF" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.tag }} + name: ${{ steps.tag.outputs.tag }} + generate_release_notes: true + # Mark as prerelease for v*.*.* tags containing - (e.g. + # v0.1.0-rc1). Plain semver releases are stable. + prerelease: ${{ contains(steps.tag.outputs.tag, '-') }} + files: | + dist/tracebloc-* + dist/SHA256SUMS + dist/install.sh + dist/install.ps1 + + # Phase 5 follow-up: bump the Homebrew tap formula on each + # release. Disabled by default — flip the `if:` to `true` (and + # add a HOMEBREW_TAP_TOKEN repo secret with write access to + # tracebloc/homebrew-tap) when that repo exists. + bump-homebrew-tap: + name: Bump Homebrew tap formula + runs-on: ubuntu-latest + needs: publish + if: false # ← flip when tracebloc/homebrew-tap is set up + steps: + - name: Checkout cli (for the formula template) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Determine release tag + id: tag + run: | + REF="${{ inputs.ref || github.ref_name }}" + echo "tag=$REF" >> $GITHUB_OUTPUT + echo "version=${REF#v}" >> $GITHUB_OUTPUT + + - name: Download SHA256SUMS from the release + run: | + curl -fsSL \ + "https://github.com/tracebloc/cli/releases/download/${{ steps.tag.outputs.tag }}/SHA256SUMS" \ + -o SHA256SUMS + # Extract the per-platform shasums the formula needs. + DARWIN_AMD64_SHA=$(grep "darwin-amd64" SHA256SUMS | awk '{print $1}') + DARWIN_ARM64_SHA=$(grep "darwin-arm64" SHA256SUMS | awk '{print $1}') + LINUX_AMD64_SHA=$(grep "linux-amd64" SHA256SUMS | awk '{print $1}') + LINUX_ARM64_SHA=$(grep "linux-arm64" SHA256SUMS | awk '{print $1}') + echo "darwin_amd64_sha=$DARWIN_AMD64_SHA" >> $GITHUB_OUTPUT + echo "darwin_arm64_sha=$DARWIN_ARM64_SHA" >> $GITHUB_OUTPUT + echo "linux_amd64_sha=$LINUX_AMD64_SHA" >> $GITHUB_OUTPUT + echo "linux_arm64_sha=$LINUX_ARM64_SHA" >> $GITHUB_OUTPUT + id: shasums + + - name: Render formula from template + run: | + mkdir -p out/Formula + # Substitute placeholders in the formula template. Pure + # sed — no Ruby installation needed in the runner. + sed \ + -e "s/__VERSION__/${{ steps.tag.outputs.version }}/g" \ + -e "s/__TAG__/${{ steps.tag.outputs.tag }}/g" \ + -e "s/__SHA_DARWIN_AMD64__/${{ steps.shasums.outputs.darwin_amd64_sha }}/g" \ + -e "s/__SHA_DARWIN_ARM64__/${{ steps.shasums.outputs.darwin_arm64_sha }}/g" \ + -e "s/__SHA_LINUX_AMD64__/${{ steps.shasums.outputs.linux_amd64_sha }}/g" \ + -e "s/__SHA_LINUX_ARM64__/${{ steps.shasums.outputs.linux_arm64_sha }}/g" \ + scripts/homebrew-formula.rb.tmpl > out/Formula/tracebloc.rb + echo "--- Rendered formula ---" + cat out/Formula/tracebloc.rb + + - name: Push to tracebloc/homebrew-tap + env: + GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + run: | + git clone https://x-access-token:${GH_TOKEN}@github.com/tracebloc/homebrew-tap.git tap + cp out/Formula/tracebloc.rb tap/Formula/tracebloc.rb + cd tap + git config user.name "tracebloc-cli-release-bot" + git config user.email "bot@tracebloc.io" + git add Formula/tracebloc.rb + git commit -m "tracebloc ${{ steps.tag.outputs.tag }}" + git push diff --git a/scripts/RELEASE_CHECKLIST.md b/scripts/RELEASE_CHECKLIST.md new file mode 100644 index 0000000..2011431 --- /dev/null +++ b/scripts/RELEASE_CHECKLIST.md @@ -0,0 +1,107 @@ +# Release checklist + +The release workflow is fully automated — pushing a `v*.*.*` tag +triggers it. This document covers the per-release manual steps that +either ARE or AREN'T automated, so the on-call engineer doesn't +have to reverse-engineer the surface area on release day. + +## What runs automatically on `git push origin v0.1.0` + +1. `.github/workflows/release.yml` fires. +2. Matrix build of 5 platform binaries (linux/{amd64,arm64}, + darwin/{amd64,arm64}, windows/amd64). +3. Each binary is signed with cosign keyless OIDC; the per-binary + `.cert` + `.sig` files are produced. +4. SHA256SUMS file is aggregated across the matrix. +5. `install.sh` + `install.ps1` from `scripts/` are staged into + the release. +6. A GitHub Release is created with the tag, auto-generated notes, + and all artifacts attached. `prerelease=true` if the tag + contains a `-` (e.g. `v0.1.0-rc1`). +7. (If the `bump-homebrew-tap` job is enabled — see below) + the Homebrew tap formula in `tracebloc/homebrew-tap` is + updated with the new version + per-platform SHAs. + +## What needs manual setup ONCE (not per-release) + +These steps land outside the `tracebloc/cli` repo and are NOT +on the per-release checklist; they're foundational. + +### Homebrew tap repo + +1. Create `tracebloc/homebrew-tap` (must use the `homebrew-` + prefix for `brew tap tracebloc/tap` to resolve). +2. Add an initial `Formula/tracebloc.rb` (the release workflow + will overwrite this on the first tag — any valid-Ruby + placeholder is fine). +3. Mint a fine-grained PAT with `Contents: read+write` on the + tap repo. Save as `HOMEBREW_TAP_TOKEN` repo secret on + `tracebloc/cli`. +4. Flip `bump-homebrew-tap`'s `if: false` to `if: true` in + `release.yml`. + +### Vanity install URL (deferred) + +`install.tracebloc.io` is the eventual customer-facing URL but +isn't on the v0.1 critical path — the GitHub Release raw URL +(`https://github.com/tracebloc/cli/releases/latest/download/install.sh`) +serves v0.1 customers. Migrating to the vanity URL is a v0.2 +follow-up (DNS + CNAME or Cloudflare worker). + +## Per-release manual steps + +The release-cutter runs these on tag day. + +### 1. Pre-flight + +- [ ] All v0.1 phase tickets closed (`#147-#153`) +- [ ] `develop` is green on CI + Bugbot +- [ ] Local smoke: `go test -race ./...` passes +- [ ] Real EKS smoke: `tracebloc dataset push ./cats-dogs ...` + end-to-end reports the expected row count + +### 2. Tag + push + +```bash +# From develop, with a clean working tree +git tag -a v0.1.0 -m "tracebloc CLI v0.1.0" +git push origin v0.1.0 +``` + +The workflow takes 5-10 minutes. Monitor at +`https://github.com/tracebloc/cli/actions`. + +### 3. Verify the release + +- [ ] All 5 binaries attached to the GitHub Release +- [ ] SHA256SUMS file present +- [ ] install.sh + install.ps1 present +- [ ] Each binary has a `.cert` + `.sig` pair +- [ ] (If tap is set up) the `tracebloc/homebrew-tap` repo has + a new commit bumping `Formula/tracebloc.rb` + +### 4. Sanity-test each install path on a clean host + +```bash +# Linux: +curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh +tracebloc version + +# macOS: +curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh +# OR (if tap is set up): +brew install tracebloc/tap/tracebloc +tracebloc version + +# Windows (PowerShell): +irm https://github.com/tracebloc/cli/releases/latest/download/install.ps1 | iex +tracebloc version +``` + +### 5. Announce + +- [ ] Bump `images.ingestor.digest` in `tracebloc/client` if a new + ingestor release is coupled to this CLI release +- [ ] Update `README.md` install instructions to reference the new + version (or leave at `latest` — preferred) +- [ ] Post in the team channel + customer Slack diff --git a/scripts/homebrew-formula.rb.tmpl b/scripts/homebrew-formula.rb.tmpl new file mode 100644 index 0000000..b755b6a --- /dev/null +++ b/scripts/homebrew-formula.rb.tmpl @@ -0,0 +1,63 @@ +# Homebrew formula template for the tracebloc CLI. +# +# The release workflow (release.yml > bump-homebrew-tap) renders this +# template per release by substituting: +# +# __VERSION__ → the bare semver (e.g. "0.1.0") +# __TAG__ → the full tag (e.g. "v0.1.0") +# __SHA_DARWIN_AMD64__ → SHA256 of the darwin/amd64 binary +# __SHA_DARWIN_ARM64__ → SHA256 of the darwin/arm64 binary +# __SHA_LINUX_AMD64__ → SHA256 of the linux/amd64 binary +# __SHA_LINUX_ARM64__ → SHA256 of the linux/arm64 binary +# +# The rendered file is committed to tracebloc/homebrew-tap as +# Formula/tracebloc.rb. Customers install via: +# +# brew install tracebloc/tap/tracebloc +# +# Brew's `tap/tracebloc` shorthand maps to github.com/tracebloc/homebrew-tap +# automatically (the `homebrew-` prefix is brew convention). +class Tracebloc < Formula + desc "Declarative data ingestion CLI for the tracebloc platform" + homepage "https://github.com/tracebloc/cli" + version "__VERSION__" + license "Apache-2.0" + + # Per-platform bottle: brew picks the right one for the host OS+arch + # automatically. We ship pre-built binaries (not source) because the + # CLI's Go module pulls ~80MB of k8s.io transitive deps — source + # builds would be uncomfortably slow on customer laptops. + on_macos do + if Hardware::CPU.arm? + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-darwin-arm64" + sha256 "__SHA_DARWIN_ARM64__" + else + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-darwin-amd64" + sha256 "__SHA_DARWIN_AMD64__" + end + end + + on_linux do + if Hardware::CPU.arm? + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-linux-arm64" + sha256 "__SHA_LINUX_ARM64__" + else + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-linux-amd64" + sha256 "__SHA_LINUX_AMD64__" + end + end + + def install + # The release artifact is a bare binary, not an archive. Just + # rename it to "tracebloc" and drop it into HOMEBREW_PREFIX/bin. + bin.install Dir["*"].first => "tracebloc" + end + + test do + # Sanity smoke: brew runs this on every install if --with-test is + # set. The version subcommand exits 0 and prints SOMETHING with + # the version in it — that's enough to confirm the binary's not + # corrupted. + assert_match version.to_s, shell_output("#{bin}/tracebloc version") + end +end diff --git a/scripts/install.ps1 b/scripts/install.ps1 new file mode 100644 index 0000000..4b41768 --- /dev/null +++ b/scripts/install.ps1 @@ -0,0 +1,221 @@ +# tracebloc CLI installer for Windows (PowerShell 5.1+). +# +# Usage: +# irm https://github.com/tracebloc/cli/releases/latest/download/install.ps1 | iex +# # Or pin a version: +# $env:RELEASE_VERSION='v0.1.0'; irm | iex +# +# What it does: +# 1. Detects arch (amd64 only on Windows; arm64 not yet shipped) +# 2. Resolves the latest release tag (or honors $env:RELEASE_VERSION) +# 3. Downloads tracebloc--windows-amd64.exe + SHA256SUMS +# 4. Verifies SHA256 +# 5. (Optional) Verifies cosign signature if cosign.exe is on PATH +# 6. Installs to $env:USERPROFILE\AppData\Local\Programs\tracebloc\tracebloc.exe +# and PATH-adds it via user-scope env var +# +# PowerShell 5.1 is the floor (ships with Windows 10 21H1+). PS7+ also +# works. Older PS versions miss `Invoke-WebRequest -UseBasicParsing`'s +# default, but the script forces it. + +# Strict mode + halt on errors — the customer sees a clear error if +# anything fails, not a half-installed binary. +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Fail prints a clean error message + exits. With Stop preference, +# Write-Error throws a terminating error BEFORE any subsequent exit +# runs, surfacing as a verbose PowerShell error record with stack +# trace + line numbers — confusing for a customer-facing installer. +# Bugbot PR #11 r3 flagged the original Write-Error + exit 1 +# patterns as both dead code (exit never reached) AND ugly UX. +# Fail's Write-Host + explicit exit produces a one-line red error +# the customer can actually act on. +function Fail([string]$msg) { + Write-Host "Error: $msg" -ForegroundColor Red + exit 1 +} + +# --------------------------------------------------------------------- +# Config knobs (env-overridable, mirrors install.sh). +# --------------------------------------------------------------------- +$ReleaseVersion = if ($env:RELEASE_VERSION) { $env:RELEASE_VERSION } else { 'latest' } +$InstallPrefix = if ($env:INSTALL_PREFIX) { $env:INSTALL_PREFIX } ` + else { Join-Path $env:LOCALAPPDATA 'Programs\tracebloc' } +$GitHubRepo = 'tracebloc/cli' +$BinaryName = 'tracebloc.exe' + +# --------------------------------------------------------------------- +# Detect arch. +# --------------------------------------------------------------------- +function Get-Arch { + # PROCESSOR_ARCHITECTURE is the canonical Windows arch env var: + # AMD64 → x64 binary needed + # ARM64 → Windows on ARM (not yet shipped — see below) + $proc = $env:PROCESSOR_ARCHITECTURE + switch ($proc) { + 'AMD64' { return 'amd64' } + 'ARM64' { + Fail "Windows ARM64 binary isn't released yet. File an issue at https://github.com/tracebloc/cli if you need it." + } + default { + Fail "Unsupported PROCESSOR_ARCHITECTURE: $proc" + } + } +} + +$arch = Get-Arch + +# --------------------------------------------------------------------- +# Resolve the release tag if "latest". +# --------------------------------------------------------------------- +function Resolve-Tag { + if ($script:ReleaseVersion -ne 'latest') { + return $script:ReleaseVersion + } + # Follow the /releases/latest redirect to find the tag, same trick + # install.sh uses. -MaximumRedirection 0 makes Invoke-WebRequest + # surface the Location header instead of following. + try { + $resp = Invoke-WebRequest ` + -Uri "https://github.com/$script:GitHubRepo/releases/latest" ` + -MaximumRedirection 0 ` + -UseBasicParsing ` + -ErrorAction SilentlyContinue + } catch { + # PowerShell treats 3xx as an error when MaximumRedirection=0. + # The Exception.Response carries the redirect we want. + $resp = $_.Exception.Response + } + $loc = $null + if ($resp -and $resp.Headers) { + # PS5.1: Headers is a Dictionary[string,string]; PS7: a + # HttpResponseHeaders that needs different access. Try both. + try { $loc = $resp.Headers['Location'] } catch {} + if (-not $loc) { try { $loc = $resp.Headers.Location } catch {} } + } + if (-not $loc) { + Fail "Couldn't resolve the 'latest' release tag from GitHub. Pass `$env:RELEASE_VERSION explicitly." + } + # Location: https://github.com/tracebloc/cli/releases/tag/vX.Y.Z + return Split-Path -Leaf $loc +} + +$tag = Resolve-Tag +Write-Host "Installing tracebloc CLI $tag (windows/$arch)..." + +# --------------------------------------------------------------------- +# Download artifacts. +# --------------------------------------------------------------------- +$binaryFile = "tracebloc-$tag-windows-$arch.exe" +$baseUrl = "https://github.com/$GitHubRepo/releases/download/$tag" +$tmpDir = New-Item -ItemType Directory -Path (Join-Path $env:TEMP "tracebloc-install-$tag-$([guid]::NewGuid())") -Force + +try { + Write-Host "Downloading binary..." + Invoke-WebRequest -Uri "$baseUrl/$binaryFile" -OutFile (Join-Path $tmpDir $binaryFile) -UseBasicParsing + + Write-Host "Downloading SHA256SUMS..." + Invoke-WebRequest -Uri "$baseUrl/SHA256SUMS" -OutFile (Join-Path $tmpDir 'SHA256SUMS') -UseBasicParsing + + # ------------------------------------------------------------- + # Verify SHA256. + # ------------------------------------------------------------- + Write-Host "Verifying SHA256..." + $sumsContent = Get-Content (Join-Path $tmpDir 'SHA256SUMS') + $expected = ($sumsContent | Where-Object { $_ -match " $([regex]::Escape($binaryFile))$" } | + Select-Object -First 1) -replace ' .*$','' + if (-not $expected) { + Fail "SHA256SUMS doesn't contain an entry for $binaryFile — release artifacts may be incomplete." + } + $actual = (Get-FileHash -Algorithm SHA256 -Path (Join-Path $tmpDir $binaryFile)).Hash.ToLower() + if ($actual -ne $expected) { + Fail "SHA256 mismatch!`n expected: $expected`n actual: $actual`n refusing to install." + } + Write-Host " ✓ checksum matches" + + # ------------------------------------------------------------- + # Cosign signature verification (optional). + # ------------------------------------------------------------- + if (Get-Command cosign -ErrorAction SilentlyContinue) { + Write-Host "Verifying cosign signature..." + # Separate "download .sig/.cert" (recoverable if absent — old + # releases predate signing) from "verify the downloaded sig" + # (NOT recoverable — a failed verification means the binary + # is potentially tampered, refuse to install). Bugbot PR #11 + # caught the prior structure: with $ErrorActionPreference = + # 'Stop', Write-Error inside the try-block was thrown and + # caught by the same catch that handled missing-sig, so a + # failed verify silently downgraded to "skip + continue." + $sigDownloaded = $false + try { + Invoke-WebRequest -Uri "$baseUrl/$binaryFile.sig" -OutFile (Join-Path $tmpDir "$binaryFile.sig") -UseBasicParsing + Invoke-WebRequest -Uri "$baseUrl/$binaryFile.cert" -OutFile (Join-Path $tmpDir "$binaryFile.cert") -UseBasicParsing + $sigDownloaded = $true + } catch { + Write-Host " ⚠ couldn't download .sig/.cert — release may pre-date signing." + } + if ($sigDownloaded) { + # Verify OUTSIDE the try/catch: a non-zero $LASTEXITCODE + # from cosign is a hard refusal, not a swallowed + # exception. & invokes cosign as an external process, + # which doesn't interact with $ErrorActionPreference. + & cosign verify-blob ` + --certificate-identity-regexp "https://github.com/$GitHubRepo/.github/workflows/release.yml@.*" ` + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' ` + --certificate (Join-Path $tmpDir "$binaryFile.cert") ` + --signature (Join-Path $tmpDir "$binaryFile.sig") ` + (Join-Path $tmpDir $binaryFile) 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Host "Error: cosign signature verification FAILED — refusing to install." -ForegroundColor Red + exit 1 + } + Write-Host " ✓ cosign signature valid" + } + } else { + Write-Host " (cosign not installed; SHA256 verified, signature skipped)" + } + + # ------------------------------------------------------------- + # Install to $InstallPrefix. + # ------------------------------------------------------------- + New-Item -ItemType Directory -Path $InstallPrefix -Force | Out-Null + $target = Join-Path $InstallPrefix $BinaryName + Move-Item -Path (Join-Path $tmpDir $binaryFile) -Destination $target -Force + + Write-Host "" + Write-Host "✓ tracebloc CLI installed: $target" + Write-Host "" + Write-Host "Verify with:" + Write-Host " $target version" + + # PATH advice. User-scope PATH edit so this survives reboots. + # + # Null-guard the existing $userPath before concatenation. On + # fresh Windows installs (or accounts that never set user-scope + # PATH), GetEnvironmentVariable returns $null. The naive + # `"$userPath;$InstallPrefix"` interpolation would then produce + # `";C:\..."` — a leading semicolon = empty PATH entry, which on + # Windows resolves to the CURRENT WORKING DIRECTORY. That's a + # well-known PATH-injection vector (binary planted in cwd runs + # ahead of real ones). Bugbot PR #11 r3 flagged the security + # concern. + $userPath = [Environment]::GetEnvironmentVariable('Path', 'User') + $existingEntries = if ($userPath) { $userPath -split ';' } else { @() } + if ($existingEntries -notcontains $InstallPrefix) { + Write-Host "" + Write-Host "Note: $InstallPrefix is not on `$env:Path. Adding it for your user:" + $newPath = if ($userPath) { "$userPath;$InstallPrefix" } else { $InstallPrefix } + [Environment]::SetEnvironmentVariable('Path', $newPath, 'User') + Write-Host " (open a new PowerShell window to pick up the change)" + } + + Write-Host "" + Write-Host "First steps:" + Write-Host " tracebloc cluster info # confirm CLI can reach your cluster" + Write-Host " tracebloc dataset push --help # see the dominant flow" +} +finally { + # Always clean up the temp dir, even on early exit / Ctrl-C. + Remove-Item -Recurse -Force -Path $tmpDir -ErrorAction SilentlyContinue +} diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..d2a194e --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env sh +# tracebloc CLI installer for Linux + macOS. +# +# Usage: +# curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh +# curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh -s -- --version v0.1.0 +# +# What it does: +# 1. Detects OS (linux/darwin) + arch (amd64/arm64) of the host +# 2. Resolves the latest release tag (or honors --version) +# 3. Downloads tracebloc--- from the GitHub Release +# 4. Verifies SHA256 against the release's SHA256SUMS file +# 5. (Optional) Verifies cosign signature if cosign is on PATH +# 6. Installs to /usr/local/bin/tracebloc (falls back to $HOME/.local/bin +# with PATH advice if /usr/local/bin isn't writable) +# +# Why /bin/sh + POSIX-only constructs: +# The customer's distro might not have bash. /bin/sh is POSIX-mandated. +# No bashisms (no [[ ]], no <(), no ${var/...}). Tested against dash, +# busybox sh, and bash. + +set -eu + +# -------------------------------------------------------------------- +# Configuration knobs (override via env or args). +# -------------------------------------------------------------------- +INSTALL_PREFIX="${INSTALL_PREFIX:-/usr/local/bin}" +RELEASE_VERSION="${RELEASE_VERSION:-latest}" +GITHUB_REPO="tracebloc/cli" +BINARY_NAME="tracebloc" + +usage() { + cat <] [--prefix ] [--help] + +Options: + --version Install a specific version (e.g. v0.1.0). Default: latest. + --prefix Install directory. Default: /usr/local/bin (falls back to + \$HOME/.local/bin if not writable). + --help Show this help. + +Environment overrides: + RELEASE_VERSION Same as --version. + INSTALL_PREFIX Same as --prefix. +EOF +} + +# -------------------------------------------------------------------- +# Arg parsing — minimal POSIX-shell loop, no getopt (not portable). +# -------------------------------------------------------------------- +while [ $# -gt 0 ]; do + case "$1" in + --version) + RELEASE_VERSION="$2" + shift 2 + ;; + --prefix) + INSTALL_PREFIX="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Error: unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +# -------------------------------------------------------------------- +# Detect OS + arch. +# -------------------------------------------------------------------- +detect_os() { + uname_s="$(uname -s)" + case "$uname_s" in + Linux) echo "linux" ;; + Darwin) echo "darwin" ;; + *) + echo "Error: unsupported OS: $uname_s" >&2 + echo "tracebloc CLI is released for linux + darwin via this script;" >&2 + echo "Windows users can run install.ps1 from a PowerShell prompt." >&2 + exit 1 + ;; + esac +} + +detect_arch() { + uname_m="$(uname -m)" + case "$uname_m" in + x86_64|amd64) echo "amd64" ;; + arm64|aarch64) echo "arm64" ;; + *) + echo "Error: unsupported arch: $uname_m" >&2 + echo "tracebloc CLI is released for amd64 + arm64; if you need" >&2 + echo "another arch, please file an issue at github.com/tracebloc/cli." >&2 + exit 1 + ;; + esac +} + +OS="$(detect_os)" +ARCH="$(detect_arch)" + +# -------------------------------------------------------------------- +# Resolve the release tag if "latest". +# -------------------------------------------------------------------- +resolve_tag() { + if [ "$RELEASE_VERSION" != "latest" ]; then + echo "$RELEASE_VERSION" + return + fi + # Use the redirect-trail of /releases/latest to learn the tag — + # avoids hitting the rate-limited /api/repos endpoint for the + # zero-auth one-liner case. + redirect_url="$(curl -fsSI \ + "https://github.com/${GITHUB_REPO}/releases/latest" \ + | awk '/^[Ll]ocation:/ { print $2 }' \ + | tr -d '\r')" + if [ -z "$redirect_url" ]; then + echo "Error: couldn't resolve the 'latest' release tag from GitHub." >&2 + echo "Pass --version to install a specific release." >&2 + exit 1 + fi + # The redirect URL ends in /tag/; basename gives us the tag. + basename "$redirect_url" +} + +TAG="$(resolve_tag)" +echo "Installing tracebloc CLI $TAG ($OS/$ARCH)..." + +# -------------------------------------------------------------------- +# Download binary + SHA256SUMS + (optional) cosign sig/cert. +# -------------------------------------------------------------------- +BINARY_FILE="${BINARY_NAME}-${TAG}-${OS}-${ARCH}" +BASE_URL="https://github.com/${GITHUB_REPO}/releases/download/${TAG}" + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT INT TERM + +echo "Downloading binary..." +if ! curl -fsSL "$BASE_URL/$BINARY_FILE" -o "$TMP/$BINARY_FILE"; then + echo "Error: failed to download $BASE_URL/$BINARY_FILE" >&2 + exit 1 +fi + +echo "Downloading SHA256SUMS..." +if ! curl -fsSL "$BASE_URL/SHA256SUMS" -o "$TMP/SHA256SUMS"; then + echo "Error: failed to download SHA256SUMS — release may be malformed" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Verify SHA256. +# -------------------------------------------------------------------- +echo "Verifying SHA256..." +expected="$(grep " $BINARY_FILE$" "$TMP/SHA256SUMS" | awk '{print $1}')" +if [ -z "$expected" ]; then + echo "Error: SHA256SUMS doesn't contain an entry for $BINARY_FILE" >&2 + echo " — release artifacts may be incomplete." >&2 + exit 1 +fi +# sha256sum (GNU coreutils) vs shasum -a 256 (macOS): detect which is on PATH. +# If neither is available, refuse to install — running an unverified +# binary from the internet is exactly what this script exists to +# prevent. Bugbot PR #11 caught the previous "warn + continue + still +# print ✓ matches" branch as both a security regression AND a +# misleading-log issue. Almost every modern Linux distro ships coreutils +# (sha256sum) by default; macOS ships /usr/bin/shasum as part of the +# base Perl install. A host with neither is unusual enough that +# erroring out is the right call — the customer can install coreutils +# / xcode-select / similar and re-run. +if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$TMP/$BINARY_FILE" | awk '{print $1}')" +elif command -v shasum >/dev/null 2>&1; then + actual="$(shasum -a 256 "$TMP/$BINARY_FILE" | awk '{print $1}')" +else + echo "Error: neither sha256sum nor shasum is on PATH — can't verify the" >&2 + echo " downloaded binary's integrity. Install one of:" >&2 + echo " apt install coreutils # Debian/Ubuntu" >&2 + echo " dnf install coreutils # Fedora/RHEL" >&2 + echo " apk add coreutils # Alpine" >&2 + echo " (macOS ships /usr/bin/shasum by default — PATH issue?)" >&2 + echo " and re-run." >&2 + exit 1 +fi +if [ "$actual" != "$expected" ]; then + echo "Error: SHA256 mismatch!" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + echo " refusing to install." >&2 + exit 1 +fi +echo " ✓ checksum matches" + +# -------------------------------------------------------------------- +# Verify cosign signature if cosign is on PATH (optional). +# -------------------------------------------------------------------- +if command -v cosign >/dev/null 2>&1; then + echo "Verifying cosign signature..." + if ! curl -fsSL "$BASE_URL/$BINARY_FILE.sig" -o "$TMP/$BINARY_FILE.sig"; then + echo " ⚠ couldn't download .sig — release may pre-date signing." >&2 + elif ! curl -fsSL "$BASE_URL/$BINARY_FILE.cert" -o "$TMP/$BINARY_FILE.cert"; then + echo " ⚠ couldn't download .cert — release may pre-date signing." >&2 + elif ! cosign verify-blob \ + --certificate-identity-regexp \ + "https://github.com/${GITHUB_REPO}/.github/workflows/release.yml@.*" \ + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ + --certificate "$TMP/$BINARY_FILE.cert" \ + --signature "$TMP/$BINARY_FILE.sig" \ + "$TMP/$BINARY_FILE" 2>/dev/null; then + echo "Error: cosign signature verification FAILED — refusing to install." >&2 + exit 1 + else + echo " ✓ cosign signature valid" + fi +else + # Not having cosign isn't fatal — the SHA256 check above is the + # baseline. Recommend cosign for higher-trust installs. + echo " (cosign not installed; SHA256 verified, signature skipped)" +fi + +# -------------------------------------------------------------------- +# Install to a writable prefix. +# +# Bugbot PR #11 r2 caught a UX bug in the previous flow: POSIX +# `test -w` is false for paths that don't exist, so a custom +# --prefix /opt/tracebloc (legitimate, just not created yet) was +# silently overridden with the ~/.local/bin fallback. The right +# semantic: try to mkdir the customer's chosen prefix; if THAT +# fails (no write perms on parent), THEN fall back. +# -------------------------------------------------------------------- +PREFIX="$INSTALL_PREFIX" +if ! mkdir -p "$PREFIX" 2>/dev/null || [ ! -w "$PREFIX" ]; then + # The customer's chosen prefix isn't usable (no write perms on + # parent, /usr/local/bin without sudo, etc.). Fall back to a + # per-user dir; the PATH-advice block below tells them how to + # pick it up. + FALLBACK="$HOME/.local/bin" + echo "Note: $PREFIX isn't writable (couldn't mkdir or no -w); falling back to $FALLBACK" + mkdir -p "$FALLBACK" + PREFIX="$FALLBACK" +fi + +chmod +x "$TMP/$BINARY_FILE" +mv "$TMP/$BINARY_FILE" "$PREFIX/$BINARY_NAME" + +echo "" +echo "✓ tracebloc CLI installed: $PREFIX/$BINARY_NAME" +echo "" +echo "Verify with:" +echo " $PREFIX/$BINARY_NAME version" +echo "" + +# PATH guidance for the fallback case. +case ":$PATH:" in + *":$PREFIX:"*) ;; # already on PATH + *) + echo "Note: $PREFIX is not on \$PATH. Add this to your shell rc file:" + echo " export PATH=\"\$PATH:$PREFIX\"" + echo "" + ;; +esac + +echo "First steps:" +echo " $BINARY_NAME cluster info # confirm CLI can reach your cluster" +echo " $BINARY_NAME dataset push --help # see the dominant flow" From 806f17b00292e563ade3a7f172274db598f2a4a1 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Tue, 2 Jun 2026 12:58:46 +0200 Subject: [PATCH 07/17] fix(dataset push): make live ingestion work end-to-end (#12) The first real-cluster run of `dataset push` surfaced three issues that the mock-only test suite couldn't catch. All three are now validated end-to-end against a live tracebloc/client release (image_classification, 6/6 records, 100% success, rows confirmed in MySQL). 1. Staging collided with the ingestor's DEST_PATH (the blocker). The CLI staged source files into /data/shared/
, which is exactly the ingestor's DEST_PATH (data-ingestors config.DEST_PATH = STORAGE_PATH/TABLE_NAME). Its DuplicateValidator rejects a pre-existing, non-empty destination, so the CLI's own staging tripped the check on every run. StagedPrefix now stages under /data/shared/.tracebloc-staging/
, leaving the destination fresh. Added FinalDestPrefix for the real destination shown in pre-flight. 2. Image target_size had no override. image_classification defaults to 512x512 and the ingestor VALIDATES the resolution (it does not resize), so any other size hard-failed the Image Resolution Validator. Added --target-size WxH, plus auto-detect from the first image when the flag is omitted, emitted as spec.file_options.target_size. 3. Watch treated a broken log stream as fatal. A Pod replaced/restarted/ deleted mid-follow (e.g. a backoffLimit retry) returned exit 9 even when the Job ultimately succeeded. The Job's terminal status is now the source of truth: a non-ctx stream error is only fatal if the Job outcome can't be determined. Tests: new push/detect_test.go (ParseTargetSize, DetectImageSize); spec_test.go (staging != dest regression, target_size passes schema); watch_test.go (Job status wins on stream break); stream_test.go updated to derive paths from StagedPrefix. go build / vet / test all green. Co-authored-by: Claude Opus 4.8 --- internal/cli/dataset.go | 67 +++++++++++++++++------ internal/push/detect.go | 70 ++++++++++++++++++++++++ internal/push/detect_test.go | 88 ++++++++++++++++++++++++++++++ internal/push/spec.go | 100 ++++++++++++++++++++++++++++------ internal/push/spec_test.go | 94 +++++++++++++++++++++++++++++++- internal/push/stage.go | 8 +-- internal/push/stream_test.go | 43 ++++++++++----- internal/submit/watch.go | 72 ++++++++++++++++-------- internal/submit/watch_test.go | 56 +++++++++++++++++++ 9 files changed, 520 insertions(+), 78 deletions(-) create mode 100644 internal/push/detect.go create mode 100644 internal/push/detect_test.go diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go index 5e76e0f..eb97ea2 100644 --- a/internal/cli/dataset.go +++ b/internal/cli/dataset.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "path/filepath" "github.com/spf13/cobra" "gopkg.in/yaml.v3" @@ -74,6 +75,7 @@ func newDatasetPushCmd() *cobra.Command { category string intent string labelColumn string + targetSize string // Operations flags. dryRun bool @@ -145,6 +147,7 @@ Exit codes: Context: contextOverride, Namespace: nsOverride, Spec: push.SpecArgs{Table: table, Category: category, Intent: intent, LabelColumn: labelColumn}, + TargetSizeFlag: targetSize, DryRun: dryRun, IngestorSAName: ingestorSAName, StagePodImage: stagePodImage, @@ -175,6 +178,9 @@ Exit codes: "intent: train|test") cmd.Flags().StringVar(&labelColumn, "label-column", "", "column name in labels.csv that holds the label") + cmd.Flags().StringVar(&targetSize, "target-size", "", + "image resolution as WxH (e.g. 512x512). Default: auto-detected from the first image. "+ + "All images must share this resolution — the ingestor validates it, it does not resize.") cmd.Flags().BoolVar(&dryRun, "dry-run", false, "validate + discover + walk, but don't create any cluster resources") @@ -209,6 +215,7 @@ type runDatasetPushArgs struct { Context string Namespace string Spec push.SpecArgs + TargetSizeFlag string // raw --target-size; resolved after Discover DryRun bool IngestorSAName string StagePodImage string @@ -258,11 +265,48 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush "tracebloc/client#147 non-goals.", a.Spec.Category)} } - // 3. Synthesize the spec from flags + validate against schema. + // 3. Walk the local directory FIRST. Enforces layout + size caps, + // and gives us the image list the target-size auto-detect below + // needs. Both this and the schema check are local "fail fast" + // steps; doing the walk first lets the synthesized spec carry + // the resolved target_size. + layout, err := push.Discover(a.LocalPath) + if err != nil { + return &exitError{code: 3, err: err} + } + + // 3a. Resolve the image target resolution. The ingestor's + // image_classification default is 512x512 and it VALIDATES + // (it does not resize), so a mismatch hard-fails the run with + // an "incorrect resolution" error. Honour an explicit + // --target-size; otherwise auto-detect from the first image so + // the common "all my images are NxN" case just works without + // the customer needing to know the knob exists. + if a.TargetSizeFlag != "" { + w, h, perr := push.ParseTargetSize(a.TargetSizeFlag) + if perr != nil { + return &exitError{code: 2, err: perr} + } + a.Spec.TargetSize = []int{w, h} + } else if len(layout.Images) > 0 { + if w, h, derr := push.DetectImageSize(layout.Images[0]); derr == nil { + a.Spec.TargetSize = []int{w, h} + _, _ = fmt.Fprintf(out, + "Auto-detected image target size %dx%d from %s (override with --target-size).\n", + w, h, filepath.Base(layout.Images[0])) + } else { + _, _ = fmt.Fprintf(errOut, + "Note: couldn't auto-detect image size (%v); using the ingestor "+ + "default. Pass --target-size WxH if ingestion reports a "+ + "resolution mismatch.\n", derr) + } + } + + // 4. Synthesize the spec from flags + validate against schema. // Catches "bad category", "missing intent" etc. BEFORE we - // touch the filesystem or the cluster. The error formatter - // is the same one ingest validate uses, so a customer who - // YAML'd manually first sees identical wording. + // touch the cluster. The error formatter is the same one + // ingest validate uses, so a customer who YAML'd manually + // first sees identical wording. spec := a.Spec.Build() specBytes, err := yaml.Marshal(spec) if err != nil { @@ -295,14 +339,6 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush return &exitError{code: 2, err: errors.New("synthesized spec failed schema validation; check the flag values above")} } - // 4. Walk the local directory. Enforces layout + size caps; - // customer sees a clear pointer to expected layout if they - // pass the wrong directory. - layout, err := push.Discover(a.LocalPath) - if err != nil { - return &exitError{code: 3, err: err} - } - // 5. Cluster discovery — same kubeconfig path as `cluster info`. // Errors mirror that command's exit-code contract (3 for // kubeconfig, 4 for missing release) so behaviour is @@ -527,13 +563,12 @@ func printPushPreflight( _, _ = fmt.Fprintf(out, " category: %s\n", spec["category"]) _, _ = fmt.Fprintf(out, " intent: %s\n", spec["intent"]) _, _ = fmt.Fprintf(out, " label column: %s\n", spec["label"]) - _, _ = fmt.Fprintf(out, " destination: %s\n", push.StagedPrefix(spec["table"].(string))) + _, _ = fmt.Fprintf(out, " destination: %s\n", push.FinalDestPrefix(spec["table"].(string))) _, _ = fmt.Fprintln(out) if !dryRun { - _, _ = fmt.Fprintf(out, "Next: stage %d files (%s) → %s\n", - 1+len(layout.Images), push.HumanBytes(layout.TotalBytes), - push.StagedPrefix(spec["table"].(string))) + _, _ = fmt.Fprintf(out, "Next: stage %d files (%s) for table %q\n", + 1+len(layout.Images), push.HumanBytes(layout.TotalBytes), spec["table"]) _, _ = fmt.Fprintln(out) } } diff --git a/internal/push/detect.go b/internal/push/detect.go new file mode 100644 index 0000000..9a2b838 --- /dev/null +++ b/internal/push/detect.go @@ -0,0 +1,70 @@ +package push + +import ( + "fmt" + "image" + "os" + "strconv" + "strings" + + // Register the stdlib image decoders so image.DecodeConfig can + // read the headers of the formats the image_classification layout + // accepts. webp is NOT in the stdlib — DetectImageSize returns an + // error for it and the caller falls back to requiring + // --target-size. (.jpg/.jpeg both decode via image/jpeg.) + _ "image/gif" + _ "image/jpeg" + _ "image/png" +) + +// DetectImageSize returns the pixel width and height of the image at +// path by decoding only its header (image.DecodeConfig — it does not +// read the pixel data, so it's cheap even for large images). +// +// Supports the stdlib-registered formats (jpeg, png, gif). Returns an +// error for formats without a registered decoder (notably webp); the +// caller treats that as "couldn't auto-detect" and falls back to the +// ingestor default, advising --target-size. +func DetectImageSize(path string) (width, height int, err error) { + f, err := os.Open(path) + if err != nil { + return 0, 0, err + } + defer func() { _ = f.Close() }() + + cfg, _, err := image.DecodeConfig(f) + if err != nil { + return 0, 0, fmt.Errorf("decoding image header %q: %w", path, err) + } + return cfg.Width, cfg.Height, nil +} + +// ParseTargetSize parses a --target-size flag value into [width, +// height]. Accepts "WxH" (the documented form, e.g. "512x512") and +// "W,H" as a convenience. Both dimensions must be positive integers. +func ParseTargetSize(s string) (width, height int, err error) { + sep := "x" + if strings.Contains(s, ",") { + sep = "," + } + parts := strings.Split(s, sep) + if len(parts) != 2 { + return 0, 0, fmt.Errorf( + "target size %q must be WxH (e.g. 512x512)", s) + } + width, err = strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + return 0, 0, fmt.Errorf( + "target size %q: width is not an integer: %w", s, err) + } + height, err = strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return 0, 0, fmt.Errorf( + "target size %q: height is not an integer: %w", s, err) + } + if width <= 0 || height <= 0 { + return 0, 0, fmt.Errorf( + "target size %q: width and height must both be positive", s) + } + return width, height, nil +} diff --git a/internal/push/detect_test.go b/internal/push/detect_test.go new file mode 100644 index 0000000..14348bf --- /dev/null +++ b/internal/push/detect_test.go @@ -0,0 +1,88 @@ +package push + +import ( + "image" + "image/png" + "os" + "path/filepath" + "testing" +) + +// TestParseTargetSize covers the --target-size flag parser: the +// documented WxH form, the W,H convenience form, and the rejection +// cases (missing dimension, non-integer, non-positive, wrong arity). +func TestParseTargetSize(t *testing.T) { + cases := []struct { + in string + w, h int + wantErr bool + }{ + {"512x512", 512, 512, false}, + {"640x480", 640, 480, false}, + {"512,512", 512, 512, false}, + {"1x1", 1, 1, false}, + {"512", 0, 0, true}, + {"512x", 0, 0, true}, + {"x512", 0, 0, true}, + {"0x512", 0, 0, true}, + {"-4x512", 0, 0, true}, + {"512x512x512", 0, 0, true}, + {"abcxdef", 0, 0, true}, + {"", 0, 0, true}, + } + for _, c := range cases { + w, h, err := ParseTargetSize(c.in) + if c.wantErr { + if err == nil { + t.Errorf("ParseTargetSize(%q) = (%d,%d,nil), want error", c.in, w, h) + } + continue + } + if err != nil { + t.Errorf("ParseTargetSize(%q) unexpected error: %v", c.in, err) + continue + } + if w != c.w || h != c.h { + t.Errorf("ParseTargetSize(%q) = (%d,%d), want (%d,%d)", c.in, w, h, c.w, c.h) + } + } +} + +// TestDetectImageSize_PNG: a real (generated) PNG's header is decoded +// to its true dimensions. Pins the auto-detect path used when the +// customer doesn't pass --target-size. +func TestDetectImageSize_PNG(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "img.png") + f, err := os.Create(p) + if err != nil { + t.Fatal(err) + } + if err := png.Encode(f, image.NewRGBA(image.Rect(0, 0, 320, 200))); err != nil { + t.Fatal(err) + } + _ = f.Close() + + w, h, err := DetectImageSize(p) + if err != nil { + t.Fatalf("DetectImageSize: %v", err) + } + if w != 320 || h != 200 { + t.Errorf("DetectImageSize = (%d,%d), want (320,200)", w, h) + } +} + +// TestDetectImageSize_Unsupported: a non-image (or unregistered +// format) returns an error so the caller falls back to the ingestor +// default + advises --target-size, rather than silently using a +// bogus size. +func TestDetectImageSize_Unsupported(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "note.txt") + if err := os.WriteFile(p, []byte("not an image"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := DetectImageSize(p); err == nil { + t.Error("DetectImageSize on non-image returned nil error; want a decode error") + } +} diff --git a/internal/push/spec.go b/internal/push/spec.go index 8aabb0a..ebb7d98 100644 --- a/internal/push/spec.go +++ b/internal/push/spec.go @@ -133,6 +133,16 @@ type SpecArgs struct { // shorthand because passthrough is the only policy // image_classification cares about. LabelColumn string + + // TargetSize, when len==2, pins the image resolution as [W, H]. + // The ingestor's image_classification default is 512x512 and it + // VALIDATES (it does not resize), so a dataset whose images don't + // match the default hard-fails. Setting this emits + // spec.file_options.target_size so the customer's actual + // resolution wins. Empty (len 0) ⇒ omit and let the ingestor + // default apply. Populated by the CLI from --target-size or by + // auto-detecting the first image. + TargetSize []int } // Build produces the ingest.v1.json-conforming spec map. The @@ -157,7 +167,7 @@ type SpecArgs struct { // calls StagedPrefix, which panics on an unsafe name. func (a SpecArgs) Build() map[string]any { prefix := StagedPrefix(a.Table) - return map[string]any{ + spec := map[string]any{ "apiVersion": "tracebloc.io/v1", "kind": "IngestConfig", "category": a.Category, @@ -170,40 +180,94 @@ func (a SpecArgs) Build() map[string]any { "images": path.Join(prefix, "images") + "/", "label": a.LabelColumn, } + // Emit the image resolution under spec.file_options.target_size — + // the same override key the helm flow + data-ingestors' + // conventions.resolve honour (it merges spec.file_options over the + // per-category default). Without this, image_classification + // defaults to 512x512 and the ingestor's Image Resolution + // Validator rejects any other size. + if len(a.TargetSize) == 2 { + spec["spec"] = map[string]any{ + "file_options": map[string]any{ + "target_size": []int{a.TargetSize[0], a.TargetSize[1]}, + }, + } + } + return spec } -// StagedPrefix returns the in-cluster destination directory the CLI -// writes files into for a given table. Used in two places that -// MUST agree: +// SharedRoot is the in-cluster mount path of the chart's shared PVC +// (cluster.SharedPVCMountPath). Both the ephemeral stage Pod and the +// ingestor Job mount client-pvc here, so any path under it is visible +// to both — which is why the CLI's staging area lives under it. +const SharedRoot = "/data/shared" + +// stagingDirName is the hidden directory under SharedRoot where the +// CLI lands a run's SOURCE files. It is deliberately SEPARATE from +// the ingestor's destination (SharedRoot/
): +// +// data-ingestors computes DEST_PATH = STORAGE_PATH/TABLE_NAME = +// SharedRoot/
, and its DuplicateValidator FAILS if that path +// already exists non-empty. If the CLI staged straight into +// SharedRoot/
(as it did originally), its own staging would +// create exactly the non-empty destination the validator rejects — +// so every push failed the duplicate check. Staging under +// SharedRoot/.tracebloc-staging/
keeps the destination fresh +// while remaining on the same PVC the ingestor reads. +const stagingDirName = ".tracebloc-staging" + +// StagedPrefix returns the in-cluster directory the CLI streams a +// dataset's SOURCE files into for a given table. The synthesized +// spec's csv/images point here; the ingestor reads from here and +// writes the processed table to FinalDestPrefix(table). +// +// Two call sites MUST agree on this value: // -// 1. Phase 3 (this PR + PR-b): the path the ephemeral stage Pod -// creates and tars files into. +// 1. The ephemeral stage Pod's tar target (StreamLayout). // 2. The csv/images fields in Build() above, which jobs-manager -// reads to know where the ingestor Job will find them. +// hands to the ingestor Job so it reads what we just staged. // -// Exported because Phase 3's PR-b (stage Pod construction) needs -// it from the same place, and Phase 4 (submit) might want to print -// it as part of "what we pushed." +// It is intentionally NOT SharedRoot/
: that is the ingestor's +// DEST_PATH, whose DuplicateValidator rejects a pre-existing, +// non-empty directory. See stagingDirName. // // PRECONDITION: table must already have passed ValidateTableName. // This function panics on an unsafe name rather than returning an -// escape path — a name that escapes /data/shared is a caller bug -// (validation was skipped), and a panic surfaces it loudly in -// tests instead of silently letting PR-b's stage Pod write to, -// say, /etc. Every production call path runs ValidateTableName -// first (see cli.runDatasetPush), so the panic is unreachable in -// correct code. +// escape path — a name that escapes SharedRoot is a caller bug +// (validation was skipped), and a panic surfaces it loudly in tests +// instead of silently letting the stage Pod write to, say, /etc. +// Every production call path runs ValidateTableName first (see +// cli.runDatasetPush), so the panic is unreachable in correct code. func StagedPrefix(table string) string { // Deliberately NOT path.Join here: path.Join cleans ".." // segments, which is exactly the silent traversal we're // guarding against. Plain concatenation keeps the name as a // literal segment so the assertion below can detect a bad one. - prefix := "/data/shared/" + table if !tableNamePattern.MatchString(table) { panic(fmt.Sprintf( "push.StagedPrefix: unsafe table name %q — caller must "+ "ValidateTableName before constructing a PVC path", table)) } - return prefix + return SharedRoot + "/" + stagingDirName + "/" + table +} + +// FinalDestPrefix returns where the ingestor writes the processed +// table: SharedRoot/
, matching data-ingestors' config.DEST_PATH +// (STORAGE_PATH/TABLE_NAME). This is what the training side reads and +// what the CLI shows the customer as the destination. The CLI never +// writes here directly — doing so would trip the ingestor's +// DuplicateValidator; it stages to StagedPrefix(table) and the +// ingestor produces this path. +// +// PRECONDITION: table must already have passed ValidateTableName. +// Panics on an unsafe name, same rationale as StagedPrefix. +func FinalDestPrefix(table string) string { + if !tableNamePattern.MatchString(table) { + panic(fmt.Sprintf( + "push.FinalDestPrefix: unsafe table name %q — caller must "+ + "ValidateTableName before constructing a PVC path", + table)) + } + return SharedRoot + "/" + table } diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go index 05c7961..a6301cb 100644 --- a/internal/push/spec_test.go +++ b/internal/push/spec_test.go @@ -102,8 +102,98 @@ func TestStagedPrefix_PerTableIsolation(t *testing.T) { if a, b := StagedPrefix("cats"), StagedPrefix("dogs"); a == b { t.Errorf("StagedPrefix(%q) == StagedPrefix(%q) = %q, want distinct", "cats", "dogs", a) } - if got := StagedPrefix("table_a"); got != "/data/shared/table_a" { - t.Errorf("StagedPrefix(%q) = %q, want /data/shared/table_a", "table_a", got) + if got := StagedPrefix("table_a"); got != "/data/shared/.tracebloc-staging/table_a" { + t.Errorf("StagedPrefix(%q) = %q, want /data/shared/.tracebloc-staging/table_a", "table_a", got) + } +} + +// TestStagedPrefix_DoesNotCollideWithDest is the regression pin for +// the live-discovered blocker (#26): the CLI must stage SOURCE files +// somewhere the ingestor's DEST_PATH (= FinalDestPrefix = /data/ +// shared/
) does NOT contain. If staging ever lands at or under +// the destination again, the ingestor's DuplicateValidator will +// reject the CLI's own staging as a pre-existing non-empty +// destination, and every push fails the duplicate check. +func TestStagedPrefix_DoesNotCollideWithDest(t *testing.T) { + const table = "cats_dogs_train" + staged := StagedPrefix(table) + dest := FinalDestPrefix(table) + + if staged == dest { + t.Fatalf("StagedPrefix == FinalDestPrefix == %q; the CLI would stage "+ + "into the ingestor's DEST_PATH and trip DuplicateValidator", staged) + } + // The destination must not contain the staging dir either — + // otherwise DEST is non-empty (it holds the staging subtree) and + // the validator still fails. + if strings.HasPrefix(staged, dest+"/") { + t.Errorf("StagedPrefix %q is under FinalDestPrefix %q; DEST would be "+ + "non-empty at ingest time", staged, dest) + } + if got := FinalDestPrefix(table); got != "/data/shared/"+table { + t.Errorf("FinalDestPrefix(%q) = %q, want /data/shared/%s", table, got, table) + } +} + +// TestBuild_WithTargetSize_PassesSchema pins the #27 plumbing: when +// the CLI resolves an image resolution (via --target-size or +// auto-detect), Build emits spec.file_options.target_size and the +// result still validates against the embedded v1 schema. A drift here +// would make every push that sets a target size fail validation. +func TestBuild_WithTargetSize_PassesSchema(t *testing.T) { + spec := SpecArgs{ + Table: "cats_dogs_train", + Category: "image_classification", + Intent: "train", + LabelColumn: "label", + TargetSize: []int{256, 256}, + }.Build() + + // The nested override must be present and well-shaped. + specBlock, ok := spec["spec"].(map[string]any) + if !ok { + t.Fatalf("Build() with TargetSize didn't emit a spec block: %#v", spec["spec"]) + } + fo, ok := specBlock["file_options"].(map[string]any) + if !ok { + t.Fatalf("spec.file_options missing/wrong type: %#v", specBlock["file_options"]) + } + ts, ok := fo["target_size"].([]int) + if !ok || len(ts) != 2 || ts[0] != 256 || ts[1] != 256 { + t.Fatalf("spec.file_options.target_size = %#v, want [256 256]", fo["target_size"]) + } + + specBytes, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("yaml.Marshal: %v", err) + } + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + _, errs, parseErr := v.ValidateYAML(specBytes) + if parseErr != nil { + t.Fatalf("ValidateYAML parse error on our own output: %v\n%s", parseErr, specBytes) + } + if len(errs) != 0 { + t.Fatalf("spec with target_size failed schema validation: %s\nspec:\n%s", + schema.FormatErrors(errs), specBytes) + } +} + +// TestBuild_NoTargetSize_OmitsSpecBlock: when no resolution is set, +// Build must NOT emit a spec block (the ingestor's per-category +// default applies). Asserting the omission keeps the minimal-spec +// contract that the original schema test relies on. +func TestBuild_NoTargetSize_OmitsSpecBlock(t *testing.T) { + spec := SpecArgs{ + Table: "t", + Category: "image_classification", + Intent: "train", + LabelColumn: "label", + }.Build() + if _, present := spec["spec"]; present { + t.Errorf("Build() with no TargetSize emitted a spec block; want omitted") } } diff --git a/internal/push/stage.go b/internal/push/stage.go index f8e9472..ec39607 100644 --- a/internal/push/stage.go +++ b/internal/push/stage.go @@ -137,8 +137,8 @@ func Stage(ctx context.Context, opts StageOptions) error { // 5. Stream the tar. This is where actual bytes flow. The // progress bar (if TTY) renders during this call. - _, _ = fmt.Fprintf(opts.Out, "Streaming %d files (%s) to %s...\n", - 1+len(opts.Layout.Images), HumanBytes(opts.Layout.TotalBytes), StagedPrefix(opts.Table)) + _, _ = fmt.Fprintf(opts.Out, "Streaming %d files (%s) for table %q...\n", + 1+len(opts.Layout.Images), HumanBytes(opts.Layout.TotalBytes), opts.Table) if err := StreamLayout(ctx, opts.Executor, opts.Namespace, podName, "stage", @@ -147,7 +147,7 @@ func Stage(ctx context.Context, opts StageOptions) error { } // 6. Print "done" message. The deferred cleanup runs after this. - _, _ = fmt.Fprintf(opts.Out, "Staged %d files to %s\n", - 1+len(opts.Layout.Images), StagedPrefix(opts.Table)) + _, _ = fmt.Fprintf(opts.Out, "Staged %d files for table %q\n", + 1+len(opts.Layout.Images), opts.Table) return nil } diff --git a/internal/push/stream_test.go b/internal/push/stream_test.go index 62bde45..ea0e055 100644 --- a/internal/push/stream_test.go +++ b/internal/push/stream_test.go @@ -187,12 +187,19 @@ func TestStreamLayout_RemoteCommand(t *testing.T) { // - tar BEFORE any rm of $DEST (preserves on tar failure) // - mv AFTER tar succeeds (atomic-ish swap) // + // `dest` is StagedPrefix(table) — the CLI's SOURCE staging dir, + // which (since #26) lives under SharedRoot/.tracebloc-staging/ so + // it never collides with the ingestor's DEST_PATH. Derive the + // expected paths from it so this test tracks StagedPrefix rather + // than hardcoding the prefix. + dest := StagedPrefix("my_table") + // Extract the staging path with a regex so the random hex // suffix doesn't pin us to a specific invocation's bytes. - stagingRE := regexp.MustCompile(`/data/shared/my_table\.staging-[0-9a-f]{8}`) + stagingRE := regexp.MustCompile(regexp.QuoteMeta(dest) + `\.staging-[0-9a-f]{8}`) stagingPaths := stagingRE.FindAllString(script, -1) if len(stagingPaths) == 0 { - t.Fatalf("remote script has no /data/shared/my_table.staging-<8hex> path (race-safety regression):\n%s", script) + t.Fatalf("remote script has no %s.staging-<8hex> path (race-safety regression):\n%s", dest, script) } // All staging mentions must refer to the SAME suffix in a single // invocation. If we see two distinct suffixes that's a bug: @@ -209,7 +216,7 @@ func TestStreamLayout_RemoteCommand(t *testing.T) { for _, want := range []string{ `mkdir -p "` + staging + `"`, `tar -xf - -C "` + staging + `"`, - `mv "` + staging + `" "/data/shared/my_table"`, + `mv "` + staging + `" "` + dest + `"`, } { if !strings.Contains(script, want) { t.Errorf("remote script missing %q: %s", want, script) @@ -221,18 +228,24 @@ func TestStreamLayout_RemoteCommand(t *testing.T) { // (backup-and-swap, so a mv failure can be rolled back). // The contract is the same: tar runs while $DEST is intact. tarIdx := strings.Index(script, `tar -xf - -C "`+staging+`"`) - destBackupMvIdx := strings.Index(script, `mv "/data/shared/my_table" "`) + destBackupMvIdx := strings.Index(script, `mv "`+dest+`" "`) if tarIdx < 0 || destBackupMvIdx < 0 { t.Fatalf("remote script missing tar or destination-backup mv: %s", script) } if tarIdx >= destBackupMvIdx { t.Errorf("remote script touches $DEST BEFORE tar succeeds — partial-transfer could destroy previous data:\n%s", script) } - // Single-segment guarantee: both rm targets must end with - // /my_table or /my_table.staging-* — never just /data/shared. - if strings.Contains(script, `rm -rf "/data/shared"`) || - strings.Contains(script, `rm -rf "/data/shared/"`) { - t.Errorf("remote script rm-rfs the parent /data/shared (would nuke sibling tables):\n%s", script) + // Single-segment guarantee: rm targets must never be the shared + // root or the staging parent themselves (that would nuke sibling + // tables / every in-flight push). + for _, forbidden := range []string{ + `rm -rf "/data/shared"`, + `rm -rf "/data/shared/"`, + `rm -rf "` + SharedRoot + "/" + stagingDirName + `"`, + } { + if strings.Contains(script, forbidden) { + t.Errorf("remote script contains dangerous %q (would nuke sibling tables/pushes):\n%s", forbidden, script) + } } // Bugbot r9 + r10: orphan cleanup for previously-failed pushes @@ -253,7 +266,7 @@ func TestStreamLayout_RemoteCommand(t *testing.T) { // must be backed up to .old- BEFORE the new dataset // arrives, and restored if the main mv fails. Pin the key // shape pieces — backup mv, primary mv, rollback mv, cleanup. - backupRE := regexp.MustCompile(`/data/shared/my_table\.old-[0-9a-f]{8}`) + backupRE := regexp.MustCompile(regexp.QuoteMeta(dest) + `\.old-[0-9a-f]{8}`) backupPaths := backupRE.FindAllString(script, -1) if len(backupPaths) == 0 { t.Fatalf("remote script has no .old- backup path (r10 rollback regression):\n%s", script) @@ -272,15 +285,15 @@ func TestStreamLayout_RemoteCommand(t *testing.T) { // suffixes would defeat the "find -name ...staging-* ...old-*" // orphan-cleanup symmetry, AND would risk collision with a // concurrent push's .old-. - if strings.TrimPrefix(backup, "/data/shared/my_table.old-") != - strings.TrimPrefix(staging, "/data/shared/my_table.staging-") { + if strings.TrimPrefix(backup, dest+".old-") != + strings.TrimPrefix(staging, dest+".staging-") { t.Errorf("backup and staging suffixes diverge: %q vs %q", backup, staging) } // Backup mv (DEST → .old) must appear BEFORE primary mv // (.staging → DEST), or rollback wouldn't have anything to // restore. - backupMvIdx := strings.Index(script, `mv "/data/shared/my_table" "`+backup+`"`) - primaryMvIdx := strings.Index(script, `mv "`+staging+`" "/data/shared/my_table"`) + backupMvIdx := strings.Index(script, `mv "`+dest+`" "`+backup+`"`) + primaryMvIdx := strings.Index(script, `mv "`+staging+`" "`+dest+`"`) if backupMvIdx < 0 || primaryMvIdx < 0 { t.Fatalf("remote script missing backup or primary mv:\n%s", script) } @@ -299,7 +312,7 @@ func TestStreamLayout_StagingSuffixIsUniquePerInvocation(t *testing.T) { if err != nil { t.Fatalf("Discover: %v", err) } - stagingRE := regexp.MustCompile(`/data/shared/t\.staging-[0-9a-f]{8}`) + stagingRE := regexp.MustCompile(regexp.QuoteMeta(StagedPrefix("t")) + `\.staging-[0-9a-f]{8}`) collect := func() string { fe := &fakeExecutor{} diff --git a/internal/submit/watch.go b/internal/submit/watch.go index 3280e81..3ef8d47 100644 --- a/internal/submit/watch.go +++ b/internal/submit/watch.go @@ -220,18 +220,10 @@ func WatchJob( // a structured representation of the banner without // requiring a second log fetch post-completion. summary, logErr := streamPodLogsAndParse(watchCtx, cs, namespace, podName, out) - // Filter out the two ctx-flavored errors — both are "observation - // gave up early," not "stream failed." They get classified below - // into Detached (customer SIGINT, JobWatchTimeout expiry). Any - // other error is a real streaming failure (network mid-stream, - // API server tantrum) and bubbles up as a watch error. - if logErr != nil && - !errors.Is(logErr, context.Canceled) && - !errors.Is(logErr, context.DeadlineExceeded) { - return nil, fmt.Errorf("streaming logs from Pod %s/%s: %w", namespace, podName, logErr) - } - // 3. Detach branches: + // 3. Detach branches — checked FIRST, since the customer's SIGINT + // or the watch-cap expiry is the operative intent and takes + // precedence over any stream error: // - customerCtx canceled = SIGINT // - watchCtx expired (DeadlineExceeded) = JobWatchTimeout cap // hit during streaming (1-hour observation window exceeded) @@ -265,20 +257,54 @@ func WatchJob( // inheriting watchCtx's depleted budget caused successful // slow ingestions to misreport as Unknown. // - // The fresh ctx still propagates SIGINT (parent is - // customerCtx, which carries signal.NotifyContext's - // cancel). If the customer Ctrl-C's during this 30s - // window, we fall into the detach branch below — same - // contract as during the log stream. + // The Job — not the log stream — is the source of truth for + // success/failure, so we ALWAYS consult it here, INCLUDING when + // the log stream broke for a non-ctx reason (#28: the watched + // Pod was replaced / restarted / deleted mid-follow, e.g. a + // backoffLimit retry). A broken stream is only fatal if we also + // can't determine the Job's outcome. + // + // The fresh ctx still propagates SIGINT (parent is customerCtx, + // which carries signal.NotifyContext's cancel); a Ctrl-C in this + // window falls into the detach branches below. finalCtx, finalCancel := context.WithTimeout(customerCtx, 30*time.Second) defer finalCancel() - outcome, err := finalJobStatus(finalCtx, cs, namespace, jobName) - if err != nil { + outcome, statusErr := finalJobStatus(finalCtx, cs, namespace, jobName) + + // A non-ctx log-stream error is incidental if the Job still + // reached a terminal state. Previously ANY such error (e.g. + // "container is terminated" once the Pod was replaced by a retry) + // returned exit 9 even when the Job ultimately succeeded. #28. + streamFailed := logErr != nil && + !errors.Is(logErr, context.Canceled) && + !errors.Is(logErr, context.DeadlineExceeded) + if streamFailed { + // SIGINT during the final-status poll → graceful detach. + if errors.Is(customerCtx.Err(), context.Canceled) { + return &WatchResult{ + Outcome: JobOutcomeDetached, + PodName: podName, + Summary: summary, + DetachReason: DetachReasonSignal, + }, nil + } + // Job reached a terminal state → the stream error was + // incidental (Pod replaced/restarted). Report the real + // outcome the customer cares about. + if statusErr == nil && (outcome == JobOutcomeSucceeded || outcome == JobOutcomeFailed) { + return &WatchResult{Outcome: outcome, PodName: podName, Summary: summary}, nil + } + // Couldn't confirm a terminal Job state → the stream failure + // is the actionable signal; surface it. + return nil, fmt.Errorf("streaming logs from Pod %s/%s: %w", namespace, podName, logErr) + } + + // 5. Clean-stream path: classify on the Job status alone. + if statusErr != nil { // Treat SIGINT during finalJobStatus as graceful detach - // (same as during the log stream — jobs-manager already - // accepted the run, the customer is just stopping the - // observation). Bugbot PR #10 r2 flagged the "exit 9 on - // post-stream SIGINT" inconsistency. + // (jobs-manager already accepted the run; the customer is + // just stopping the observation). Bugbot PR #10 r2 flagged + // the "exit 9 on post-stream SIGINT" inconsistency. if errors.Is(customerCtx.Err(), context.Canceled) { return &WatchResult{ Outcome: JobOutcomeDetached, @@ -287,7 +313,7 @@ func WatchJob( DetachReason: DetachReasonSignal, }, nil } - return nil, fmt.Errorf("reading final Job status for %s/%s: %w", namespace, jobName, err) + return nil, fmt.Errorf("reading final Job status for %s/%s: %w", namespace, jobName, statusErr) } return &WatchResult{ Outcome: outcome, diff --git a/internal/submit/watch_test.go b/internal/submit/watch_test.go index 2be8b05..4da6fb9 100644 --- a/internal/submit/watch_test.go +++ b/internal/submit/watch_test.go @@ -263,6 +263,62 @@ func TestWatchJob_PodWaitTimeoutMapsToDetach(t *testing.T) { } } +// TestWatchJob_TerminalJobStatusWins is the #28 regression pin: the +// Job — not the log stream — is the source of truth for the outcome. +// With a Running Pod and a Job already reporting Complete, WatchJob +// must return Succeeded. This holds whether the (fake) log stream +// yields data or breaks: if the stream errored, the new +// "streamFailed but Job terminal → report outcome" branch still +// resolves to the Job's verdict instead of bubbling exit-9. Before +// the fix, a broken stream (e.g. a Pod replaced by a retry, or +// deleted mid-follow) returned an error even on a successful Job. +func TestWatchJob_TerminalJobStatusWins(t *testing.T) { + cs := fake.NewClientset( + jobPod("ingestor-xyz", "ingestor", corev1.PodRunning), + jobWithCondition("ingestor", batchv1.JobComplete), + ) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var out bytes.Buffer + wr, err := WatchJob(ctx, cs, "tracebloc", "ingestor", &out) + if err != nil { + t.Fatalf("WatchJob returned error; want nil + Succeeded: %v", err) + } + if wr == nil { + t.Fatal("WatchJob returned nil result") + } + if wr.Outcome != JobOutcomeSucceeded { + t.Fatalf("Outcome = %v, want Succeeded (Job condition is the source of truth)", wr.Outcome) + } +} + +// TestWatchJob_TerminalFailedJobReported: the mirror of the above for +// a Failed Job — the watch reports Failed (→ exit 9), not a generic +// watch error, even if the stream broke. +func TestWatchJob_TerminalFailedJobReported(t *testing.T) { + cs := fake.NewClientset( + jobPod("ingestor-xyz", "ingestor", corev1.PodRunning), + jobWithCondition("ingestor", batchv1.JobFailed), + ) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var out bytes.Buffer + wr, err := WatchJob(ctx, cs, "tracebloc", "ingestor", &out) + if err != nil { + t.Fatalf("WatchJob returned error; want nil + Failed: %v", err) + } + if wr == nil { + t.Fatal("WatchJob returned nil result") + } + if wr.Outcome != JobOutcomeFailed { + t.Fatalf("Outcome = %v, want Failed", wr.Outcome) + } +} + // TestJobOutcome_String: stringer pin so diagnostic output stays // stable. func TestJobOutcome_String(t *testing.T) { From 857df92d05d4b941f852c920e245107f2ff4aabd Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:01:07 +0200 Subject: [PATCH 08/17] feat(dataset push): support the tabular / time-series modality family (#13) Extends `dataset push` from image_classification-only to also cover tabular_classification, tabular_regression, time_series_forecasting, and time_to_event_prediction. The Python ingestor already supports these; this adds the CLI-side flag / layout / spec surface. Validated end-to-end on a live cluster (tabular_classification, 8/8 records, 100%, rows confirmed in MySQL). - Category dispatch (push/category.go): image vs tabular families, mirroring data-ingestors' conventions.py groupings. - Tabular local layout (push/tabular.go): a single CSV (no sidecar files), staged via the existing machinery (CSV + empty image list). - Schema: auto-inferred from the CSV (INT/FLOAT/VARCHAR) so customers don't hand-write one; --schema col:TYPE,... overrides. Reserved framework columns (id, data_id, ...) are skipped so a CSV carrying an id column doesn't produce a schema the ingestor rejects (the #135b guard). - Label: string form for tabular_classification; object form with policy=bucket (default) for the regression-class categories so the raw numeric target never leaves the cluster. Added --label-policy and --time-column. - Build() branches by category (push/spec.go); pre-flight is category-aware (data CSV + column count for tabular). Tests: push/tabular_test.go (DiscoverTabular, InferSchema incl. reserved-skip, ParseSchema); spec_test.go (tabular Build passes the schema for all three label shapes, regression defaults to bucket); updated the unsupported-category gate test. go build / vet / test green. Stacked on cli#12 (the dataset-push live-ingestion fixes). Co-authored-by: Claude Opus 4.8 --- internal/cli/dataset.go | 195 ++++++++++++++++++++--------- internal/cli/dataset_test.go | 20 +-- internal/push/category.go | 51 ++++++++ internal/push/spec.go | 81 ++++++++++-- internal/push/spec_test.go | 78 ++++++++++++ internal/push/tabular.go | 229 ++++++++++++++++++++++++++++++++++ internal/push/tabular_test.go | 153 +++++++++++++++++++++++ 7 files changed, 728 insertions(+), 79 deletions(-) create mode 100644 internal/push/category.go create mode 100644 internal/push/tabular.go create mode 100644 internal/push/tabular_test.go diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go index eb97ea2..23e9b9e 100644 --- a/internal/cli/dataset.go +++ b/internal/cli/dataset.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "path/filepath" + "strings" "github.com/spf13/cobra" "gopkg.in/yaml.v3" @@ -68,14 +69,17 @@ func newDatasetPushCmd() *cobra.Command { contextOverride string nsOverride string - // Ingest-spec flags. The set is intentionally - // image_classification-only for v0.1 per epic #147 - // non-goals; other categories are one-PR additions in v0.2. + // Ingest-spec flags. image_classification + the tabular / + // time-series family are supported today; text + detection + + // segmentation land in later increments. table string category string intent string labelColumn string targetSize string + schemaFlag string + labelPolicy string + timeColumn string // Operations flags. dryRun bool @@ -142,12 +146,16 @@ Exit codes: RunE: func(cmd *cobra.Command, args []string) error { return runDatasetPush(cmd.Context(), cmd.OutOrStdout(), cmd.ErrOrStderr(), runDatasetPushArgs{ - LocalPath: args[0], - Kubeconfig: kubeconfigPath, - Context: contextOverride, - Namespace: nsOverride, - Spec: push.SpecArgs{Table: table, Category: category, Intent: intent, LabelColumn: labelColumn}, + LocalPath: args[0], + Kubeconfig: kubeconfigPath, + Context: contextOverride, + Namespace: nsOverride, + Spec: push.SpecArgs{ + Table: table, Category: category, Intent: intent, + LabelColumn: labelColumn, LabelPolicy: labelPolicy, TimeColumn: timeColumn, + }, TargetSizeFlag: targetSize, + SchemaFlag: schemaFlag, DryRun: dryRun, IngestorSAName: ingestorSAName, StagePodImage: stagePodImage, @@ -173,14 +181,23 @@ Exit codes: cmd.Flags().StringVar(&table, "table", "", "destination table name (MySQL identifier; matches /data/shared/
/ on the PVC)") cmd.Flags().StringVar(&category, "category", "image_classification", - "task category (v0.1 only supports image_classification; see tracebloc/client#147 non-goals)") + "task category: image_classification, tabular_classification, tabular_regression, "+ + "time_series_forecasting, time_to_event_prediction") cmd.Flags().StringVar(&intent, "intent", "", "intent: train|test") cmd.Flags().StringVar(&labelColumn, "label-column", "", - "column name in labels.csv that holds the label") + "name of the label/target column (in labels.csv for image categories, in the data CSV for tabular)") cmd.Flags().StringVar(&targetSize, "target-size", "", - "image resolution as WxH (e.g. 512x512). Default: auto-detected from the first image. "+ + "image categories only: resolution as WxH (e.g. 512x512). Default: auto-detected from the first image. "+ "All images must share this resolution — the ingestor validates it, it does not resize.") + cmd.Flags().StringVar(&schemaFlag, "schema", "", + "tabular/time-series only: column types as col:TYPE,col:TYPE (e.g. age:INT,price:FLOAT). "+ + "Default: inferred from the CSV (INT/FLOAT/VARCHAR).") + cmd.Flags().StringVar(&labelPolicy, "label-policy", "", + "regression-class only (tabular_regression, time_series_forecasting, time_to_event_prediction): "+ + "passthrough|bucket (default bucket — bins the target so the raw value never leaves the cluster)") + cmd.Flags().StringVar(&timeColumn, "time-column", "", + "time_to_event_prediction only: name of the time/duration column (default: a column named \"time\")") cmd.Flags().BoolVar(&dryRun, "dry-run", false, "validate + discover + walk, but don't create any cluster resources") @@ -215,7 +232,8 @@ type runDatasetPushArgs struct { Context string Namespace string Spec push.SpecArgs - TargetSizeFlag string // raw --target-size; resolved after Discover + TargetSizeFlag string // raw --target-size; resolved after Discover (image) + SchemaFlag string // raw --schema; resolved or inferred after Discover (tabular) DryRun bool IngestorSAName string StagePodImage string @@ -249,56 +267,103 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush return &exitError{code: 2, err: err} } - // 2. v0.1 category gate. Runs BEFORE schema validation because - // schema-valid-but-unsupported categories (e.g. - // tabular_classification) would otherwise fail with the - // schema's "missing property 'schema'" message — confusing - // for the customer who has no way to set --schema in v0.1. - // Nonsense categories (typos) also hit this gate; the - // "only image_classification in v0.1" message is more - // actionable than the schema's 11-option enum list anyway. - // Bugbot's review-on-self caught the missing gate on PR-a. - if a.Spec.Category != "" && a.Spec.Category != "image_classification" { + // 2. Category gate. Runs BEFORE schema validation so an + // unsupported category gets a clear, actionable CLI message + // rather than the schema's terse enum / missing-property error. + // Supported today: image_classification + the tabular / + // time-series family. The other image categories need sidecar + // (annotation/mask) staging the CLI doesn't do yet, and the + // text family needs a texts/sequences dir — both land in later + // increments. A typo'd category also lands here with a clear + // list rather than the schema's 11-option enum dump. + switch { + case a.Spec.Category == "": + // Left empty by a caller; let the schema produce the canonical + // "category is required" error downstream. + case push.IsTabular(a.Spec.Category) || a.Spec.Category == "image_classification": + // supported + case push.IsImage(a.Spec.Category): return &exitError{code: 2, err: fmt.Errorf( - "category %q is not supported in v0.1 (only image_classification). "+ - "Other categories are one-PR additions in v0.2 — see "+ - "tracebloc/client#147 non-goals.", a.Spec.Category)} + "category %q isn't supported by the CLI yet — it needs annotation/mask "+ + "sidecar staging that's coming in a later release. Supported image "+ + "category: image_classification.", a.Spec.Category)} + default: + return &exitError{code: 2, err: fmt.Errorf( + "category %q isn't supported by the CLI yet. Supported: image_classification, "+ + "tabular_classification, tabular_regression, time_series_forecasting, "+ + "time_to_event_prediction. (Text / detection / segmentation are coming; "+ + "use the helm flow for those meanwhile.)", a.Spec.Category)} } - // 3. Walk the local directory FIRST. Enforces layout + size caps, - // and gives us the image list the target-size auto-detect below - // needs. Both this and the schema check are local "fail fast" - // steps; doing the walk first lets the synthesized spec carry - // the resolved target_size. - layout, err := push.Discover(a.LocalPath) + // 3. Walk the local directory FIRST (local "fail fast"), dispatched + // by category family. Image categories expect labels.csv + + // images/; tabular / time-series categories expect a single + // data CSV. The walk also yields what the per-category + // resolution below needs (the image list for target-size, the + // CSV for schema inference). + var ( + layout *push.LocalLayout + err error + ) + if push.IsTabular(a.Spec.Category) { + layout, err = push.DiscoverTabular(a.LocalPath) + } else { + layout, err = push.Discover(a.LocalPath) + } if err != nil { return &exitError{code: 3, err: err} } - // 3a. Resolve the image target resolution. The ingestor's - // image_classification default is 512x512 and it VALIDATES - // (it does not resize), so a mismatch hard-fails the run with - // an "incorrect resolution" error. Honour an explicit - // --target-size; otherwise auto-detect from the first image so - // the common "all my images are NxN" case just works without - // the customer needing to know the knob exists. - if a.TargetSizeFlag != "" { - w, h, perr := push.ParseTargetSize(a.TargetSizeFlag) - if perr != nil { - return &exitError{code: 2, err: perr} + // 3a. Per-category spec resolution from the local data, so the + // synthesized spec carries the right fields before validation. + if push.IsTabular(a.Spec.Category) { + // Column schema: an explicit --schema wins; otherwise infer + // INT/FLOAT/VARCHAR types from the CSV so the customer doesn't + // hand-write one for the common case. + if a.SchemaFlag != "" { + sch, perr := push.ParseSchema(a.SchemaFlag) + if perr != nil { + return &exitError{code: 2, err: perr} + } + a.Spec.Schema = sch + } else { + sch, skipped, ierr := push.InferSchema(layout.LabelsCSV) + if ierr != nil { + return &exitError{code: 3, err: fmt.Errorf("inferring schema from CSV: %w", ierr)} + } + a.Spec.Schema = sch + _, _ = fmt.Fprintf(out, + "Inferred schema for %d column(s) from %s (override with --schema).\n", + len(sch), filepath.Base(layout.LabelsCSV)) + if len(skipped) > 0 { + _, _ = fmt.Fprintf(out, + " (skipped framework-managed column(s): %s)\n", strings.Join(skipped, ", ")) + } } - a.Spec.TargetSize = []int{w, h} - } else if len(layout.Images) > 0 { - if w, h, derr := push.DetectImageSize(layout.Images[0]); derr == nil { + } else { + // Image target resolution: the ingestor's image_classification + // default is 512x512 and it VALIDATES (it does not resize), so + // a mismatch hard-fails. Honour an explicit --target-size; + // otherwise auto-detect from the first image so the common + // "all my images are NxN" case just works. + if a.TargetSizeFlag != "" { + w, h, perr := push.ParseTargetSize(a.TargetSizeFlag) + if perr != nil { + return &exitError{code: 2, err: perr} + } a.Spec.TargetSize = []int{w, h} - _, _ = fmt.Fprintf(out, - "Auto-detected image target size %dx%d from %s (override with --target-size).\n", - w, h, filepath.Base(layout.Images[0])) - } else { - _, _ = fmt.Fprintf(errOut, - "Note: couldn't auto-detect image size (%v); using the ingestor "+ - "default. Pass --target-size WxH if ingestion reports a "+ - "resolution mismatch.\n", derr) + } else if len(layout.Images) > 0 { + if w, h, derr := push.DetectImageSize(layout.Images[0]); derr == nil { + a.Spec.TargetSize = []int{w, h} + _, _ = fmt.Fprintf(out, + "Auto-detected image target size %dx%d from %s (override with --target-size).\n", + w, h, filepath.Base(layout.Images[0])) + } else { + _, _ = fmt.Fprintf(errOut, + "Note: couldn't auto-detect image size (%v); using the ingestor "+ + "default. Pass --target-size WxH if ingestion reports a "+ + "resolution mismatch.\n", derr) + } } } @@ -538,10 +603,20 @@ func printPushPreflight( // as cli/cluster.go and cli/ingest.go: a pipe-write failure // shouldn't convert success into failure. The exit code is // the contract. + cat, _ := spec["category"].(string) + tabular := push.IsTabular(cat) + _, _ = fmt.Fprintf(out, "Local dataset:\n") _, _ = fmt.Fprintf(out, " root: %s\n", layout.Root) - _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) - _, _ = fmt.Fprintf(out, " images: %d files\n", len(layout.Images)) + if tabular { + _, _ = fmt.Fprintf(out, " data CSV: %s\n", layout.LabelsCSV) + if sch, ok := spec["schema"].(map[string]string); ok { + _, _ = fmt.Fprintf(out, " columns: %d\n", len(sch)) + } + } else { + _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) + _, _ = fmt.Fprintf(out, " images: %d files\n", len(layout.Images)) + } _, _ = fmt.Fprintf(out, " total size: %s\n", push.HumanBytes(layout.TotalBytes)) _, _ = fmt.Fprintln(out) @@ -562,7 +637,15 @@ func printPushPreflight( _, _ = fmt.Fprintf(out, " table: %s\n", spec["table"]) _, _ = fmt.Fprintf(out, " category: %s\n", spec["category"]) _, _ = fmt.Fprintf(out, " intent: %s\n", spec["intent"]) - _, _ = fmt.Fprintf(out, " label column: %s\n", spec["label"]) + switch lbl := spec["label"].(type) { + case string: + _, _ = fmt.Fprintf(out, " label column: %s\n", lbl) + case map[string]any: + _, _ = fmt.Fprintf(out, " label column: %v (policy: %v)\n", lbl["column"], lbl["policy"]) + } + if tc, ok := spec["time_column"].(string); ok && tc != "" { + _, _ = fmt.Fprintf(out, " time column: %s\n", tc) + } _, _ = fmt.Fprintf(out, " destination: %s\n", push.FinalDestPrefix(spec["table"].(string))) _, _ = fmt.Fprintln(out) diff --git a/internal/cli/dataset_test.go b/internal/cli/dataset_test.go index 96d7ea0..c31986e 100644 --- a/internal/cli/dataset_test.go +++ b/internal/cli/dataset_test.go @@ -63,19 +63,19 @@ func execDatasetPush(t *testing.T, args []string) (exitCode int, stdout, stderr return ExitCodeFromError(err), so.String(), se.String() } -// TestDatasetPush_UnsupportedCategory_ExitsTwo: v0.1 only supports -// image_classification end-to-end (epic #147 non-goals); the -// CLI-side gate runs before schema validation so a customer who -// passes --category=tabular_classification gets the actionable -// "v0.1 supports only image_classification" message instead of -// the schema's confusing "missing property 'schema'" (which the -// customer has no way to supply in v0.1). Bugbot review-on-self -// caught the missing gate on PR-a; landing it here. +// TestDatasetPush_UnsupportedCategory_ExitsTwo: the CLI-side category +// gate runs before schema validation so a customer who passes a +// not-yet-supported category gets an actionable message (exit 2) +// rather than the schema's confusing missing-property error. Today's +// supported set is image_classification + the tabular / time-series +// family; the other image categories (which need annotation/mask +// sidecar staging), the text family, and nonsense values are gated +// out here. Bugbot review-on-self caught the missing gate on PR-a. func TestDatasetPush_UnsupportedCategory_ExitsTwo(t *testing.T) { root := imgcLayout(t) for _, badCategory := range []string{ - "tabular_classification", // schema-valid but v0.1-unsupported - "object_detection", // ditto + "object_detection", // image category, needs sidecar staging (later) + "text_classification", // text family (later) "definitely-not-a-category", // nonsense; gate catches this too } { t.Run(badCategory, func(t *testing.T) { diff --git a/internal/push/category.go b/internal/push/category.go new file mode 100644 index 0000000..6cf21c7 --- /dev/null +++ b/internal/push/category.go @@ -0,0 +1,51 @@ +package push + +// Category families. These mirror data-ingestors' +// tracebloc_ingestor/cli/conventions.py groupings so the CLI's +// per-category behaviour (which flags are required, which local +// layout to expect, which spec fields to emit) stays in lock-step +// with what the ingestor actually resolves. +// +// Kept as a single source of truth here rather than scattered +// string comparisons across spec.go / dataset.go. + +// imageCategories take a labels CSV + an images/ directory (plus, +// for some, extra sidecar dirs handled in later increments). +var imageCategories = map[string]bool{ + "image_classification": true, + "object_detection": true, + "keypoint_detection": true, + "semantic_segmentation": true, + "instance_segmentation": true, +} + +// tabularCategories take a single CSV whose columns are described by +// a `schema` (column → SQL type) map. No sidecar files. +var tabularCategories = map[string]bool{ + "tabular_classification": true, + "tabular_regression": true, + "time_series_forecasting": true, + "time_to_event_prediction": true, +} + +// regressionClassCategories predict a numeric target rather than a +// class. The schema requires the label in object form with an +// explicit `policy` so the raw target never ships to the central +// backend by default (policy=bucket bins it first). +var regressionClassCategories = map[string]bool{ + "tabular_regression": true, + "time_series_forecasting": true, + "time_to_event_prediction": true, +} + +// IsImage reports whether category uses the labels.csv + images/ +// local layout. +func IsImage(category string) bool { return imageCategories[category] } + +// IsTabular reports whether category uses the single-CSV + schema +// local layout (no sidecar files). +func IsTabular(category string) bool { return tabularCategories[category] } + +// IsRegressionClass reports whether category predicts a numeric +// target and therefore needs label.policy (object label form). +func IsRegressionClass(category string) bool { return regressionClassCategories[category] } diff --git a/internal/push/spec.go b/internal/push/spec.go index ebb7d98..6767440 100644 --- a/internal/push/spec.go +++ b/internal/push/spec.go @@ -141,8 +141,27 @@ type SpecArgs struct { // spec.file_options.target_size so the customer's actual // resolution wins. Empty (len 0) ⇒ omit and let the ingestor // default apply. Populated by the CLI from --target-size or by - // auto-detecting the first image. + // auto-detecting the first image. (Image categories only.) TargetSize []int + + // Schema is the column→SQL-type map for tabular / time-series + // categories (required by the schema for those). Populated by the + // CLI from --schema or by inferring types from the CSV. Ignored + // for image categories. + Schema map[string]string + + // LabelPolicy is "passthrough" or "bucket". Regression-class + // categories (tabular_regression, time_series_forecasting, + // time_to_event_prediction) require the object label form with a + // policy; the CLI defaults it to "bucket" so the raw numeric + // target never ships to the central backend. Ignored for + // classification categories (which emit the string label form). + LabelPolicy string + + // TimeColumn names the time column for time_to_event_prediction. + // Emitted as the top-level `time_column` field. Empty ⇒ the + // ingestor falls back to a column named "time". + TimeColumn string } // Build produces the ingest.v1.json-conforming spec map. The @@ -173,19 +192,31 @@ func (a SpecArgs) Build() map[string]any { "category": a.Category, "table": a.Table, "intent": a.Intent, - // Trailing slash on `images` matches the schema example - // (data-ingestors/examples/yaml/image_classification.yaml, - // line 14). The ingestor treats it as a directory glob. - "csv": path.Join(prefix, "labels.csv"), - "images": path.Join(prefix, "images") + "/", - "label": a.LabelColumn, + "csv": path.Join(prefix, "labels.csv"), + } + if IsTabular(a.Category) { + a.buildTabular(spec) + } else { + // Image categories (and any not-yet-special-cased category — + // the schema validator produces the canonical error for those). + a.buildImage(spec, prefix) } + return spec +} + +// buildImage fills in the image-category fields: the images/ sidecar +// dir, the label column, and the optional target_size override. +func (a SpecArgs) buildImage(spec map[string]any, prefix string) { + // Trailing slash on `images` matches the schema example + // (data-ingestors/examples/yaml/image_classification.yaml); the + // ingestor treats it as a directory glob. + spec["images"] = path.Join(prefix, "images") + "/" + spec["label"] = a.LabelColumn // Emit the image resolution under spec.file_options.target_size — - // the same override key the helm flow + data-ingestors' - // conventions.resolve honour (it merges spec.file_options over the - // per-category default). Without this, image_classification - // defaults to 512x512 and the ingestor's Image Resolution - // Validator rejects any other size. + // the same override key data-ingestors' conventions.resolve + // honours. Without it, image_classification defaults to 512x512 + // and the ingestor's Image Resolution Validator rejects any other + // size. if len(a.TargetSize) == 2 { spec["spec"] = map[string]any{ "file_options": map[string]any{ @@ -193,7 +224,31 @@ func (a SpecArgs) Build() map[string]any { }, } } - return spec +} + +// buildTabular fills in the tabular / time-series fields: the column +// schema, the label (string form for classification, object form with +// a policy for regression-class), and time_column for +// time_to_event_prediction. +func (a SpecArgs) buildTabular(spec map[string]any) { + spec["schema"] = a.Schema + if IsRegressionClass(a.Category) { + // Regression-class tasks require the object label form with an + // explicit policy (the schema enforces this). Default to + // `bucket` so the raw numeric target never ships to the central + // backend unless the customer opts into passthrough. + policy := a.LabelPolicy + if policy == "" { + policy = "bucket" + } + spec["label"] = map[string]any{"column": a.LabelColumn, "policy": policy} + } else { + // tabular_classification: plain column-name label. + spec["label"] = a.LabelColumn + } + if a.Category == "time_to_event_prediction" && a.TimeColumn != "" { + spec["time_column"] = a.TimeColumn + } } // SharedRoot is the in-cluster mount path of the chart's shared PVC diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go index a6301cb..79f3b80 100644 --- a/internal/push/spec_test.go +++ b/internal/push/spec_test.go @@ -197,6 +197,84 @@ func TestBuild_NoTargetSize_OmitsSpecBlock(t *testing.T) { } } +// TestBuild_Tabular_PassesSchema pins the tabular Build branch: it +// emits schema-valid specs for the three label shapes — a string +// label (tabular_classification), an object label+policy +// (tabular_regression), and an object label + time_column +// (time_to_event_prediction) — and never emits an images field. +func TestBuild_Tabular_PassesSchema(t *testing.T) { + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + check := func(name string, a SpecArgs, wantLabelObject bool) { + t.Run(name, func(t *testing.T) { + spec := a.Build() + if _, hasImages := spec["images"]; hasImages { + t.Errorf("tabular spec emitted an images field: %v", spec) + } + if _, hasSchema := spec["schema"]; !hasSchema { + t.Errorf("tabular spec missing schema: %v", spec) + } + if wantLabelObject { + if _, ok := spec["label"].(map[string]any); !ok { + t.Errorf("label = %#v, want object form {column, policy}", spec["label"]) + } + } else { + if _, ok := spec["label"].(string); !ok { + t.Errorf("label = %#v, want string form", spec["label"]) + } + } + b, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + _, errs, parseErr := v.ValidateYAML(b) + if parseErr != nil { + t.Fatalf("parse: %v\n%s", parseErr, b) + } + if len(errs) != 0 { + t.Fatalf("schema validation failed: %s\n%s", schema.FormatErrors(errs), b) + } + }) + } + + check("tabular_classification", SpecArgs{ + Table: "t_clf", Category: "tabular_classification", Intent: "train", + LabelColumn: "label", + Schema: map[string]string{"f0": "FLOAT", "f1": "FLOAT", "label": "INT"}, + }, false) + + check("tabular_regression", SpecArgs{ + Table: "t_reg", Category: "tabular_regression", Intent: "train", + LabelColumn: "price", + Schema: map[string]string{"sqft": "FLOAT", "price": "FLOAT"}, + }, true) + + check("time_to_event_prediction", SpecArgs{ + Table: "t_tte", Category: "time_to_event_prediction", Intent: "train", + LabelColumn: "DEATH_EVENT", TimeColumn: "time", + Schema: map[string]string{"age": "INT", "time": "INT", "DEATH_EVENT": "INT"}, + }, true) +} + +// TestBuild_Tabular_RegressionDefaultsPolicyBucket: regression-class +// categories default to policy=bucket so the raw numeric target never +// ships to the central backend unless the customer opts out. +func TestBuild_Tabular_RegressionDefaultsPolicyBucket(t *testing.T) { + spec := SpecArgs{ + Table: "t", Category: "tabular_regression", Intent: "train", + LabelColumn: "y", Schema: map[string]string{"x": "FLOAT", "y": "FLOAT"}, + }.Build() + lbl, ok := spec["label"].(map[string]any) + if !ok { + t.Fatalf("label = %#v, want object", spec["label"]) + } + if lbl["policy"] != "bucket" { + t.Errorf("default policy = %v, want bucket", lbl["policy"]) + } +} + // TestValidateTableName_Accepts pins the names that MUST pass — // the real-world example tables plus a few edge shapes (single // char, leading underscore, mixed case, digits). A regression diff --git a/internal/push/tabular.go b/internal/push/tabular.go new file mode 100644 index 0000000..8b10e74 --- /dev/null +++ b/internal/push/tabular.go @@ -0,0 +1,229 @@ +package push + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// reservedColumns are framework-managed columns the ingestor adds +// itself; a user schema must not redeclare them — data-ingestors' +// database.create_table rejects collisions with a clear error. Schema +// auto-inference skips them so a CSV that happens to carry an `id` +// (or data_id, filename, …) column doesn't produce a schema the +// ingestor refuses. `label` is intentionally NOT reserved — it's the +// mapped label column. Mirrors database.py's _RESERVED set. +var reservedColumns = map[string]bool{ + "id": true, + "created_at": true, + "updated_at": true, + "status": true, + "data_intent": true, + "data_id": true, + "filename": true, + "extension": true, + "annotation": true, + "ingestor_id": true, +} + +// schemaInferenceSampleRows caps how many data rows InferSchema reads +// to decide each column's type. The whole CSV would be more accurate +// but a few thousand rows is plenty to distinguish INT/FLOAT/text in +// practice, and bounds the work for large files. A column whose true +// type only reveals itself past the sample (e.g. an int column that +// turns float on row 10k) is the case --schema exists to override. +const schemaInferenceSampleRows = 5000 + +// DiscoverTabular validates a local directory for a tabular / +// time-series ingestion. Unlike the image layout, tabular categories +// have NO sidecar files — the dataset IS a single CSV. The directory +// must contain exactly one .csv file; that becomes the labels/data +// CSV staged for the ingestor. +// +// The returned LocalLayout reuses the image layout's LabelsCSV field +// (staged as labels.csv) with an empty Images slice, so the existing +// tar/stream machinery handles it unchanged. +func DiscoverTabular(rootDir string) (*LocalLayout, error) { + abs, err := filepath.Abs(rootDir) + if err != nil { + return nil, fmt.Errorf("resolving %q: %w", rootDir, err) + } + st, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("reading dataset directory %q: %w", abs, err) + } + if !st.IsDir() { + return nil, fmt.Errorf( + "%q is not a directory; pass the directory containing the dataset CSV", abs) + } + + entries, err := os.ReadDir(abs) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", abs, err) + } + var csvs []string + for _, e := range entries { + if e.IsDir() { + continue + } + if strings.EqualFold(filepath.Ext(e.Name()), ".csv") { + csvs = append(csvs, e.Name()) + } + } + sort.Strings(csvs) + switch len(csvs) { + case 0: + return nil, fmt.Errorf( + "no .csv file found in %q. Tabular / time-series categories expect a "+ + "single CSV holding the dataset (one column per feature, plus the "+ + "label column).", abs) + case 1: + // happy path + default: + return nil, fmt.Errorf( + "found %d .csv files in %q (%s); the tabular layout expects exactly one. "+ + "Put the dataset CSV in its own directory and re-run.", + len(csvs), abs, strings.Join(csvs, ", ")) + } + + csvPath := filepath.Join(abs, csvs[0]) + // Lstat (not Stat) so a symlinked CSV is rejected rather than + // silently followed — mirrors the image layout's symlink guard. + info, err := os.Lstat(csvPath) + if err != nil { + return nil, fmt.Errorf("stat %s: %w", csvs[0], err) + } + if err := rejectSymlink(info, csvs[0]); err != nil { + return nil, err + } + if info.Size() > MaxSingleFileBytes { + return nil, sizeError(csvs[0], info.Size(), MaxSingleFileBytes) + } + + layout := &LocalLayout{Root: abs, LabelsCSV: csvPath, TotalBytes: info.Size()} + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. For larger datasets, the "+ + "cloud-source path is on the v0.2 roadmap (tracebloc/client#147).", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + return layout, nil +} + +// ParseSchema parses a --schema flag value of the form +// "col:TYPE,col:TYPE,..." into a column→type map. Types are passed +// through verbatim (the ingestor validates them against the SQL types +// it supports: INT, BIGINT, FLOAT, BOOLEAN, DATE, DATETIME, +// TIMESTAMP, TIME, TEXT, VARCHAR(n), ...). Whitespace around tokens +// is trimmed. +func ParseSchema(s string) (map[string]string, error) { + out := map[string]string{} + for _, pair := range strings.Split(s, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + col, typ, ok := strings.Cut(pair, ":") + col, typ = strings.TrimSpace(col), strings.TrimSpace(typ) + if !ok || col == "" || typ == "" { + return nil, fmt.Errorf( + "schema entry %q must be col:TYPE (e.g. age:INT,price:FLOAT)", pair) + } + out[col] = typ + } + if len(out) == 0 { + return nil, fmt.Errorf("--schema is empty; expected col:TYPE,col:TYPE,...") + } + return out, nil +} + +// InferSchema reads the CSV header and a sample of rows and infers a +// column→SQL-type map: all-integer columns → INT, otherwise +// all-numeric → FLOAT, otherwise VARCHAR(255). Empty cells are +// ignored when judging a column; a column with no non-empty sampled +// value falls back to VARCHAR(255). +// +// It's a convenience so customers don't hand-write a --schema for the +// common case. Non-numeric specials (timestamps, dates, booleans) +// infer as VARCHAR(255); pass --schema to declare them precisely. +// +// Framework-managed columns (see reservedColumns — id, data_id, …) +// are skipped and returned as the second value so the caller can tell +// the customer they weren't included. +func InferSchema(csvPath string) (schema map[string]string, skipped []string, err error) { + f, err := os.Open(csvPath) + if err != nil { + return nil, nil, err + } + defer func() { _ = f.Close() }() + + r := csv.NewReader(f) + header, err := r.Read() + if err != nil { + return nil, nil, fmt.Errorf("reading CSV header from %s: %w", csvPath, err) + } + if len(header) == 0 { + return nil, nil, fmt.Errorf("CSV %s has no columns", csvPath) + } + + // Per-column running judgement. + couldBeInt := make([]bool, len(header)) + couldBeFloat := make([]bool, len(header)) + sawValue := make([]bool, len(header)) + for i := range header { + couldBeInt[i] = true + couldBeFloat[i] = true + } + + for n := 0; n < schemaInferenceSampleRows; n++ { + row, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, nil, fmt.Errorf("reading CSV row from %s: %w", csvPath, err) + } + for i := 0; i < len(header) && i < len(row); i++ { + v := strings.TrimSpace(row[i]) + if v == "" { + continue + } + sawValue[i] = true + if couldBeInt[i] { + if _, e := strconv.ParseInt(v, 10, 64); e != nil { + couldBeInt[i] = false + } + } + if couldBeFloat[i] { + if _, e := strconv.ParseFloat(v, 64); e != nil { + couldBeFloat[i] = false + } + } + } + } + + schema = make(map[string]string, len(header)) + for i, col := range header { + col = strings.TrimSpace(col) + if reservedColumns[col] { + // Framework-managed (id, data_id, …): the ingestor adds it + // and rejects a schema that redeclares it. Skip + report. + skipped = append(skipped, col) + continue + } + switch { + case sawValue[i] && couldBeInt[i]: + schema[col] = "INT" + case sawValue[i] && couldBeFloat[i]: + schema[col] = "FLOAT" + default: + schema[col] = "VARCHAR(255)" + } + } + return schema, skipped, nil +} diff --git a/internal/push/tabular_test.go b/internal/push/tabular_test.go new file mode 100644 index 0000000..322101c --- /dev/null +++ b/internal/push/tabular_test.go @@ -0,0 +1,153 @@ +package push + +import ( + "os" + "path/filepath" + "testing" +) + +func writeFile(t *testing.T, dir, name, body string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +// TestDiscoverTabular_SingleCSV: a directory with exactly one CSV +// resolves to a layout whose LabelsCSV is that file and whose Images +// slice is empty (so the existing tar/stream machinery stages just +// the CSV). +func TestDiscoverTabular_SingleCSV(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", "a,b\n1,2\n") + + layout, err := DiscoverTabular(dir) + if err != nil { + t.Fatalf("DiscoverTabular: %v", err) + } + if layout.LabelsCSV != csv { + t.Errorf("LabelsCSV = %q, want %q", layout.LabelsCSV, csv) + } + if len(layout.Images) != 0 { + t.Errorf("Images = %v, want empty (tabular has no sidecar files)", layout.Images) + } + if layout.TotalBytes == 0 { + t.Errorf("TotalBytes = 0, want the CSV's size") + } +} + +// TestDiscoverTabular_NoCSV and _MultipleCSV: the layout requires +// exactly one CSV; zero or many is a clear, actionable error rather +// than a guess. +func TestDiscoverTabular_NoCSV(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "notes.txt", "hello") + if _, err := DiscoverTabular(dir); err == nil { + t.Error("DiscoverTabular with no .csv returned nil error") + } +} + +func TestDiscoverTabular_MultipleCSV(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "a.csv", "x\n1\n") + writeFile(t, dir, "b.csv", "y\n2\n") + if _, err := DiscoverTabular(dir); err == nil { + t.Error("DiscoverTabular with two .csv files returned nil error") + } +} + +// TestInferSchema covers the INT / FLOAT / VARCHAR inference from a +// CSV header + sample rows. Integer-only columns → INT, numeric (with +// a decimal) → FLOAT, anything else → VARCHAR(255). +func TestInferSchema(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", + "count,age,price,name\n1,30,9.99,alice\n2,40,19.5,bob\n") + + schema, _, err := InferSchema(csv) + if err != nil { + t.Fatalf("InferSchema: %v", err) + } + want := map[string]string{ + "count": "INT", + "age": "INT", + "price": "FLOAT", + "name": "VARCHAR(255)", + } + for col, typ := range want { + if schema[col] != typ { + t.Errorf("schema[%q] = %q, want %q (full: %v)", col, schema[col], typ, schema) + } + } +} + +// TestInferSchema_EmptyColumnIsVarchar: a column with no non-empty +// sampled value can't be typed, so it falls back to VARCHAR(255) +// rather than being mislabeled INT/FLOAT. +func TestInferSchema_EmptyColumnIsVarchar(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", "filled,empty\n1,\n2,\n") + schema, _, err := InferSchema(csv) + if err != nil { + t.Fatalf("InferSchema: %v", err) + } + if schema["empty"] != "VARCHAR(255)" { + t.Errorf("schema[empty] = %q, want VARCHAR(255)", schema["empty"]) + } + if schema["filled"] != "INT" { + t.Errorf("schema[filled] = %q, want INT", schema["filled"]) + } +} + +// TestInferSchema_SkipsReservedColumns: a CSV with an `id` (or other +// framework-managed) column must NOT produce a schema that includes +// it — data-ingestors' create_table rejects such collisions (the +// #135b guard). The reserved columns come back in the skipped list. +func TestInferSchema_SkipsReservedColumns(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", "id,feature_00,label\n1,1.5,0\n2,2.5,1\n") + + schema, skipped, err := InferSchema(csv) + if err != nil { + t.Fatalf("InferSchema: %v", err) + } + if _, present := schema["id"]; present { + t.Errorf("schema includes reserved column id: %v", schema) + } + if schema["feature_00"] != "FLOAT" || schema["label"] != "INT" { + t.Errorf("schema = %v, want feature_00:FLOAT, label:INT", schema) + } + foundID := false + for _, s := range skipped { + if s == "id" { + foundID = true + } + } + if !foundID { + t.Errorf("skipped = %v, want it to contain id", skipped) + } +} + +func TestParseSchema(t *testing.T) { + got, err := ParseSchema("age:INT, price:FLOAT ,name:VARCHAR(255)") + if err != nil { + t.Fatalf("ParseSchema: %v", err) + } + want := map[string]string{"age": "INT", "price": "FLOAT", "name": "VARCHAR(255)"} + if len(got) != len(want) { + t.Fatalf("ParseSchema len = %d, want %d (%v)", len(got), len(want), got) + } + for k, v := range want { + if got[k] != v { + t.Errorf("ParseSchema[%q] = %q, want %q", k, got[k], v) + } + } + + for _, bad := range []string{"", "age", "age:", ":INT", "age=INT"} { + if _, err := ParseSchema(bad); err == nil { + t.Errorf("ParseSchema(%q) = nil error, want rejection", bad) + } + } +} From 9712728452ec490aece671b0998aec4f14095fca Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:01:40 +0200 Subject: [PATCH 09/17] feat(dataset push): support the text family + generic sidecar staging (#14) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds text_classification and masked_language_modeling, and generalizes the staging machinery from images-only to arbitrary sidecar directories plus extra root files — the shared piece the remaining families need. - Generic sidecar staging (walk.go, stream.go): LocalLayout gains Sidecars (dir name -> files, staged under "/") and ExtraFiles (dest -> src, staged at the table root), plus FileCount(). The tar writer packages them after images, sorted for determinism. Images stays for image_classification. - text.go: DiscoverText for text_classification (labels.csv + texts/) and masked_language_modeling (labels.csv + sequences/ + a required tokenizer.json at root, staged as an ExtraFile). discoverSidecarFiles is a reusable walker (object detection / segmentation will reuse it). - Build (spec.go): the text branch emits texts/ or sequences/ + a label (text_classification only; MLM has none). - dataset.go: category dispatch + gate now accept the text family; pre-flight is text-aware (sidecar file count + tokenizer line). Validated live: text_classification — staged labels.csv + 5 texts/ -> ingestor Job 100% (5/5), rows confirmed in MySQL. MLM note: code-complete + unit-tested + accepted by the current data-ingestors schema/engine (proven by the e2e), but the *deployed* ingdemo jobs-manager carries a stale embedded schema predating MLM, so a live MLM push is rejected server-side (HTTP 400) until that image is refreshed. Not a CLI issue — the CLI surfaces the 400 cleanly. Tests: push/text_test.go (DiscoverText for both categories incl. MLM-requires-tokenizer + missing-dir errors; text Build passes the schema, MLM has no label); updated the unsupported-category gate test. go build / vet / test green. Stacked on cli#13 (tabular family) -> cli#12 (live-ingestion fixes). Co-authored-by: Claude Opus 4.8 --- internal/cli/dataset.go | 41 ++++++--- internal/cli/dataset_test.go | 4 +- internal/push/category.go | 24 +++++ internal/push/spec.go | 21 ++++- internal/push/stage.go | 4 +- internal/push/stream.go | 47 ++++++++++ internal/push/text.go | 172 +++++++++++++++++++++++++++++++++++ internal/push/text_test.go | 133 +++++++++++++++++++++++++++ internal/push/walk.go | 38 +++++++- 9 files changed, 462 insertions(+), 22 deletions(-) create mode 100644 internal/push/text.go create mode 100644 internal/push/text_test.go diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go index 23e9b9e..a5da0e7 100644 --- a/internal/cli/dataset.go +++ b/internal/cli/dataset.go @@ -280,7 +280,7 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush case a.Spec.Category == "": // Left empty by a caller; let the schema produce the canonical // "category is required" error downstream. - case push.IsTabular(a.Spec.Category) || a.Spec.Category == "image_classification": + case push.IsTabular(a.Spec.Category) || push.IsText(a.Spec.Category) || a.Spec.Category == "image_classification": // supported case push.IsImage(a.Spec.Category): return &exitError{code: 2, err: fmt.Errorf( @@ -290,9 +290,11 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush default: return &exitError{code: 2, err: fmt.Errorf( "category %q isn't supported by the CLI yet. Supported: image_classification, "+ - "tabular_classification, tabular_regression, time_series_forecasting, "+ - "time_to_event_prediction. (Text / detection / segmentation are coming; "+ - "use the helm flow for those meanwhile.)", a.Spec.Category)} + "text_classification, masked_language_modeling, and the tabular / "+ + "time-series family (tabular_classification, tabular_regression, "+ + "time_series_forecasting, time_to_event_prediction). (Object detection / "+ + "keypoint / segmentation are coming; use the helm flow for those meanwhile.)", + a.Spec.Category)} } // 3. Walk the local directory FIRST (local "fail fast"), dispatched @@ -305,9 +307,12 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush layout *push.LocalLayout err error ) - if push.IsTabular(a.Spec.Category) { + switch { + case push.IsTabular(a.Spec.Category): layout, err = push.DiscoverTabular(a.LocalPath) - } else { + case push.IsText(a.Spec.Category): + layout, err = push.DiscoverText(a.Spec.Category, a.LocalPath) + default: layout, err = push.Discover(a.LocalPath) } if err != nil { @@ -316,7 +321,8 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush // 3a. Per-category spec resolution from the local data, so the // synthesized spec carries the right fields before validation. - if push.IsTabular(a.Spec.Category) { + switch { + case push.IsTabular(a.Spec.Category): // Column schema: an explicit --schema wins; otherwise infer // INT/FLOAT/VARCHAR types from the CSV so the customer doesn't // hand-write one for the common case. @@ -340,7 +346,7 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush " (skipped framework-managed column(s): %s)\n", strings.Join(skipped, ", ")) } } - } else { + case push.IsImage(a.Spec.Category): // Image target resolution: the ingestor's image_classification // default is 512x512 and it VALIDATES (it does not resize), so // a mismatch hard-fails. Honour an explicit --target-size; @@ -365,6 +371,10 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush "resolution mismatch.\n", derr) } } + default: + // Text family: no extra per-category resolution. The label (for + // text_classification) comes straight from --label-column; + // masked_language_modeling needs neither a label nor a schema. } // 4. Synthesize the spec from flags + validate against schema. @@ -604,16 +614,23 @@ func printPushPreflight( // shouldn't convert success into failure. The exit code is // the contract. cat, _ := spec["category"].(string) - tabular := push.IsTabular(cat) _, _ = fmt.Fprintf(out, "Local dataset:\n") _, _ = fmt.Fprintf(out, " root: %s\n", layout.Root) - if tabular { + switch { + case push.IsTabular(cat): _, _ = fmt.Fprintf(out, " data CSV: %s\n", layout.LabelsCSV) if sch, ok := spec["schema"].(map[string]string); ok { _, _ = fmt.Fprintf(out, " columns: %d\n", len(sch)) } - } else { + case push.IsText(cat): + dir := push.TextSidecarDir(cat) + _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) + _, _ = fmt.Fprintf(out, " %-15s%d files\n", dir+":", len(layout.Sidecars[dir])) + if _, ok := layout.ExtraFiles["tokenizer.json"]; ok { + _, _ = fmt.Fprintf(out, " %-15s%s\n", "tokenizer:", "tokenizer.json") + } + default: _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) _, _ = fmt.Fprintf(out, " images: %d files\n", len(layout.Images)) } @@ -651,7 +668,7 @@ func printPushPreflight( if !dryRun { _, _ = fmt.Fprintf(out, "Next: stage %d files (%s) for table %q\n", - 1+len(layout.Images), push.HumanBytes(layout.TotalBytes), spec["table"]) + layout.FileCount(), push.HumanBytes(layout.TotalBytes), spec["table"]) _, _ = fmt.Fprintln(out) } } diff --git a/internal/cli/dataset_test.go b/internal/cli/dataset_test.go index c31986e..ed4bef9 100644 --- a/internal/cli/dataset_test.go +++ b/internal/cli/dataset_test.go @@ -74,8 +74,8 @@ func execDatasetPush(t *testing.T, args []string) (exitCode int, stdout, stderr func TestDatasetPush_UnsupportedCategory_ExitsTwo(t *testing.T) { root := imgcLayout(t) for _, badCategory := range []string{ - "object_detection", // image category, needs sidecar staging (later) - "text_classification", // text family (later) + "object_detection", // image category, needs annotation sidecar (later) + "keypoint_detection", // image category, needs keypoint flags (later) "definitely-not-a-category", // nonsense; gate catches this too } { t.Run(badCategory, func(t *testing.T) { diff --git a/internal/push/category.go b/internal/push/category.go index 6cf21c7..c20728a 100644 --- a/internal/push/category.go +++ b/internal/push/category.go @@ -38,6 +38,15 @@ var regressionClassCategories = map[string]bool{ "time_to_event_prediction": true, } +// textCategories take a labels CSV + a directory of text files +// (texts/ for classification, sequences/ for masked language +// modeling). masked_language_modeling additionally needs a +// tokenizer.json at the dataset root and has NO label. +var textCategories = map[string]bool{ + "text_classification": true, + "masked_language_modeling": true, +} + // IsImage reports whether category uses the labels.csv + images/ // local layout. func IsImage(category string) bool { return imageCategories[category] } @@ -49,3 +58,18 @@ func IsTabular(category string) bool { return tabularCategories[category] } // IsRegressionClass reports whether category predicts a numeric // target and therefore needs label.policy (object label form). func IsRegressionClass(category string) bool { return regressionClassCategories[category] } + +// IsText reports whether category uses the labels.csv + text-file +// directory (texts/ or sequences/) local layout. +func IsText(category string) bool { return textCategories[category] } + +// TextSidecarDir returns the sidecar directory name a text category +// expects: "sequences" for masked_language_modeling, "texts" for +// text_classification. (Used both as the local subdir to stage and +// the spec field to emit.) +func TextSidecarDir(category string) string { + if category == "masked_language_modeling" { + return "sequences" + } + return "texts" +} diff --git a/internal/push/spec.go b/internal/push/spec.go index 6767440..4cf901e 100644 --- a/internal/push/spec.go +++ b/internal/push/spec.go @@ -194,9 +194,12 @@ func (a SpecArgs) Build() map[string]any { "intent": a.Intent, "csv": path.Join(prefix, "labels.csv"), } - if IsTabular(a.Category) { + switch { + case IsTabular(a.Category): a.buildTabular(spec) - } else { + case IsText(a.Category): + a.buildText(spec, prefix) + default: // Image categories (and any not-yet-special-cased category — // the schema validator produces the canonical error for those). a.buildImage(spec, prefix) @@ -204,6 +207,20 @@ func (a SpecArgs) Build() map[string]any { return spec } +// buildText fills in the text-family fields: the text-file sidecar +// directory (texts/ for text_classification, sequences/ for +// masked_language_modeling) and the label. masked_language_modeling +// has NO label (the schema doesn't require one for it). +func (a SpecArgs) buildText(spec map[string]any, prefix string) { + dir := TextSidecarDir(a.Category) + // Trailing slash matches the directory-glob convention the + // ingestor uses for sidecar dirs. + spec[dir] = path.Join(prefix, dir) + "/" + if a.Category == "text_classification" { + spec["label"] = a.LabelColumn + } +} + // buildImage fills in the image-category fields: the images/ sidecar // dir, the label column, and the optional target_size override. func (a SpecArgs) buildImage(spec map[string]any, prefix string) { diff --git a/internal/push/stage.go b/internal/push/stage.go index ec39607..eed5885 100644 --- a/internal/push/stage.go +++ b/internal/push/stage.go @@ -138,7 +138,7 @@ func Stage(ctx context.Context, opts StageOptions) error { // 5. Stream the tar. This is where actual bytes flow. The // progress bar (if TTY) renders during this call. _, _ = fmt.Fprintf(opts.Out, "Streaming %d files (%s) for table %q...\n", - 1+len(opts.Layout.Images), HumanBytes(opts.Layout.TotalBytes), opts.Table) + opts.Layout.FileCount(), HumanBytes(opts.Layout.TotalBytes), opts.Table) if err := StreamLayout(ctx, opts.Executor, opts.Namespace, podName, "stage", @@ -148,6 +148,6 @@ func Stage(ctx context.Context, opts StageOptions) error { // 6. Print "done" message. The deferred cleanup runs after this. _, _ = fmt.Fprintf(opts.Out, "Staged %d files for table %q\n", - 1+len(opts.Layout.Images), opts.Table) + opts.Layout.FileCount(), opts.Table) return nil } diff --git a/internal/push/stream.go b/internal/push/stream.go index baba3cb..bdeff33 100644 --- a/internal/push/stream.go +++ b/internal/push/stream.go @@ -10,6 +10,7 @@ import ( "os" "path" "path/filepath" + "sort" corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -388,12 +389,58 @@ func writeLayoutTar(w io.Writer, layout *LocalLayout) (err error) { } } + // Extra root-level files (e.g. masked_language_modeling's + // tokenizer.json), staged at the table root under their dest name. + // Sorted for deterministic stream order. + for _, dest := range sortedKeys(layout.ExtraFiles) { + n, err := writeTarFile(tw, layout.ExtraFiles[dest], dest) + if err != nil { + return fmt.Errorf("packaging %s: %w", dest, err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s after streaming %s (reached %s)", + HumanBytes(MaxTotalBytes), dest, HumanBytes(totalBytes)) + } + } + + // Generic sidecar directories (texts/, sequences/, and — later — + // annotations/, masks/), each staged under "/". + // Sorted by dir name for deterministic stream order. + for _, name := range sortedKeys(layout.Sidecars) { + for _, abs := range layout.Sidecars[name] { + dst := path.Join(name, filepath.Base(abs)) + n, err := writeTarFile(tw, abs, dst) + if err != nil { + return fmt.Errorf("packaging %s: %w", abs, err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s after streaming %s (reached %s)", + HumanBytes(MaxTotalBytes), dst, HumanBytes(totalBytes)) + } + } + } + // tw.Close() in the defer above writes the tar footer // (two zero blocks). Without that, GNU tar treats the archive // as truncated and refuses to extract. return nil } +// sortedKeys returns a map's string keys in sorted order, for +// deterministic iteration when packaging ExtraFiles / Sidecars. +func sortedKeys[V any](m map[string]V) []string { + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + sort.Strings(ks) + return ks +} + // writeTarFile writes one file from `src` into tw under the // archive-relative name `dst`. Streams the file body — no full- // read into memory — so a single 500 MiB image doesn't balloon diff --git a/internal/push/text.go b/internal/push/text.go new file mode 100644 index 0000000..163292e --- /dev/null +++ b/internal/push/text.go @@ -0,0 +1,172 @@ +package push + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// textExtensions are the file types the text / MLM ingestor reads by +// default (the schema's file_options.extension allows .txt / .text). +var textExtensions = map[string]struct{}{ + ".txt": {}, + ".text": {}, +} + +// DiscoverText validates a local directory for a text-family ingestion +// (text_classification or masked_language_modeling): +// +// - /labels.csv (required) +// - //*.txt (required; sidecar = texts | sequences) +// - /tokenizer.json (required for masked_language_modeling) +// +// The returned layout stages the CSV (as labels.csv), the text files +// under "/", and — for MLM — tokenizer.json at the table root +// (the ingestor reads it from SRC_PATH/tokenizer.json for [MASK]/[PAD]). +func DiscoverText(category, rootDir string) (*LocalLayout, error) { + abs, err := filepath.Abs(rootDir) + if err != nil { + return nil, fmt.Errorf("resolving %q: %w", rootDir, err) + } + st, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("reading dataset directory %q: %w", abs, err) + } + if !st.IsDir() { + return nil, fmt.Errorf( + "%q is not a directory; pass the directory containing labels.csv + the text files", abs) + } + + layout := &LocalLayout{ + Root: abs, + Sidecars: map[string][]string{}, + ExtraFiles: map[string]string{}, + } + dirName := TextSidecarDir(category) + + // labels.csv (required) — same Lstat-based symlink guard as the + // image layout. + labelsPath := filepath.Join(abs, "labels.csv") + labelsStat, err := os.Lstat(labelsPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing labels.csv in %q. Text categories expect "+ + "/labels.csv + /%s/.", abs, dirName) + } + return nil, fmt.Errorf("stat labels.csv: %w", err) + } + if err := rejectSymlink(labelsStat, "labels.csv"); err != nil { + return nil, err + } + if labelsStat.IsDir() { + return nil, fmt.Errorf("%q is a directory, not a file", labelsPath) + } + if labelsStat.Size() > MaxSingleFileBytes { + return nil, sizeError("labels.csv", labelsStat.Size(), MaxSingleFileBytes) + } + layout.LabelsCSV = labelsPath + layout.TotalBytes += labelsStat.Size() + + // Sidecar text dir (required). + files, sidecarBytes, err := discoverSidecarFiles(abs, dirName, textExtensions) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, fmt.Errorf( + "no .txt files found in %q. Text categories expect "+ + "/%s/*.txt.", filepath.Join(abs, dirName), dirName) + } + layout.Sidecars[dirName] = files + layout.TotalBytes += sidecarBytes + + // masked_language_modeling needs a tokenizer.json at the root. + if category == "masked_language_modeling" { + tokPath := filepath.Join(abs, "tokenizer.json") + tokStat, terr := os.Lstat(tokPath) + if terr != nil { + if errors.Is(terr, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing tokenizer.json in %q. masked_language_modeling requires a "+ + "tokenizer.json (HuggingFace tokenizers format) at the dataset root; "+ + "the ingestor reads it for the [MASK]/[PAD] tokens.", abs) + } + return nil, fmt.Errorf("stat tokenizer.json: %w", terr) + } + if err := rejectSymlink(tokStat, "tokenizer.json"); err != nil { + return nil, err + } + if tokStat.IsDir() { + return nil, fmt.Errorf("%q is a directory, not a file", tokPath) + } + if tokStat.Size() > MaxSingleFileBytes { + return nil, sizeError("tokenizer.json", tokStat.Size(), MaxSingleFileBytes) + } + layout.ExtraFiles["tokenizer.json"] = tokPath + layout.TotalBytes += tokStat.Size() + } + + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. For larger datasets, the "+ + "cloud-source path is on the v0.2 roadmap (tracebloc/client#147).", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + return layout, nil +} + +// discoverSidecarFiles walks / (non-recursive) for files +// whose extension is in exts, rejecting symlinks and enforcing the +// single-file cap. Returns the absolute paths + their total size. A +// missing directory is an error (the caller's category requires it). +// +// Shared by the text family today; object detection / segmentation +// (annotations/, masks/) will reuse it in a later increment. +func discoverSidecarFiles(root, dirName string, exts map[string]struct{}) ([]string, int64, error) { + dir := filepath.Join(root, dirName) + dirStat, err := os.Lstat(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, 0, fmt.Errorf("missing %s/ subdirectory in %q", dirName, root) + } + return nil, 0, fmt.Errorf("stat %s/: %w", dirName, err) + } + if err := rejectSymlink(dirStat, dirName); err != nil { + return nil, 0, err + } + if !dirStat.IsDir() { + return nil, 0, fmt.Errorf("%q exists but is not a directory", dir) + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil, 0, fmt.Errorf("reading %s/: %w", dirName, err) + } + var ( + files []string + total int64 + ) + for _, entry := range entries { + if entry.IsDir() { + continue + } + if _, ok := exts[strings.ToLower(filepath.Ext(entry.Name()))]; !ok { + continue + } + info, err := entry.Info() + if err != nil { + return nil, 0, fmt.Errorf("stat %q: %w", entry.Name(), err) + } + if err := rejectSymlink(info, filepath.Join(dirName, entry.Name())); err != nil { + return nil, 0, err + } + if info.Size() > MaxSingleFileBytes { + return nil, 0, sizeError(filepath.Join(dirName, entry.Name()), info.Size(), MaxSingleFileBytes) + } + files = append(files, filepath.Join(dir, entry.Name())) + total += info.Size() + } + return files, total, nil +} diff --git a/internal/push/text_test.go b/internal/push/text_test.go new file mode 100644 index 0000000..66ae629 --- /dev/null +++ b/internal/push/text_test.go @@ -0,0 +1,133 @@ +package push + +import ( + "os" + "path/filepath" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/tracebloc/cli/internal/schema" +) + +// mkTextDir builds a text-family dataset dir: labels.csv + a sidecar +// directory (texts/ or sequences/) with two .txt files, optionally +// plus a tokenizer.json at the root (for MLM). +func mkTextDir(t *testing.T, sidecar string, withTokenizer bool) string { + t.Helper() + dir := t.TempDir() + writeFile(t, dir, "labels.csv", "filename,label\na.txt,pos\nb.txt,neg\n") + sub := filepath.Join(dir, sidecar) + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sub, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sub, "b.txt"), []byte("world"), 0o644); err != nil { + t.Fatal(err) + } + if withTokenizer { + writeFile(t, dir, "tokenizer.json", `{"version":"1.0"}`) + } + return dir +} + +// TestDiscoverText_Classification: text_classification stages +// labels.csv + the texts/ directory, no images, no extra files. +func TestDiscoverText_Classification(t *testing.T) { + dir := mkTextDir(t, "texts", false) + layout, err := DiscoverText("text_classification", dir) + if err != nil { + t.Fatalf("DiscoverText: %v", err) + } + if len(layout.Sidecars["texts"]) != 2 { + t.Errorf("texts files = %d, want 2", len(layout.Sidecars["texts"])) + } + if len(layout.Images) != 0 { + t.Errorf("Images should be empty for text, got %v", layout.Images) + } + if len(layout.ExtraFiles) != 0 { + t.Errorf("ExtraFiles should be empty for text_classification, got %v", layout.ExtraFiles) + } + if got := layout.FileCount(); got != 3 { // labels.csv + 2 texts + t.Errorf("FileCount = %d, want 3", got) + } +} + +// TestDiscoverText_MLM_RequiresTokenizer: masked_language_modeling +// errors without tokenizer.json, and stages it as an ExtraFile when +// present (the ingestor reads SRC_PATH/tokenizer.json). +func TestDiscoverText_MLM_RequiresTokenizer(t *testing.T) { + if _, err := DiscoverText("masked_language_modeling", mkTextDir(t, "sequences", false)); err == nil { + t.Error("DiscoverText(MLM) without tokenizer.json returned nil error") + } + + layout, err := DiscoverText("masked_language_modeling", mkTextDir(t, "sequences", true)) + if err != nil { + t.Fatalf("DiscoverText(MLM): %v", err) + } + if len(layout.Sidecars["sequences"]) != 2 { + t.Errorf("sequences files = %d, want 2", len(layout.Sidecars["sequences"])) + } + if layout.ExtraFiles["tokenizer.json"] == "" { + t.Errorf("tokenizer.json not staged as an ExtraFile: %v", layout.ExtraFiles) + } + if got := layout.FileCount(); got != 4 { // labels.csv + 2 sequences + tokenizer + t.Errorf("FileCount = %d, want 4", got) + } +} + +// TestDiscoverText_MissingSidecarDir: a text dataset without its +// text-file directory is a clear error, not a silent empty stage. +func TestDiscoverText_MissingSidecarDir(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "labels.csv", "filename,label\na.txt,pos\n") + if _, err := DiscoverText("text_classification", dir); err == nil { + t.Error("DiscoverText without texts/ returned nil error") + } +} + +// TestBuild_Text_PassesSchema: the text Build branch emits the right +// sidecar field (texts vs sequences), a label for text_classification +// but NOT for masked_language_modeling, never an images field, and a +// schema-valid spec. +func TestBuild_Text_PassesSchema(t *testing.T) { + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + check := func(name string, a SpecArgs, wantSidecar string, wantLabel bool) { + t.Run(name, func(t *testing.T) { + spec := a.Build() + if _, ok := spec[wantSidecar]; !ok { + t.Errorf("spec missing %q field: %v", wantSidecar, spec) + } + if _, hasImages := spec["images"]; hasImages { + t.Errorf("text spec emitted an images field: %v", spec) + } + if _, hasLabel := spec["label"]; hasLabel != wantLabel { + t.Errorf("label present = %v, want %v (%v)", hasLabel, wantLabel, spec) + } + b, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + _, errs, parseErr := v.ValidateYAML(b) + if parseErr != nil { + t.Fatalf("parse: %v\n%s", parseErr, b) + } + if len(errs) != 0 { + t.Fatalf("schema validation failed: %s\n%s", schema.FormatErrors(errs), b) + } + }) + } + + check("text_classification", SpecArgs{ + Table: "t_txt", Category: "text_classification", Intent: "train", LabelColumn: "label", + }, "texts", true) + + check("masked_language_modeling", SpecArgs{ + Table: "t_mlm", Category: "masked_language_modeling", Intent: "train", + }, "sequences", false) +} diff --git a/internal/push/walk.go b/internal/push/walk.go index bb7b15f..a6bb163 100644 --- a/internal/push/walk.go +++ b/internal/push/walk.go @@ -46,16 +46,46 @@ type LocalLayout struct { // Images is the list of absolute paths to image files under // Root/images/. Order is filesystem-walk order — Discover // doesn't sort, so callers that need determinism (e.g. - // reproducible-build tests) sort before use. + // reproducible-build tests) sort before use. Empty for non-image + // categories (which use Sidecars instead). Images []string + // Sidecars maps a sidecar directory name (e.g. "texts", + // "sequences", "annotations", "masks") to the absolute paths of + // the files in it. Each is staged under "/" — the + // generic counterpart to Images, used by the text family (and, + // later, object detection / segmentation). nil for image_ + // classification (which uses Images) and tabular (no sidecars). + Sidecars map[string][]string + + // ExtraFiles maps a staged destination filename to its absolute + // source path, for single root-level files beyond labels.csv — + // e.g. masked_language_modeling's tokenizer.json, which the + // ingestor reads from SRC_PATH/tokenizer.json. Staged verbatim at + // the table root. + ExtraFiles map[string]string + // TotalBytes is the sum of all files Discover will stage — - // labels.csv plus every entry in Images. Pre-computed during - // the walk so the size-cap check + the progress bar (PR-b) - // can read it without re-stat'ing. + // labels.csv plus every entry in Images / Sidecars / ExtraFiles. + // Pre-computed during the walk so the size-cap check + the + // progress bar can read it without re-stat'ing. TotalBytes int64 } +// FileCount returns the total number of files this layout stages: +// labels.csv, every ExtraFile, and every Images / Sidecars entry. Used +// for the "staging N files" messaging so it's accurate across all +// category families. +func (l *LocalLayout) FileCount() int { + n := 1 // labels.csv + n += len(l.Images) + n += len(l.ExtraFiles) + for _, files := range l.Sidecars { + n += len(files) + } + return n +} + // imageExtensions accepts the file types the chart's // image_classification ingestor processes by default. From // data-ingestors' FileTypeValidator(images) defaults: .jpg, .jpeg, From 67ecb781d85abb2c24193b03b4609bc5b19cc7b1 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:05:03 +0200 Subject: [PATCH 10/17] feat(dataset push): object_detection + keypoint_detection (+ schema re-sync) (#15) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the two remaining engine-supported image categories, taking the CLI to 9/10 modalities (only semantic_segmentation remains, blocked on the ingestor — data-ingestors#136). - object_detection: reuses the generic sidecar walker for annotations/ (.xml). Validated live end-to-end — 128 records (bounding boxes) ingested, rows confirmed in MySQL. - keypoint_detection: labels.csv + images/ (keypoint coords live in the CSV's Annotation column, read server-side). Adds --number-of-keypoints (required; no default). Emits target_size + number_of_keypoints as TOP-LEVEL fields, which the schema's keypoint conditional requires. - Re-synced the embedded schema from data-ingestors develop. The vendored copy was stale: it lacked keypoint's top-level target_size + number_of_keypoints and their required-for-keypoint conditional, so the CLI couldn't validate a keypoint spec at all. `ingest validate` and dataset push now validate keypoint correctly. Schema-skew findings (deployment/release hygiene, NOT CLI bugs): * sync-schema.sh defaults to data-ingestors *master*, which is stale (lacks both MLM and keypoint); the current schema is on *develop*. Repoint the sync source to develop, or promote develop -> master. (sync --check vs master flags this drift — pre-existing, surfaced here.) * The deployed ingdemo client runs jobs-manager and the ingestor on DIFFERENT schema versions: the ingestor (newer) REQUIRES keypoint's top-level fields; jobs-manager (older) REJECTS them as additional properties. So keypoint can't be ingested there until both components are refreshed to a matching schema. The CLI's emission is correct against the current/consistent schema (unit-verified). OD is unaffected (no new fields). Tests: push/image_extras_test.go (DiscoverObjectDetection + missing-annotations); spec_test.go (OD emits annotations; keypoint emits top-level target_size + number_of_keypoints; both pass the schema); updated the unsupported-category gate test (now segmentation only). go build / vet / test green. Stacked on cli#14 (text) -> #13 (tabular) -> #12 (fixes). Co-authored-by: Claude Opus 4.8 --- internal/cli/dataset.go | 57 ++++++++++++++++++++--------- internal/cli/dataset_test.go | 4 +- internal/push/image_extras.go | 51 ++++++++++++++++++++++++++ internal/push/image_extras_test.go | 59 ++++++++++++++++++++++++++++++ internal/push/spec.go | 45 ++++++++++++++++++----- internal/push/spec_test.go | 55 ++++++++++++++++++++++++++++ internal/schema/ingest.v1.json | 22 +++++++++++ 7 files changed, 264 insertions(+), 29 deletions(-) create mode 100644 internal/push/image_extras.go create mode 100644 internal/push/image_extras_test.go diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go index a5da0e7..1551767 100644 --- a/internal/cli/dataset.go +++ b/internal/cli/dataset.go @@ -72,14 +72,15 @@ func newDatasetPushCmd() *cobra.Command { // Ingest-spec flags. image_classification + the tabular / // time-series family are supported today; text + detection + // segmentation land in later increments. - table string - category string - intent string - labelColumn string - targetSize string - schemaFlag string - labelPolicy string - timeColumn string + table string + category string + intent string + labelColumn string + targetSize string + schemaFlag string + labelPolicy string + timeColumn string + numberOfKeypoints int // Operations flags. dryRun bool @@ -153,6 +154,7 @@ Exit codes: Spec: push.SpecArgs{ Table: table, Category: category, Intent: intent, LabelColumn: labelColumn, LabelPolicy: labelPolicy, TimeColumn: timeColumn, + NumberOfKeypoints: numberOfKeypoints, }, TargetSizeFlag: targetSize, SchemaFlag: schemaFlag, @@ -198,6 +200,8 @@ Exit codes: "passthrough|bucket (default bucket — bins the target so the raw value never leaves the cluster)") cmd.Flags().StringVar(&timeColumn, "time-column", "", "time_to_event_prediction only: name of the time/duration column (default: a column named \"time\")") + cmd.Flags().IntVar(&numberOfKeypoints, "number-of-keypoints", 0, + "keypoint_detection only: number of keypoints per sample (required; e.g. 17 for COCO pose)") cmd.Flags().BoolVar(&dryRun, "dry-run", false, "validate + discover + walk, but don't create any cluster resources") @@ -280,21 +284,24 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush case a.Spec.Category == "": // Left empty by a caller; let the schema produce the canonical // "category is required" error downstream. - case push.IsTabular(a.Spec.Category) || push.IsText(a.Spec.Category) || a.Spec.Category == "image_classification": + case push.IsTabular(a.Spec.Category) || push.IsText(a.Spec.Category) || + a.Spec.Category == "image_classification" || + a.Spec.Category == "object_detection" || + a.Spec.Category == "keypoint_detection": // supported case push.IsImage(a.Spec.Category): + // semantic_segmentation / instance_segmentation return &exitError{code: 2, err: fmt.Errorf( - "category %q isn't supported by the CLI yet — it needs annotation/mask "+ - "sidecar staging that's coming in a later release. Supported image "+ - "category: image_classification.", a.Spec.Category)} + "category %q isn't supported by the CLI yet. semantic_segmentation is "+ + "blocked on the ingestor's mask-sidecar support (data-ingestors#136), and "+ + "instance_segmentation isn't implemented. Supported image categories: "+ + "image_classification, object_detection, keypoint_detection.", a.Spec.Category)} default: return &exitError{code: 2, err: fmt.Errorf( - "category %q isn't supported by the CLI yet. Supported: image_classification, "+ - "text_classification, masked_language_modeling, and the tabular / "+ - "time-series family (tabular_classification, tabular_regression, "+ - "time_series_forecasting, time_to_event_prediction). (Object detection / "+ - "keypoint / segmentation are coming; use the helm flow for those meanwhile.)", - a.Spec.Category)} + "category %q isn't a recognized task category. Supported: image_classification, "+ + "object_detection, keypoint_detection, text_classification, "+ + "masked_language_modeling, tabular_classification, tabular_regression, "+ + "time_series_forecasting, time_to_event_prediction.", a.Spec.Category)} } // 3. Walk the local directory FIRST (local "fail fast"), dispatched @@ -312,7 +319,10 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush layout, err = push.DiscoverTabular(a.LocalPath) case push.IsText(a.Spec.Category): layout, err = push.DiscoverText(a.Spec.Category, a.LocalPath) + case a.Spec.Category == "object_detection": + layout, err = push.DiscoverObjectDetection(a.LocalPath) default: + // image_classification + keypoint_detection: labels.csv + images/. layout, err = push.Discover(a.LocalPath) } if err != nil { @@ -347,6 +357,14 @@ func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPush } } case push.IsImage(a.Spec.Category): + // keypoint_detection needs --number-of-keypoints (dataset- + // specific, no default). Catch it here with an actionable + // message rather than letting the ingestor fail mid-run. + if a.Spec.Category == "keypoint_detection" && a.Spec.NumberOfKeypoints <= 0 { + return &exitError{code: 2, err: errors.New( + "keypoint_detection requires --number-of-keypoints (e.g. " + + "--number-of-keypoints 17); it's dataset-specific and has no default")} + } // Image target resolution: the ingestor's image_classification // default is 512x512 and it VALIDATES (it does not resize), so // a mismatch hard-fails. Honour an explicit --target-size; @@ -633,6 +651,9 @@ func printPushPreflight( default: _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) _, _ = fmt.Fprintf(out, " images: %d files\n", len(layout.Images)) + if anns := layout.Sidecars["annotations"]; len(anns) > 0 { + _, _ = fmt.Fprintf(out, " annotations: %d files\n", len(anns)) + } } _, _ = fmt.Fprintf(out, " total size: %s\n", push.HumanBytes(layout.TotalBytes)) _, _ = fmt.Fprintln(out) diff --git a/internal/cli/dataset_test.go b/internal/cli/dataset_test.go index ed4bef9..5d5cfcf 100644 --- a/internal/cli/dataset_test.go +++ b/internal/cli/dataset_test.go @@ -74,8 +74,8 @@ func execDatasetPush(t *testing.T, args []string) (exitCode int, stdout, stderr func TestDatasetPush_UnsupportedCategory_ExitsTwo(t *testing.T) { root := imgcLayout(t) for _, badCategory := range []string{ - "object_detection", // image category, needs annotation sidecar (later) - "keypoint_detection", // image category, needs keypoint flags (later) + "semantic_segmentation", // blocked on the ingestor (data-ingestors#136) + "instance_segmentation", // not implemented "definitely-not-a-category", // nonsense; gate catches this too } { t.Run(badCategory, func(t *testing.T) { diff --git a/internal/push/image_extras.go b/internal/push/image_extras.go new file mode 100644 index 0000000..5e1e0f1 --- /dev/null +++ b/internal/push/image_extras.go @@ -0,0 +1,51 @@ +package push + +import ( + "fmt" + "path/filepath" +) + +// xmlExtensions are the annotation file types object_detection reads +// (Pascal VOC XML). +var xmlExtensions = map[string]struct{}{".xml": {}} + +// DiscoverObjectDetection validates a local object_detection dataset: +// +// - /labels.csv (required) +// - /images/* (required) +// - /annotations/*.xml (required; Pascal VOC) +// +// It builds on the image-classification layout (labels.csv + images/) +// and adds the annotations/ sidecar via the shared sidecar walker, so +// the existing tar/stream machinery stages annotations under +// "annotations/". +func DiscoverObjectDetection(rootDir string) (*LocalLayout, error) { + layout, err := Discover(rootDir) // labels.csv + images/ (+ caps + symlink guards) + if err != nil { + return nil, err + } + + annotations, annoBytes, err := discoverSidecarFiles(layout.Root, "annotations", xmlExtensions) + if err != nil { + return nil, err + } + if len(annotations) == 0 { + return nil, fmt.Errorf( + "no .xml annotation files found in %q. object_detection expects "+ + "/annotations/*.xml (Pascal VOC).", + filepath.Join(layout.Root, "annotations")) + } + if layout.Sidecars == nil { + layout.Sidecars = map[string][]string{} + } + layout.Sidecars["annotations"] = annotations + layout.TotalBytes += annoBytes + + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. For larger datasets, the "+ + "cloud-source path is on the v0.2 roadmap (tracebloc/client#147).", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + return layout, nil +} diff --git a/internal/push/image_extras_test.go b/internal/push/image_extras_test.go new file mode 100644 index 0000000..8ed8d71 --- /dev/null +++ b/internal/push/image_extras_test.go @@ -0,0 +1,59 @@ +package push + +import ( + "os" + "path/filepath" + "testing" +) + +// mkODDir builds an object_detection dataset dir: labels.csv + +// images/001.jpg, plus annotations/001.xml when withAnnotations. +func mkODDir(t *testing.T, withAnnotations bool) string { + t.Helper() + dir := t.TempDir() + writeFile(t, dir, "labels.csv", "image_label,filename\ncat,001.jpg\n") + imgs := filepath.Join(dir, "images") + if err := os.MkdirAll(imgs, 0o755); err != nil { + t.Fatal(err) + } + // JPEG magic bytes; Discover checks extension + size, not decode. + if err := os.WriteFile(filepath.Join(imgs, "001.jpg"), []byte("\xff\xd8\xff\xe0"), 0o644); err != nil { + t.Fatal(err) + } + if withAnnotations { + ann := filepath.Join(dir, "annotations") + if err := os.MkdirAll(ann, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(ann, "001.xml"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// TestDiscoverObjectDetection: a valid OD layout yields images + +// the annotations sidecar, staged together. +func TestDiscoverObjectDetection(t *testing.T) { + layout, err := DiscoverObjectDetection(mkODDir(t, true)) + if err != nil { + t.Fatalf("DiscoverObjectDetection: %v", err) + } + if len(layout.Images) != 1 { + t.Errorf("images = %d, want 1", len(layout.Images)) + } + if len(layout.Sidecars["annotations"]) != 1 { + t.Errorf("annotations = %d, want 1", len(layout.Sidecars["annotations"])) + } + if got := layout.FileCount(); got != 3 { // labels.csv + image + xml + t.Errorf("FileCount = %d, want 3", got) + } +} + +// TestDiscoverObjectDetection_MissingAnnotations: OD without an +// annotations/ directory is a clear error. +func TestDiscoverObjectDetection_MissingAnnotations(t *testing.T) { + if _, err := DiscoverObjectDetection(mkODDir(t, false)); err == nil { + t.Error("DiscoverObjectDetection without annotations/ returned nil error") + } +} diff --git a/internal/push/spec.go b/internal/push/spec.go index 4cf901e..2133976 100644 --- a/internal/push/spec.go +++ b/internal/push/spec.go @@ -162,6 +162,14 @@ type SpecArgs struct { // Emitted as the top-level `time_column` field. Empty ⇒ the // ingestor falls back to a column named "time". TimeColumn string + + // NumberOfKeypoints is the keypoints-per-sample count for + // keypoint_detection (required there; no convention default). + // Emitted under spec.file_options.number_of_keypoints — the schema + // allows arbitrary file_options keys, and conventions.resolve reads + // it from there, so this needs no top-level schema field. 0 ⇒ + // unset (ignored for non-keypoint categories). + NumberOfKeypoints int } // Build produces the ingest.v1.json-conforming spec map. The @@ -221,19 +229,38 @@ func (a SpecArgs) buildText(spec map[string]any, prefix string) { } } -// buildImage fills in the image-category fields: the images/ sidecar -// dir, the label column, and the optional target_size override. +// buildImage fills in the image-family fields for image_classification, +// object_detection, and keypoint_detection: the images/ dir, the label, +// object_detection's annotations/ dir, and the resolution overrides. +// +// keypoint_detection emits target_size + number_of_keypoints as +// TOP-LEVEL fields — the schema's keypoint conditional requires them +// there (both are dataset-specific, no convention defaults), and the +// ingestor validates against that conditional. image_classification +// and object_detection emit target_size under spec.file_options (the +// override key conventions.resolve reads); without it, +// image_classification defaults to 512x512 and the Image Resolution +// Validator rejects other sizes. func (a SpecArgs) buildImage(spec map[string]any, prefix string) { - // Trailing slash on `images` matches the schema example + // Trailing slash on the dir fields matches the schema example // (data-ingestors/examples/yaml/image_classification.yaml); the - // ingestor treats it as a directory glob. + // ingestor treats them as directory globs. spec["images"] = path.Join(prefix, "images") + "/" spec["label"] = a.LabelColumn - // Emit the image resolution under spec.file_options.target_size — - // the same override key data-ingestors' conventions.resolve - // honours. Without it, image_classification defaults to 512x512 - // and the ingestor's Image Resolution Validator rejects any other - // size. + if a.Category == "object_detection" { + spec["annotations"] = path.Join(prefix, "annotations") + "/" + } + + if a.Category == "keypoint_detection" { + if len(a.TargetSize) == 2 { + spec["target_size"] = []int{a.TargetSize[0], a.TargetSize[1]} + } + if a.NumberOfKeypoints > 0 { + spec["number_of_keypoints"] = a.NumberOfKeypoints + } + return + } + if len(a.TargetSize) == 2 { spec["spec"] = map[string]any{ "file_options": map[string]any{ diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go index 79f3b80..2167008 100644 --- a/internal/push/spec_test.go +++ b/internal/push/spec_test.go @@ -275,6 +275,61 @@ func TestBuild_Tabular_RegressionDefaultsPolicyBucket(t *testing.T) { } } +// TestBuild_ImageExtras_PassSchema pins object_detection and +// keypoint_detection: OD emits an annotations field; keypoint emits +// number_of_keypoints under spec.file_options (NOT a top-level field, +// so it validates against the current vendored schema) and no +// annotations. Both must validate. +func TestBuild_ImageExtras_PassSchema(t *testing.T) { + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + validate := func(t *testing.T, spec map[string]any) { + b, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + _, errs, parseErr := v.ValidateYAML(b) + if parseErr != nil { + t.Fatalf("parse: %v\n%s", parseErr, b) + } + if len(errs) != 0 { + t.Fatalf("schema validation failed: %s\n%s", schema.FormatErrors(errs), b) + } + } + + t.Run("object_detection", func(t *testing.T) { + spec := SpecArgs{ + Table: "od", Category: "object_detection", Intent: "train", + LabelColumn: "image_label", TargetSize: []int{448, 448}, + }.Build() + if _, ok := spec["annotations"]; !ok { + t.Errorf("OD spec missing annotations: %v", spec) + } + validate(t, spec) + }) + + t.Run("keypoint_detection", func(t *testing.T) { + spec := SpecArgs{ + Table: "kp", Category: "keypoint_detection", Intent: "train", + LabelColumn: "image_label", TargetSize: []int{448, 448}, NumberOfKeypoints: 9, + }.Build() + if _, ok := spec["annotations"]; ok { + t.Errorf("keypoint spec should not emit annotations: %v", spec) + } + // keypoint requires target_size + number_of_keypoints TOP-LEVEL + // (the schema's keypoint conditional), not under file_options. + if spec["number_of_keypoints"] != 9 { + t.Errorf("top-level number_of_keypoints = %v, want 9", spec["number_of_keypoints"]) + } + if ts, ok := spec["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 448 || ts[1] != 448 { + t.Errorf("top-level target_size = %#v, want [448 448]", spec["target_size"]) + } + validate(t, spec) + }) +} + // TestValidateTableName_Accepts pins the names that MUST pass — // the real-world example tables plus a few edge shapes (single // char, leading underscore, mixed case, digits). A regression diff --git a/internal/schema/ingest.v1.json b/internal/schema/ingest.v1.json index 21a0c46..d8eb83c 100644 --- a/internal/schema/ingest.v1.json +++ b/internal/schema/ingest.v1.json @@ -138,6 +138,20 @@ "description": "Name of the time column for time_to_event_prediction. Falls back to a column named `time` if unset." }, + "target_size": { + "type": "array", + "items": { "type": "integer", "minimum": 1 }, + "minItems": 2, + "maxItems": 2, + "description": "[height, width] to resize images to. Required for keypoint_detection (no default — depends on the customer's pose model). Other image categories use category defaults if unset." + }, + + "number_of_keypoints": { + "type": "integer", + "minimum": 1, + "description": "Number of keypoints per sample. Required for keypoint_detection — dataset-specific (e.g. 17 for COCO pose, 9 for the upper-body sample in templates/keypoint_detection)." + }, + "data_id": { "type": "object", "additionalProperties": false, @@ -356,6 +370,14 @@ "required": ["label"] } }, + { + "description": "keypoint_detection requires customer-supplied `target_size` and `number_of_keypoints` (no convention defaults — both are dataset-specific).", + "if": { + "properties": { "category": { "const": "keypoint_detection" } }, + "required": ["category"] + }, + "then": { "required": ["target_size", "number_of_keypoints"] } + }, { "description": "Most categories require `label`.", "if": { From 132eca14e522eea482a58a3405bbbb7e16ba2c03 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:38:48 +0200 Subject: [PATCH 11/17] test(cli): cover preflight/progress/error paths + harden smoke test (#16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Lifts unit coverage on cheap-to-test, previously-0% functions that don't need a cluster, and beefs up the CI smoke test to exercise the real binary's schema validation. - progress_test.go: NewProgress on a non-TTY returns the no-op sink; isTTY false cases. - coverage_test.go (cli): printPushPreflight renders the key facts; exitError.Error/Code; runClusterInfo exits 3 on a bad kubeconfig. - watcherr_test.go (submit): WatchError.Unwrap + IsWatchError classification (drives the exit-9 vs exit-8 mapping). - CI smoke test now runs `ingest validate` against committed fixtures (valid -> exit 0; missing-images -> exit 2) + `dataset push --help`, so a schema/validation/wiring regression is caught on the built binary without a cluster. Per-package coverage after: cli 61%, cluster 84%, push 84%, schema 81%, submit 66%. The remaining gaps are the real-I/O seams (SPDYExecutor.Exec, PortForwardJobsManager, NewClientset) — those need a real cluster and are covered by the integration harness (separate PR). Co-authored-by: Claude Opus 4.8 --- .github/workflows/build.yml | 16 ++- internal/cli/coverage_test.go | 106 ++++++++++++++++++ internal/push/progress_test.go | 44 ++++++++ internal/submit/watcherr_test.go | 33 ++++++ testdata/smoke/invalid-missing-images.yaml | 12 ++ .../smoke/valid-image-classification.yaml | 12 ++ 6 files changed, 221 insertions(+), 2 deletions(-) create mode 100644 internal/cli/coverage_test.go create mode 100644 internal/push/progress_test.go create mode 100644 internal/submit/watcherr_test.go create mode 100644 testdata/smoke/invalid-missing-images.yaml create mode 100644 testdata/smoke/valid-image-classification.yaml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2174836..0df784c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -156,8 +156,20 @@ jobs: # validation only — which is still meaningful because # client-go and most k8s deps have OS-conditional code paths. run: | - ./dist/tracebloc-linux-amd64 version - ./dist/tracebloc-linux-amd64 version --output-json | python3 -c "import json,sys; d=json.load(sys.stdin); assert d['version'], 'empty version'" + BIN=./dist/tracebloc-linux-amd64 + "$BIN" version + "$BIN" version --output-json | python3 -c "import json,sys; d=json.load(sys.stdin); assert d['version'], 'empty version'" + + # ingest validate exercises the embedded schema + validation + # wiring on the real binary (no cluster needed): a valid spec + # must pass, and an invalid one must be rejected with exit 2. + "$BIN" ingest validate testdata/smoke/valid-image-classification.yaml + if "$BIN" ingest validate testdata/smoke/invalid-missing-images.yaml; then + echo "::error::ingest validate accepted an invalid spec (missing 'images')"; exit 1 + fi + + # Command-tree wiring smoke for the dominant verb. + "$BIN" dataset push --help >/dev/null - name: Upload binary as artifact uses: actions/upload-artifact@v4 diff --git a/internal/cli/coverage_test.go b/internal/cli/coverage_test.go new file mode 100644 index 0000000..0cbe474 --- /dev/null +++ b/internal/cli/coverage_test.go @@ -0,0 +1,106 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/push" +) + +// TestPrintPushPreflight_RendersKeyFacts pins that the pre-flight +// summary surfaces the facts a customer sanity-checks before a push: +// the target release, the shared PVC, and the synthesized spec +// identity. It's the customer's last look before bytes move, so the +// content (not just "it didn't panic") is worth asserting. +func TestPrintPushPreflight_RendersKeyFacts(t *testing.T) { + layout := &push.LocalLayout{ + Root: "/tmp/cats_dogs", + LabelsCSV: "/tmp/cats_dogs/labels.csv", + Images: []string{"a.jpg", "b.jpg", "c.jpg"}, + TotalBytes: 1024, + } + release := &cluster.ParentRelease{ + ReleaseName: "ingdemo", + ChartVersion: "1.4.2", + JobsManagerService: "http://jobs-manager.ingdemo.svc.cluster.local:8080", + } + pvc := &cluster.SharedPVC{ + ClaimName: "client-pvc", + MountPath: "/data/shared", + Phase: corev1.ClaimBound, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + } + spec := map[string]any{ + "table": "cats_dogs_train", + "category": "image_classification", + "intent": "train", + "label": "label", + } + + var buf bytes.Buffer + printPushPreflight(&buf, layout, release, pvc, spec, false) + out := buf.String() + + for _, want := range []string{ + "ingdemo", "1.4.2", "client-pvc", + "cats_dogs_train", "image_classification", "train", + } { + if !strings.Contains(out, want) { + t.Errorf("pre-flight output missing %q:\n%s", want, out) + } + } +} + +// TestExitError_Methods pins the exit-code carrier: Error() surfaces +// the wrapped message (or a fallback when nil), and Code() returns the +// process exit code main() propagates. +func TestExitError_Methods(t *testing.T) { + e := &exitError{code: 7, err: errors.New("staging failed")} + if e.Error() != "staging failed" { + t.Errorf("Error() = %q, want %q", e.Error(), "staging failed") + } + if e.Code() != 7 { + t.Errorf("Code() = %d, want 7", e.Code()) + } + // err==nil: Error() falls back to a generic "exit N" string so the + // type still satisfies error without panicking. + nilErr := &exitError{code: 2} + if !strings.Contains(nilErr.Error(), "2") { + t.Errorf("Error() on nil-err exitError = %q, want it to mention the code", nilErr.Error()) + } + if nilErr.Code() != 2 { + t.Errorf("Code() = %d, want 2", nilErr.Code()) + } +} + +// TestRunClusterInfo_BadKubeconfigExitsThree: an unreadable/invalid +// kubeconfig is exit-code-3 territory (the kubeconfig/local-input +// bucket), surfaced before any cluster work. Covers the Load-error +// branch of runClusterInfo without needing a real cluster. +func TestRunClusterInfo_BadKubeconfigExitsThree(t *testing.T) { + bad := filepath.Join(t.TempDir(), "broken.yaml") + if err := os.WriteFile(bad, []byte("}{ this is not valid kubeconfig"), 0o644); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err := runClusterInfo(context.Background(), &buf, bad, "", "", "", 600) + if err == nil { + t.Fatal("runClusterInfo with a broken kubeconfig returned nil; want an exitError") + } + var ee *exitError + if !errors.As(err, &ee) { + t.Fatalf("error is not an *exitError: %v", err) + } + if ee.Code() != 3 { + t.Errorf("exit code = %d, want 3 (kubeconfig/local-input error)", ee.Code()) + } +} diff --git a/internal/push/progress_test.go b/internal/push/progress_test.go new file mode 100644 index 0000000..425e210 --- /dev/null +++ b/internal/push/progress_test.go @@ -0,0 +1,44 @@ +package push + +import ( + "bytes" + "os" + "testing" +) + +// TestNewProgress_NonTTYReturnsNoOp: a non-terminal writer (a buffer, +// a redirected file, a CI log) must get the no-op sink — an animated +// bar in non-interactive output is noise. Covers NewProgress's +// dominant branch + NoOpProgress's methods. +func TestNewProgress_NonTTYReturnsNoOp(t *testing.T) { + var buf bytes.Buffer + p := NewProgress(&buf, 1000, "Staging x") + if _, ok := p.(NoOpProgress); !ok { + t.Fatalf("NewProgress(non-TTY) = %T, want NoOpProgress", p) + } + // Must be safe no-ops that write nothing. + p.Add(500) + p.Finish() + if buf.Len() != 0 { + t.Errorf("NoOpProgress wrote %d bytes to a non-TTY; want 0", buf.Len()) + } +} + +// TestIsTTY_False: neither a bytes.Buffer nor a pipe (a non-terminal +// *os.File) is a TTY. Pins the conservative detection that keeps CI +// output clean. +func TestIsTTY_False(t *testing.T) { + var buf bytes.Buffer + if isTTY(&buf) { + t.Error("isTTY(*bytes.Buffer) = true, want false") + } + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + defer func() { _ = w.Close() }() + if isTTY(w) { + t.Error("isTTY(os.Pipe writer) = true, want false (not a terminal)") + } +} diff --git a/internal/submit/watcherr_test.go b/internal/submit/watcherr_test.go new file mode 100644 index 0000000..4ddad1c --- /dev/null +++ b/internal/submit/watcherr_test.go @@ -0,0 +1,33 @@ +package submit + +import ( + "errors" + "fmt" + "testing" +) + +// TestWatchError_UnwrapAndClassify pins WatchError's wrapping: it +// reports the inner message, unwraps to the cause (so errors.Is works +// through it), and IsWatchError recognizes it while rejecting other +// errors. This drives the orchestrator's exit-9 (ingest-side) vs +// exit-8 (submit-side) mapping, so the classification must be exact. +func TestWatchError_UnwrapAndClassify(t *testing.T) { + inner := errors.New("pod log stream broke") + we := &WatchError{Err: inner} + + if we.Error() != inner.Error() { + t.Errorf("Error() = %q, want %q", we.Error(), inner.Error()) + } + // errors.Is traverses Unwrap — covers WatchError.Unwrap. + if !errors.Is(we, inner) { + t.Error("errors.Is(WatchError, inner) = false; Unwrap not wired") + } + // And through an extra wrap layer. + wrapped := fmt.Errorf("watching ingestor Job: %w", we) + if !IsWatchError(wrapped) { + t.Error("IsWatchError(wrapped WatchError) = false, want true") + } + if IsWatchError(errors.New("unrelated")) { + t.Error("IsWatchError(plain error) = true, want false") + } +} diff --git a/testdata/smoke/invalid-missing-images.yaml b/testdata/smoke/invalid-missing-images.yaml new file mode 100644 index 0000000..75706c6 --- /dev/null +++ b/testdata/smoke/invalid-missing-images.yaml @@ -0,0 +1,12 @@ +# Smoke-test fixture: an image_classification spec missing the +# required `images` directory. `tracebloc ingest validate` must REJECT +# it with exit 2 (schema violation). Guards against a regression where +# the embedded schema or the validator silently stops enforcing +# per-category requirements. +apiVersion: tracebloc.io/v1 +kind: IngestConfig +category: image_classification +table: smoke_test +intent: train +csv: /data/shared/smoke_test/labels.csv +label: label diff --git a/testdata/smoke/valid-image-classification.yaml b/testdata/smoke/valid-image-classification.yaml new file mode 100644 index 0000000..ac803c7 --- /dev/null +++ b/testdata/smoke/valid-image-classification.yaml @@ -0,0 +1,12 @@ +# Smoke-test fixture: a minimal, valid image_classification ingest +# spec. `tracebloc ingest validate` must accept it (exit 0). Used by +# the CI smoke step to exercise the embedded schema + validation wiring +# on the real built binary — no cluster required. +apiVersion: tracebloc.io/v1 +kind: IngestConfig +category: image_classification +table: smoke_test +intent: train +csv: /data/shared/smoke_test/labels.csv +images: /data/shared/smoke_test/images/ +label: label From 18de3c119269f8cf7c2497752950184a58288d36 Mon Sep 17 00:00:00 2001 From: lukasWuttke <54042461+LukasWodka@users.noreply.github.com> Date: Tue, 2 Jun 2026 13:39:36 +0200 Subject: [PATCH 12/17] test(cli): integration harness for the real-I/O seams (kind e2e) (#17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The unit suite mocks every cluster boundary, leaving the highest-risk code — the SPDYExecutor tar-over-exec stream — at 0% coverage. Every live bug this project hit (staging<->DEST collision, validator drift, schema skew) was only findable by running it by hand. This adds an automated integration harness against a real cluster. - test/integration/ (//go:build integration): * Connectivity — cluster.Load + NewClientset against a live API server. * StageAndVerify — creates a throwaway namespace + PVC + stage pod, streams a dataset via the REAL push.SPDYExecutor tar-over-exec, then exec's back into the pod to confirm the files landed. Covers SPDYExecutor.Exec + StreamLayout + CreateStagePod / WaitForStagePodReady — the seams with zero unit coverage. - Makefile: `make test-integration` (ambient kubeconfig; -count=1, build-tagged). - .github/workflows/e2e.yml: boots kind + runs the suite. Gated to nightly / manual dispatch / `e2e`-labeled PRs (kind boot is too heavy to run per-PR). Validated locally against a live k3d cluster: both tests pass; the stream lands files on the PVC and the exec-back confirms content. The files are build-tag-gated, so normal `go test ./...` and the CI test job are unaffected. The kind CI job gets its first real run on the nightly schedule / manual dispatch / an `e2e`-labeled PR. Follow-ups (test/integration/README.md): a PortForwardJobsManager fixture; a full `dataset push` through a real jobs-manager + ingestor (needs the chart in-cluster) to catch cross-component schema skew e2e. Co-authored-by: Claude Opus 4.8 --- .github/workflows/e2e.yml | 50 ++++++++ Makefile | 10 ++ test/integration/README.md | 36 ++++++ test/integration/integration_test.go | 176 +++++++++++++++++++++++++++ 4 files changed, 272 insertions(+) create mode 100644 .github/workflows/e2e.yml create mode 100644 test/integration/README.md create mode 100644 test/integration/integration_test.go diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..660921c --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,50 @@ +name: Integration (e2e) + +# Spins a kind cluster and runs the build-tagged `integration` suite, +# which exercises the real-I/O seams the unit tests mock out: +# kubeconfig->clientset connectivity and the SPDYExecutor tar-over-exec +# stream against a live Pod + PVC (internal/push.SPDYExecutor.Exec is +# 0% in unit coverage — this is where every live bug actually lived). +# +# Heavy (kind boot + image pulls), so it does NOT run on every PR: +# - nightly (schedule) +# - manual (workflow_dispatch) +# - on a PR only when it carries the `e2e` label + +on: + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened, labeled] + +permissions: + contents: read + +jobs: + integration: + name: Integration (kind) + runs-on: ubuntu-latest + # Skip on PRs that aren't explicitly opted in via the `e2e` label; + # always run on schedule / manual dispatch. + if: >- + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'e2e') + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # Boots a single-node kind cluster and points $KUBECONFIG at it, + # which the integration suite picks up via cluster.Load's default + # kubeconfig resolution. kind ships a default StorageClass + # (local-path), so the test's PVC binds. + - name: Create kind cluster + uses: helm/kind-action@v1.10.0 + + - name: Integration tests + run: make test-integration diff --git a/Makefile b/Makefile index 0384697..0749fa4 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,16 @@ vet: test: $(GO) test -race -cover $(PKGS) +# Integration tests (build-tagged `integration`) run against a REAL +# cluster reachable via the ambient kubeconfig — kind in CI, or your +# own dev cluster locally. They cover the real-I/O seams the unit +# suite mocks (clientset connectivity, the SPDYExecutor tar-over-exec +# stream against a live Pod + PVC). -count=1 disables caching since +# these touch live cluster state. See .github/workflows/e2e.yml. +.PHONY: test-integration +test-integration: + $(GO) test -tags integration -count=1 -timeout 10m -v ./test/integration/... + .PHONY: lint lint: @command -v $(GOLANGCI_LINT) >/dev/null 2>&1 || { \ diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 0000000..b53135b --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,36 @@ +# CLI integration tests + +Build-tagged (`//go:build integration`) tests that run against a **real +Kubernetes cluster** — covering the real-I/O seams the unit suite mocks +out (and where every live bug this project hit actually lived). + +## What they cover +- `cluster.Load` + `cluster.NewClientset` — kubeconfig → live API server. +- `push.SPDYExecutor.Exec` + `push.StreamLayout` — the **tar-over-exec + stream** against a live Pod + PVC (the highest-risk, 0%-unit-covered + path), verified by exec-ing back into the pod to confirm the bytes + landed. + +## Running +```sh +make test-integration # uses your current kubeconfig context +# or: +go test -tags integration -count=1 -v ./test/integration/... +``` +Requires a reachable cluster (kind, k3d, or any dev cluster) with a +default StorageClass and the ability to pull the digest-pinned alpine +stage-pod image. Each test creates a throwaway namespace and cleans up +after itself. + +In CI these run via `.github/workflows/e2e.yml` (kind), gated to +nightly / manual dispatch / `e2e`-labeled PRs (kind boot is too heavy +for every PR). + +## Follow-ups (not yet covered) +- `PortForwardJobsManager` against a live Service (needs a small HTTP + pod + Service fixture). +- A full `dataset push` through a real jobs-manager + ingestor (needs + the `tracebloc/client` chart installed in-cluster) — the + highest-fidelity test; gated on a reproducible in-CI chart install. + This would catch cross-component issues like the jobs-manager↔ingestor + schema skew end-to-end. diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go new file mode 100644 index 0000000..daa34bb --- /dev/null +++ b/test/integration/integration_test.go @@ -0,0 +1,176 @@ +//go:build integration + +// Package integration holds CLI integration tests that run against a +// REAL Kubernetes cluster (kind in CI; any KUBECONFIG-reachable +// cluster locally). They cover the real-I/O seams the mock-based unit +// suite can't reach: kubeconfig→clientset connectivity, and — the big +// one — the SPDYExecutor tar-over-exec stream against a live Pod + PVC +// (internal/push.SPDYExecutor.Exec, 0% in unit coverage). +// +// Run: +// +// make test-integration +// # or: go test -tags integration ./test/integration/ -v +// +// Requires a reachable cluster with a default StorageClass and the +// ability to pull the digest-pinned alpine stage-pod image. Each test +// creates its own throwaway namespace and cleans up after itself. +package integration + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/push" +) + +// loadCluster builds a clientset + rest.Config from the ambient +// kubeconfig — exercising the real cluster.Load + cluster.NewClientset +// path (NewClientset is 0% in unit coverage). +func loadCluster(t *testing.T) (kubernetes.Interface, *rest.Config) { + t.Helper() + resolved, err := cluster.Load(cluster.KubeconfigOptions{}) + if err != nil { + t.Fatalf("cluster.Load (need a reachable kubeconfig): %v", err) + } + cs, err := cluster.NewClientset(resolved) + if err != nil { + t.Fatalf("cluster.NewClientset: %v", err) + } + return cs, resolved.RestConfig +} + +// TestIntegration_Connectivity proves the kubeconfig→clientset path +// reaches a live API server. +func TestIntegration_Connectivity(t *testing.T) { + cs, _ := loadCluster(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + nss, err := cs.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("list namespaces: %v", err) + } + if len(nss.Items) == 0 { + t.Fatal("cluster reports zero namespaces — unexpected") + } + t.Logf("connected: %d namespaces", len(nss.Items)) +} + +// TestIntegration_StageAndVerify is the core integration test: it +// stages a tiny dataset onto a real PVC via the real SPDYExecutor +// tar-over-exec stream, then exec's back into the pod to prove the +// files actually landed. This is the seam (push.SPDYExecutor.Exec + +// StreamLayout) that has zero unit coverage and where every live bug +// this project hit actually lived. +func TestIntegration_StageAndVerify(t *testing.T) { + cs, restConfig := loadCluster(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Throwaway namespace, auto-cleaned (cascades the PVC + pod). + ns, err := cs.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "tb-it-"}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create namespace: %v", err) + } + nsName := ns.Name + t.Cleanup(func() { + cctx, ccancel := context.WithTimeout(context.Background(), 60*time.Second) + defer ccancel() + _ = cs.CoreV1().Namespaces().Delete(cctx, nsName, metav1.DeleteOptions{}) + }) + + // Shared PVC (RWO + default StorageClass), mirroring the chart's + // client-pvc that the stage pod mounts at /data/shared. + const pvcName = "client-pvc" + const mountPath = "/data/shared" + _, err = cs.CoreV1().PersistentVolumeClaims(nsName).Create(ctx, &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: pvcName}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create PVC: %v", err) + } + + // A minimal local dataset to stage. + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "labels.csv"), + []byte("filename,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "images"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "images", "001.jpg"), + []byte("\xff\xd8\xff\xe0-integration-marker"), 0o644); err != nil { + t.Fatal(err) + } + layout, err := push.Discover(dir) + if err != nil { + t.Fatalf("push.Discover: %v", err) + } + + const table = "ittest" + exec := &push.SPDYExecutor{Config: restConfig, Client: cs} + + // Stage pod → Ready → tar-over-exec stream. + podName, err := push.CreateStagePod(ctx, cs, push.PodSpecOptions{ + Namespace: nsName, + PVCClaimName: pvcName, + PVCMountPath: mountPath, + Table: table, + ServiceAccountName: "default", + }) + if err != nil { + t.Fatalf("CreateStagePod: %v", err) + } + defer func() { + dctx, dcancel := context.WithTimeout(context.Background(), 60*time.Second) + defer dcancel() + _ = push.DeleteStagePod(dctx, cs, nsName, podName) + }() + + if _, err := push.WaitForStagePodReady(ctx, cs, nsName, podName); err != nil { + t.Fatalf("WaitForStagePodReady: %v", err) + } + + if err := push.StreamLayout(ctx, exec, nsName, podName, "stage", layout, table, push.NoOpProgress{}); err != nil { + t.Fatalf("StreamLayout (real SPDYExecutor): %v", err) + } + + // Exec back into the still-running pod to prove the bytes landed. + dest := push.StagedPrefix(table) + var stdout, stderr bytes.Buffer + if err := exec.Exec(ctx, nsName, podName, "stage", + []string{"/bin/sh", "-c", fmt.Sprintf("cat %q/labels.csv; echo; ls %q/images", dest, dest)}, + nil, &stdout, &stderr); err != nil { + t.Fatalf("verify exec: %v (stderr: %s)", err, stderr.String()) + } + got := stdout.String() + for _, want := range []string{"filename,label", "001.jpg"} { + if !strings.Contains(got, want) { + t.Errorf("staged content missing %q at %s; got:\n%s", want, dest, got) + } + } + t.Logf("staged + verified at %s:\n%s", dest, got) +} From a661a9a07c5946ab90e508a69aa9eeead444a820 Mon Sep 17 00:00:00 2001 From: Arturo Peroni Date: Wed, 3 Jun 2026 08:32:27 +0200 Subject: [PATCH 13/17] docs: refresh README + stale --help strings for shipped v0.1 (#18) README was stuck at Phase 0; updates Status (9/10 modalities), Customer-experience/install caveats, Go 1.26 floor, and the roadmap. Also fixes three user-facing help/output strings (cluster info 'coming in Phase 3'; dataset and dataset push --help referencing Phase 4 / PR-b / image-only) that contradicted the shipped binary. Docs only. Co-authored-by: Claude Opus 4.8 (1M context) --- README.md | 38 +++++++++++++++++++++++++------------- internal/cli/cluster.go | 2 +- internal/cli/dataset.go | 16 +++++++++------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index cd2746c..0c52dcd 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,13 @@ The customer-facing CLI for the tracebloc declarative ingestion path. Wraps the ## Status -🚧 **Pre-alpha** — Phase 0 of the [v0.1 roadmap](https://github.com/tracebloc/client/issues/147). Today this binary implements only `version` and `completion`. Subsequent phases (#149–#153) land schema validation, cluster discovery, data staging, submission, and distribution. +**v0.1 — feature-complete on `develop`; first release pending.** Every phase of the [v0.1 roadmap](https://github.com/tracebloc/client/issues/147) (#148–#153) is merged. The binary implements `version`, `completion`, `ingest validate`, `cluster info`, and the full `dataset push` flow — local schema validation, cluster discovery, data staging, submission, and Job watching, end to end. -For the production ingestion flow today, use the Helm chart: `helm install tracebloc/ingestor --set-file ingestConfig=./ingest.yaml`. See the chart's [README](https://github.com/tracebloc/client/blob/develop/ingestor/README.md) for the full customer journey including the data-staging recipe. +`dataset push` covers **9 of 10 task categories**: `image_classification`, `object_detection`, `keypoint_detection`, `text_classification`, `masked_language_modeling`, `tabular_classification`, `tabular_regression`, `time_series_forecasting`, and `time_to_event_prediction`. `semantic_segmentation` is pending mask-sidecar support upstream ([data-ingestors#136](https://github.com/tracebloc/data-ingestors/issues/136)); `instance_segmentation` is not yet implemented. + +No tagged release exists yet — [build from source](#building-from-source) for now. The release pipeline (GitHub Release + Homebrew tap + install scripts) is wired and fires on the first `v*` tag. + +The Helm chart remains a sibling interface for the Kubernetes-native workflow: `helm install tracebloc/ingestor --set-file ingestConfig=./ingest.yaml` (see the chart's [README](https://github.com/tracebloc/client/blob/develop/ingestor/README.md)). ## Why a CLI in addition to the chart? @@ -37,10 +41,12 @@ Customer interfaces (pick one or many): The protocol — the v1 schema + the POST endpoint — is the stable point. Everything above is interchangeable. -## v0.1 design (target customer experience, once Phase 5 ships) +## Customer experience + +> The install commands below light up once `v0.1.0` is tagged and the release pipeline publishes the artifacts. Until then, [build from source](#building-from-source). ```bash -# Install once +# Install once (after v0.1.0 is released) brew install tracebloc/tap/tracebloc # macOS curl -fsSL https://install.tracebloc.io | sh # Linux/macOS irm https://install.tracebloc.io/install.ps1 | iex # Windows @@ -73,7 +79,7 @@ go build -o tracebloc ./cmd/tracebloc ./tracebloc version ``` -Requires Go 1.22 or newer. The binary self-reports its build metadata; a `go build` without `-ldflags` reports `dev / unknown / unknown` for version/git-sha/build-date so support can tell a local hack apart from a release build. +Requires Go 1.26 or newer (the `k8s.io/*` dependencies set the floor; see `go.mod`). The binary self-reports its build metadata; a `go build` without `-ldflags` reports `dev / unknown / unknown` for version/git-sha/build-date so support can tell a local hack apart from a release build. ```bash # Release-style build with version metadata @@ -86,14 +92,20 @@ go build -ldflags "\ ## Roadmap -| Phase | Ticket | What | -|---|---|---| -| **0 (this PR)** | [#148](https://github.com/tracebloc/client/issues/148) | Repo bootstrap + Go module + CI + `tracebloc version` | -| 1 | [#149](https://github.com/tracebloc/client/issues/149) | Embed `ingest.v1.json` + `tracebloc ingest validate ` (local-only) | -| 2 | [#150](https://github.com/tracebloc/client/issues/150) | Cluster discovery + ingestor SA token via TokenRequest | -| 3 | [#151](https://github.com/tracebloc/client/issues/151) | Stage data into the shared PVC via ephemeral Pod | -| 4 | [#152](https://github.com/tracebloc/client/issues/152) | Submit to jobs-manager + watch ingestor Job + summary | -| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases, Homebrew tap, install.sh distribution | +All v0.1 phases are merged: + +| Phase | Ticket | What | Status | +|---|---|---|---| +| 0 | [#148](https://github.com/tracebloc/client/issues/148) | Repo bootstrap + Go module + CI + `tracebloc version` | ✅ | +| 1 | [#149](https://github.com/tracebloc/client/issues/149) | Embed `ingest.v1.json` + `tracebloc ingest validate ` (local-only) | ✅ | +| 2 | [#150](https://github.com/tracebloc/client/issues/150) | Cluster discovery + ingestor SA token via TokenRequest | ✅ | +| 3 | [#151](https://github.com/tracebloc/client/issues/151) | Stage data into the shared PVC via ephemeral Pod | ✅ | +| 4 | [#152](https://github.com/tracebloc/client/issues/152) | Submit to jobs-manager + watch ingestor Job + summary | ✅ | +| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases, Homebrew tap, install.sh distribution | ✅ (awaiting first `v*` tag) | + +Beyond the original phases, `dataset push` was widened from image-classification-only to 9 of 10 modalities, and the test suite gained unit-coverage wins plus a kind-based integration harness for the real-I/O seams. + +**Next (v0.2):** cloud-source ingestion (S3/GCS/HTTPS) for datasets above the 1 GiB local cap; `semantic_segmentation` ([data-ingestors#136](https://github.com/tracebloc/data-ingestors/issues/136)) and `instance_segmentation`; more `dataset` verbs (`list`, `rm`). Smaller follow-ups are tracked as [open issues](https://github.com/tracebloc/cli/issues) (#3–#7). Epic: [tracebloc/client#147](https://github.com/tracebloc/client/issues/147). diff --git a/internal/cli/cluster.go b/internal/cli/cluster.go index 80a7172..fd9da91 100644 --- a/internal/cli/cluster.go +++ b/internal/cli/cluster.go @@ -197,6 +197,6 @@ func runClusterInfo( _, _ = fmt.Fprintf(out, " expires in: never (static-secret fallback)\n") } _, _ = fmt.Fprintln(out) - _, _ = fmt.Fprintln(out, "Ready for `tracebloc dataset push` (coming in Phase 3).") + _, _ = fmt.Fprintln(out, "Ready for `tracebloc dataset push`.") return nil } diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go index 1551767..095dc1a 100644 --- a/internal/cli/dataset.go +++ b/internal/cli/dataset.go @@ -30,10 +30,9 @@ func newDatasetCmd() *cobra.Command { Long: `Commands for staging and managing datasets on the cluster's shared PVC. -Today: ` + "`dataset push`" + ` stages a local directory to the cluster's -shared PVC. Submission to jobs-manager (so the ingestor Job actually -runs) lands in Phase 4 (` + "`tracebloc/client#152`" + `); for now the staged -files are picked up by the existing helm ` + "`tracebloc/ingestor`" + ` flow. +` + "`dataset push`" + ` stages a local dataset to the cluster's shared +PVC, submits the ingestion run to jobs-manager, and watches the +ingestor Job to completion (streaming logs + the final summary). ` + "`tracebloc cluster info`" + ` is the pre-flight you'd typically run before the first push.`, @@ -112,10 +111,13 @@ func newDatasetPushCmd() *cobra.Command { cmd := &cobra.Command{ Use: "push ", Short: "Stage a local dataset to the cluster's shared PVC", - Long: `Stages a local image_classification dataset to the parent client -release's shared PVC, then (in PR-b) submits an ingestion run. + Long: `Stages a local dataset to the parent client release's shared PVC, +submits an ingestion run to jobs-manager, and watches the ingestor Job +to completion. Supports 9 task categories (image classification, +object/keypoint detection, text classification, masked language +modeling, and the tabular / time-series family); pick one with --category. -Expected local layout: +Expected local layout (image_classification shown): / labels.csv (required) From fd715e5bcfc91cf221a8838fe4505aa1643fe9a6 Mon Sep 17 00:00:00 2001 From: Arturo Peroni Date: Wed, 3 Jun 2026 10:58:46 +0200 Subject: [PATCH 14/17] docs: drop Homebrew from README until the tap ships (#20) The tracebloc/homebrew-tap repo isn't set up yet and Homebrew is deferred past the v0.1 alpha. Removes the brew install line + softens the Status/roadmap mentions (framed as a deferred follow-up). GitHub Release + install.sh/ps1 paths remain. Docs only; bump-homebrew-tap was already if:false. Co-authored-by: Claude Opus 4.8 (1M context) --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0c52dcd..8e85ebe 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The customer-facing CLI for the tracebloc declarative ingestion path. Wraps the `dataset push` covers **9 of 10 task categories**: `image_classification`, `object_detection`, `keypoint_detection`, `text_classification`, `masked_language_modeling`, `tabular_classification`, `tabular_regression`, `time_series_forecasting`, and `time_to_event_prediction`. `semantic_segmentation` is pending mask-sidecar support upstream ([data-ingestors#136](https://github.com/tracebloc/data-ingestors/issues/136)); `instance_segmentation` is not yet implemented. -No tagged release exists yet — [build from source](#building-from-source) for now. The release pipeline (GitHub Release + Homebrew tap + install scripts) is wired and fires on the first `v*` tag. +No tagged release exists yet — [build from source](#building-from-source) for now. The release pipeline (GitHub Release + install scripts) is wired and fires on the first `v*` tag. (A Homebrew tap is a later follow-up.) The Helm chart remains a sibling interface for the Kubernetes-native workflow: `helm install tracebloc/ingestor --set-file ingestConfig=./ingest.yaml` (see the chart's [README](https://github.com/tracebloc/client/blob/develop/ingestor/README.md)). @@ -47,8 +47,7 @@ The protocol — the v1 schema + the POST endpoint — is the stable point. Ever ```bash # Install once (after v0.1.0 is released) -brew install tracebloc/tap/tracebloc # macOS -curl -fsSL https://install.tracebloc.io | sh # Linux/macOS +curl -fsSL https://install.tracebloc.io | sh # Linux/macOS irm https://install.tracebloc.io/install.ps1 | iex # Windows # Per dataset @@ -101,7 +100,7 @@ All v0.1 phases are merged: | 2 | [#150](https://github.com/tracebloc/client/issues/150) | Cluster discovery + ingestor SA token via TokenRequest | ✅ | | 3 | [#151](https://github.com/tracebloc/client/issues/151) | Stage data into the shared PVC via ephemeral Pod | ✅ | | 4 | [#152](https://github.com/tracebloc/client/issues/152) | Submit to jobs-manager + watch ingestor Job + summary | ✅ | -| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases, Homebrew tap, install.sh distribution | ✅ (awaiting first `v*` tag) | +| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases + install.sh distribution (Homebrew tap deferred) | ✅ (awaiting first `v*` tag) | Beyond the original phases, `dataset push` was widened from image-classification-only to 9 of 10 modalities, and the test suite gained unit-coverage wins plus a kind-based integration harness for the real-I/O seams. From d0d8af35a3c42b68a6d293752f519783ab4dbdf2 Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Wed, 3 Jun 2026 14:07:09 +0500 Subject: [PATCH 15/17] docs(cli): refresh stale root --help long string (#21) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The root command's Long help still claimed the binary "implements only version and completion" and pointed at the roadmap for future phases — stale since v0.1 shipped the dataset/ingest/cluster commands. PR #18 refreshed the README and the three other user-facing --help strings but missed this one (it scoped to cluster info / dataset / dataset push). Describe the shipped command set instead. Docs/strings only; no behavior change. Closes #19 Co-authored-by: Claude Opus 4.8 --- internal/cli/root.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/internal/cli/root.go b/internal/cli/root.go index 1dc4184..d7d9b47 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -50,9 +50,12 @@ on the cluster's shared PVC, submitting the ingestion request, watching the resulting Job, and reporting the outcome. Customers never touch Helm, never edit YAML, never run kubectl cp manually. -Today this binary implements only ` + "`version`" + ` and ` + "`completion`" + ` — -see https://github.com/tracebloc/client/issues/147 for the v0.1 -roadmap. Subsequent phases land subcommands incrementally.`, +This binary implements the full v0.1 ingestion path: ` + "`dataset push`" + ` +(the dominant workflow above), ` + "`ingest validate`" + ` for a local +schema check, ` + "`cluster info`" + ` for discovery diagnostics, plus +` + "`version`" + ` and ` + "`completion`" + `. See +https://github.com/tracebloc/client/issues/147 for the v0.1 roadmap and +what's planned next.`, // Silence cobra's auto-printed errors + usage on every error; // we already print structured errors in handlers, and the From 93907256313eee00b9b453a1c68d7c52769353ca Mon Sep 17 00:00:00 2001 From: "Asad Iqbal (Saadi)" Date: Wed, 3 Jun 2026 14:29:38 +0500 Subject: [PATCH 16/17] docs: refresh README for the v0.1.0-alpha release (#24) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit v0.1.0-alpha is published (pre-release). The README still said "first release pending" / "no tagged release exists yet" and advertised `curl https://install.tracebloc.io | sh` — but that vanity domain isn't wired up, and GitHub's `latest` redirect skips pre-releases, so neither the domain shortcut nor a `latest`-based one-liner can install the alpha. - Status: "first release pending" -> v0.1.0-alpha published (pre-release). - Install: replace the dead install.tracebloc.io commands with the working explicit-tag invocation + a link to the signed release assets; note the domain / `latest` / Homebrew all await the first stable tag. - Roadmap: Phase 5 "awaiting first v* tag" -> alpha published. Docs only; no behavior change. Closes #23 Co-authored-by: Claude Opus 4.8 --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8e85ebe..da3f27b 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ The customer-facing CLI for the tracebloc declarative ingestion path. Wraps the ## Status -**v0.1 — feature-complete on `develop`; first release pending.** Every phase of the [v0.1 roadmap](https://github.com/tracebloc/client/issues/147) (#148–#153) is merged. The binary implements `version`, `completion`, `ingest validate`, `cluster info`, and the full `dataset push` flow — local schema validation, cluster discovery, data staging, submission, and Job watching, end to end. +**v0.1.0-alpha is published** — the first [release](https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha), a pre-release for early testers, cut from `develop`. Every phase of the [v0.1 roadmap](https://github.com/tracebloc/client/issues/147) (#148–#153) is merged. The binary implements `version`, `completion`, `ingest validate`, `cluster info`, and the full `dataset push` flow — local schema validation, cluster discovery, data staging, submission, and Job watching, end to end. `dataset push` covers **9 of 10 task categories**: `image_classification`, `object_detection`, `keypoint_detection`, `text_classification`, `masked_language_modeling`, `tabular_classification`, `tabular_regression`, `time_series_forecasting`, and `time_to_event_prediction`. `semantic_segmentation` is pending mask-sidecar support upstream ([data-ingestors#136](https://github.com/tracebloc/data-ingestors/issues/136)); `instance_segmentation` is not yet implemented. -No tagged release exists yet — [build from source](#building-from-source) for now. The release pipeline (GitHub Release + install scripts) is wired and fires on the first `v*` tag. (A Homebrew tap is a later follow-up.) +The release pipeline has shipped [`v0.1.0-alpha`](https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha): cosign-signed binaries for linux/darwin/windows (amd64/arm64), `SHA256SUMS`, and the install scripts. Install it via [Customer experience](#customer-experience) or [build from source](#building-from-source). (The `install.tracebloc.io` shortcut, the `latest` one-liner, and a Homebrew tap all wait for the first **stable** `v*` tag — see below.) The Helm chart remains a sibling interface for the Kubernetes-native workflow: `helm install tracebloc/ingestor --set-file ingestConfig=./ingest.yaml` (see the chart's [README](https://github.com/tracebloc/client/blob/develop/ingestor/README.md)). @@ -43,12 +43,13 @@ The protocol — the v1 schema + the POST endpoint — is the stable point. Ever ## Customer experience -> The install commands below light up once `v0.1.0` is tagged and the release pipeline publishes the artifacts. Until then, [build from source](#building-from-source). +> `v0.1.0-alpha` is a **pre-release**, so the `install.tracebloc.io` shortcut isn't wired up yet and GitHub's `latest` skips pre-releases. Install the alpha by explicit tag (below), or [build from source](#building-from-source). ```bash -# Install once (after v0.1.0 is released) -curl -fsSL https://install.tracebloc.io | sh # Linux/macOS -irm https://install.tracebloc.io/install.ps1 | iex # Windows +# Install the alpha — pins the pre-release tag (Linux/macOS) +curl -fsSL https://github.com/tracebloc/cli/releases/download/v0.1.0-alpha/install.sh \ + | sh -s -- --version v0.1.0-alpha +# Windows + manual/signed downloads: https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha # Per dataset tracebloc dataset push ./my-data \ @@ -100,7 +101,7 @@ All v0.1 phases are merged: | 2 | [#150](https://github.com/tracebloc/client/issues/150) | Cluster discovery + ingestor SA token via TokenRequest | ✅ | | 3 | [#151](https://github.com/tracebloc/client/issues/151) | Stage data into the shared PVC via ephemeral Pod | ✅ | | 4 | [#152](https://github.com/tracebloc/client/issues/152) | Submit to jobs-manager + watch ingestor Job + summary | ✅ | -| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases + install.sh distribution (Homebrew tap deferred) | ✅ (awaiting first `v*` tag) | +| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases + install.sh distribution (Homebrew tap deferred) | ✅ — [`v0.1.0-alpha`](https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha) published (pre-release) | Beyond the original phases, `dataset push` was widened from image-classification-only to 9 of 10 modalities, and the test suite gained unit-coverage wins plus a kind-based integration harness for the real-I/O seams. From 534d75b4cf81a9545b4ac48fd9954d5ab09f0b59 Mon Sep 17 00:00:00 2001 From: Arturo Peroni Date: Wed, 3 Jun 2026 11:31:59 +0200 Subject: [PATCH 17/17] fix: address Bugbot findings from PR #22 review (#25) 1. target_size: emit [height, width] (was [width, height]) to match the ingest.v1 schema + ingestor; only affected non-square datasets. Added TestBuild_TargetSize_EmittedHeightWidth (non-square) regression test. 2. Makefile: make ci lint now runs the same standalone tools as CI (errcheck + ineffassign + misspell) instead of golangci-lint, restoring the local==CI invariant while golangci-lint-action stays disabled (#6); golangci-lint moved to make lint-full. Co-authored-by: Claude Opus 4.8 (1M context) --- Makefile | 13 +++++++++++++ internal/push/spec.go | 14 +++++++++++--- internal/push/spec_test.go | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 0749fa4..bcf7328 100644 --- a/Makefile +++ b/Makefile @@ -46,8 +46,21 @@ test: test-integration: $(GO) test -tags integration -count=1 -timeout 10m -v ./test/integration/... +# Lint set matched to .github/workflows/build.yml's lint job: errcheck + +# ineffassign + misspell (gofmt -s is `fmt-check`, go vet is `vet`). +# golangci-lint-action is disabled in CI pending tracebloc/cli#6 — so +# until it's re-enabled there, `make ci` runs the SAME standalone tools +# the CI lint job runs, keeping the "make ci green => CI green" invariant +# this Makefile exists to protect. `make lint-full` keeps golangci-lint +# available for a richer local pass. .PHONY: lint lint: + $(GO) run github.com/kisielk/errcheck@latest ./... + $(GO) run github.com/gordonklaus/ineffassign@latest ./... + $(GO) run github.com/client9/misspell/cmd/misspell@latest -error . + +.PHONY: lint-full +lint-full: @command -v $(GOLANGCI_LINT) >/dev/null 2>&1 || { \ echo "==> $(GOLANGCI_LINT) not on PATH"; \ echo " install via: brew install golangci-lint"; \ diff --git a/internal/push/spec.go b/internal/push/spec.go index 2133976..a28d7e8 100644 --- a/internal/push/spec.go +++ b/internal/push/spec.go @@ -134,7 +134,8 @@ type SpecArgs struct { // image_classification cares about. LabelColumn string - // TargetSize, when len==2, pins the image resolution as [W, H]. + // TargetSize, when len==2, holds the resolution as [W, H] — the + // order produced by --target-size "WxH" and by DetectImageSize. // The ingestor's image_classification default is 512x512 and it // VALIDATES (it does not resize), so a dataset whose images don't // match the default hard-fails. Setting this emits @@ -142,6 +143,10 @@ type SpecArgs struct { // resolution wins. Empty (len 0) ⇒ omit and let the ingestor // default apply. Populated by the CLI from --target-size or by // auto-detecting the first image. (Image categories only.) + // + // NOTE: stored [W, H] here, but EMITTED as [height, width] by + // Build — that's the order the ingest.v1 schema documents and the + // ingestor reads. buildImage does the swap. (Bugbot, PR #22.) TargetSize []int // Schema is the column→SQL-type map for tabular / time-series @@ -253,7 +258,9 @@ func (a SpecArgs) buildImage(spec map[string]any, prefix string) { if a.Category == "keypoint_detection" { if len(a.TargetSize) == 2 { - spec["target_size"] = []int{a.TargetSize[0], a.TargetSize[1]} + // Schema documents target_size as [height, width]; TargetSize + // is stored [W, H], so swap on emit. (Bugbot, PR #22.) + spec["target_size"] = []int{a.TargetSize[1], a.TargetSize[0]} } if a.NumberOfKeypoints > 0 { spec["number_of_keypoints"] = a.NumberOfKeypoints @@ -264,7 +271,8 @@ func (a SpecArgs) buildImage(spec map[string]any, prefix string) { if len(a.TargetSize) == 2 { spec["spec"] = map[string]any{ "file_options": map[string]any{ - "target_size": []int{a.TargetSize[0], a.TargetSize[1]}, + // [height, width] per the schema; TargetSize is [W, H]. + "target_size": []int{a.TargetSize[1], a.TargetSize[0]}, }, } } diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go index 2167008..09c6a4c 100644 --- a/internal/push/spec_test.go +++ b/internal/push/spec_test.go @@ -197,6 +197,39 @@ func TestBuild_NoTargetSize_OmitsSpecBlock(t *testing.T) { } } +// TestBuild_TargetSize_EmittedHeightWidth locks the [height, width] +// emit order (Bugbot, PR #22). TargetSize is stored [W, H] but the +// ingest.v1 schema + ingestor read target_size as [height, width]. +// Square sizes can't catch a swap, so this uses a NON-square +// resolution: [W, H] = [640, 480] must emit [480, 640]. +func TestBuild_TargetSize_EmittedHeightWidth(t *testing.T) { + // image_classification → under spec.file_options + icSpec := SpecArgs{ + Table: "t", Category: "image_classification", Intent: "train", + LabelColumn: "label", TargetSize: []int{640, 480}, // [W, H] + }.Build() + sb, ok := icSpec["spec"].(map[string]any) + if !ok { + t.Fatalf("image: no spec block: %#v", icSpec["spec"]) + } + fo, ok := sb["file_options"].(map[string]any) + if !ok { + t.Fatalf("image: no file_options: %#v", sb["file_options"]) + } + if ts, ok := fo["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 480 || ts[1] != 640 { + t.Errorf("image target_size = %#v, want [480 640] (height, width)", fo["target_size"]) + } + + // keypoint_detection → top-level + kpSpec := SpecArgs{ + Table: "t", Category: "keypoint_detection", Intent: "train", + LabelColumn: "image_label", TargetSize: []int{640, 480}, NumberOfKeypoints: 9, + }.Build() + if ts, ok := kpSpec["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 480 || ts[1] != 640 { + t.Errorf("keypoint top-level target_size = %#v, want [480 640] (height, width)", kpSpec["target_size"]) + } +} + // TestBuild_Tabular_PassesSchema pins the tabular Build branch: it // emits schema-valid specs for the three label shapes — a string // label (tabular_classification), an object label+policy