diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c31819e..0df784c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,20 @@ permissions: contents: read jobs: + schema-drift: + name: Schema drift check + # Verifies the embedded internal/schema/ingest.v1.json matches + # tracebloc/data-ingestors' master. A green PR that silently + # diverges from upstream is a real correctness hazard — a + # customer's YAML could pass `tracebloc ingest validate` locally + # but be rejected by jobs-manager (or vice versa). Forcing the + # sync as a PR step keeps drift visible. + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: scripts/sync-schema.sh --check + run: ./scripts/sync-schema.sh --check + test: name: Test runs-on: ubuntu-latest @@ -49,14 +63,44 @@ jobs: go-version-file: go.mod cache: true - - name: golangci-lint - uses: golangci/golangci-lint-action@v6 - with: - version: v1.61.0 - # Pin the cache key per Go version so a Go bump invalidates - # cleanly — stale caches across Go versions cause noisy - # "invalid pkg" failures. - args: --timeout=5m + # golangci-lint-action@v6 reproducibly time-budgeted out at + # ~2 minutes on this module's k8s.io-heavy dep tree (3 runs + # in a row in early 2026 — see #6). Stand-alone tools run + # against the same code in <30 seconds and catch the bugs + # we've actually hit this week (unchecked Fprintf errors + # were errcheck-catchable; gofmt -s drift was a recurring + # nit). Going back to the action when #6 has a strategy. + + # Tool versions are unpinned (@latest) because older tagged + # releases pulled stale golang.org/x/tools that fail to build + # against current Go ("invalid array length -delta * delta"). + # Pinning is preferable for reproducibility but waiting on the + # upstream tools to ship newer tags. Track in #6. + + - name: errcheck + run: | + go install github.com/kisielk/errcheck@latest + errcheck ./... + + - name: gofmt -s + run: | + drift="$(gofmt -s -l .)" + if [ -n "$drift" ]; then + echo "::error::gofmt -s drift in:" + echo "$drift" | sed 's/^/ /' + echo "::error::run \`make fmt\` to fix" + exit 1 + fi + + - name: ineffassign + run: | + go install github.com/gordonklaus/ineffassign@latest + ineffassign ./... + + - name: misspell + run: | + go install github.com/client9/misspell/cmd/misspell@latest + misspell -error . build: name: Build (${{ matrix.os }}/${{ matrix.arch }}) @@ -112,8 +156,20 @@ jobs: # validation only — which is still meaningful because # client-go and most k8s deps have OS-conditional code paths. run: | - ./dist/tracebloc-linux-amd64 version - ./dist/tracebloc-linux-amd64 version --output-json | python3 -c "import json,sys; d=json.load(sys.stdin); assert d['version'], 'empty version'" + BIN=./dist/tracebloc-linux-amd64 + "$BIN" version + "$BIN" version --output-json | python3 -c "import json,sys; d=json.load(sys.stdin); assert d['version'], 'empty version'" + + # ingest validate exercises the embedded schema + validation + # wiring on the real binary (no cluster needed): a valid spec + # must pass, and an invalid one must be rejected with exit 2. + "$BIN" ingest validate testdata/smoke/valid-image-classification.yaml + if "$BIN" ingest validate testdata/smoke/invalid-missing-images.yaml; then + echo "::error::ingest validate accepted an invalid spec (missing 'images')"; exit 1 + fi + + # Command-tree wiring smoke for the dominant verb. + "$BIN" dataset push --help >/dev/null - name: Upload binary as artifact uses: actions/upload-artifact@v4 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..660921c --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,50 @@ +name: Integration (e2e) + +# Spins a kind cluster and runs the build-tagged `integration` suite, +# which exercises the real-I/O seams the unit tests mock out: +# kubeconfig->clientset connectivity and the SPDYExecutor tar-over-exec +# stream against a live Pod + PVC (internal/push.SPDYExecutor.Exec is +# 0% in unit coverage — this is where every live bug actually lived). +# +# Heavy (kind boot + image pulls), so it does NOT run on every PR: +# - nightly (schedule) +# - manual (workflow_dispatch) +# - on a PR only when it carries the `e2e` label + +on: + schedule: + - cron: "0 3 * * *" + workflow_dispatch: + pull_request: + types: [opened, synchronize, reopened, labeled] + +permissions: + contents: read + +jobs: + integration: + name: Integration (kind) + runs-on: ubuntu-latest + # Skip on PRs that aren't explicitly opted in via the `e2e` label; + # always run on schedule / manual dispatch. + if: >- + github.event_name != 'pull_request' || + contains(github.event.pull_request.labels.*.name, 'e2e') + steps: + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + # Boots a single-node kind cluster and points $KUBECONFIG at it, + # which the integration suite picks up via cluster.Load's default + # kubeconfig resolution. kind ships a default StorageClass + # (local-path), so the test's PVC binds. + - name: Create kind cluster + uses: helm/kind-action@v1.10.0 + + - name: Integration tests + run: make test-integration diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..bee6109 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,272 @@ +# Build, sign, and publish the official tracebloc CLI binaries. +# +# Triggered by pushing a vX.Y.Z tag (or manually via +# workflow_dispatch). Produces: +# +# - Cross-compiled binaries for linux/{amd64,arm64}, +# darwin/{amd64,arm64}, windows/amd64 +# - Cosign keyless signature for each binary (Sigstore Rekor; +# same machinery as tracebloc/data-ingestors' image release) +# - SHA256SUMS file covering every binary +# - install.sh + install.ps1 attached so the bootstrap URL +# `https://github.com/tracebloc/cli/releases/latest/download/install.sh` +# resolves to the right script for that release +# - GitHub Release with all of the above attached + release-notes +# body generated from CHANGELOG.md +# - Optional: Homebrew tap formula bump (gated on the +# HOMEBREW_TAP_TOKEN secret being present — uncommenting that +# secret in repo settings turns the bump on for future releases) +# +# Verification (one-liner customers run to check the signature): +# +# cosign verify-blob \ +# --certificate-identity-regexp \ +# 'https://github.com/tracebloc/cli/.github/workflows/release.yml@.*' \ +# --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ +# --certificate .cert \ +# --signature .sig \ +# + +name: Release + +on: + push: + tags: + - 'v*.*.*' + workflow_dispatch: + inputs: + ref: + description: 'Tag to build (e.g. v0.1.0). Must already exist on origin.' + required: true + type: string + +permissions: + contents: write # create / update the GitHub Release + id-token: write # cosign keyless OIDC + +jobs: + release: + name: Build + sign + publish + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - os: linux + arch: amd64 + - os: linux + arch: arm64 + - os: darwin + arch: amd64 + - os: darwin + arch: arm64 + - os: windows + arch: amd64 + ext: .exe + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: true + + - name: Install cosign + uses: sigstore/cosign-installer@v3 + with: + cosign-release: 'v2.4.1' + + - name: Determine release version + id: version + run: | + # On a tag push: github.ref_name = "v0.1.0" + # On workflow_dispatch: inputs.ref = "v0.1.0" + REF="${{ inputs.ref || github.ref_name }}" + # Strip the leading v for use in -X main.version + VERSION="${REF#v}" + echo "tag=$REF" >> $GITHUB_OUTPUT + echo "version=$VERSION" >> $GITHUB_OUTPUT + echo "Building $REF (version=$VERSION)" + + - name: Build binary + env: + GOOS: ${{ matrix.os }} + GOARCH: ${{ matrix.arch }} + CGO_ENABLED: '0' + run: | + mkdir -p dist + BIN_NAME="tracebloc-${{ steps.version.outputs.tag }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}" + go build \ + -trimpath \ + -ldflags "\ + -s -w \ + -X main.version=${{ steps.version.outputs.version }} \ + -X main.gitSHA=${GITHUB_SHA:0:12} \ + -X main.buildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + -o "dist/$BIN_NAME" \ + ./cmd/tracebloc + echo "Binary built: dist/$BIN_NAME" + ls -lh "dist/$BIN_NAME" + + - name: Sign binary with cosign (keyless) + env: + # Required for keyless signing in GHA. The id-token: write + # permission above grants the workflow an OIDC token Sigstore + # validates against github.com's well-known issuer. + COSIGN_YES: 'true' + run: | + BIN_NAME="tracebloc-${{ steps.version.outputs.tag }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}" + cd dist + cosign sign-blob \ + --output-certificate "$BIN_NAME.cert" \ + --output-signature "$BIN_NAME.sig" \ + "$BIN_NAME" + echo "Signed: $BIN_NAME" + ls -lh "$BIN_NAME"* + + - name: Compute SHA256 + run: | + BIN_NAME="tracebloc-${{ steps.version.outputs.tag }}-${{ matrix.os }}-${{ matrix.arch }}${{ matrix.ext }}" + cd dist + # Per-binary SHA256 file. The release step aggregates them + # into a single SHA256SUMS that the install.sh script reads. + sha256sum "$BIN_NAME" > "$BIN_NAME.sha256" + cat "$BIN_NAME.sha256" + + - name: Upload per-matrix artifacts + uses: actions/upload-artifact@v4 + with: + name: dist-${{ matrix.os }}-${{ matrix.arch }} + path: dist/ + retention-days: 7 + if-no-files-found: error + + publish: + name: Aggregate + create GitHub Release + runs-on: ubuntu-latest + needs: release + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Download all matrix artifacts + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: Aggregate SHA256SUMS + run: | + cd dist + # Combine every per-binary .sha256 into one canonical file. + # The install scripts download SHA256SUMS + their target binary, + # grep the line for that binary, and verify locally. + cat tracebloc-*.sha256 > SHA256SUMS + rm tracebloc-*.sha256 + echo "Aggregated SHA256SUMS:" + cat SHA256SUMS + + - name: Stage install scripts into the release + run: | + # The install.sh / install.ps1 ship inside the release so + # `releases/latest/download/install.sh` resolves stably (the + # raw scripts/ path would require knowing the tag). + cp scripts/install.sh dist/ + cp scripts/install.ps1 dist/ + + - name: Determine release tag + id: tag + run: | + REF="${{ inputs.ref || github.ref_name }}" + echo "tag=$REF" >> $GITHUB_OUTPUT + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ steps.tag.outputs.tag }} + name: ${{ steps.tag.outputs.tag }} + generate_release_notes: true + # Mark as prerelease for v*.*.* tags containing - (e.g. + # v0.1.0-rc1). Plain semver releases are stable. + prerelease: ${{ contains(steps.tag.outputs.tag, '-') }} + files: | + dist/tracebloc-* + dist/SHA256SUMS + dist/install.sh + dist/install.ps1 + + # Phase 5 follow-up: bump the Homebrew tap formula on each + # release. Disabled by default — flip the `if:` to `true` (and + # add a HOMEBREW_TAP_TOKEN repo secret with write access to + # tracebloc/homebrew-tap) when that repo exists. + bump-homebrew-tap: + name: Bump Homebrew tap formula + runs-on: ubuntu-latest + needs: publish + if: false # ← flip when tracebloc/homebrew-tap is set up + steps: + - name: Checkout cli (for the formula template) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref || github.ref }} + + - name: Determine release tag + id: tag + run: | + REF="${{ inputs.ref || github.ref_name }}" + echo "tag=$REF" >> $GITHUB_OUTPUT + echo "version=${REF#v}" >> $GITHUB_OUTPUT + + - name: Download SHA256SUMS from the release + run: | + curl -fsSL \ + "https://github.com/tracebloc/cli/releases/download/${{ steps.tag.outputs.tag }}/SHA256SUMS" \ + -o SHA256SUMS + # Extract the per-platform shasums the formula needs. + DARWIN_AMD64_SHA=$(grep "darwin-amd64" SHA256SUMS | awk '{print $1}') + DARWIN_ARM64_SHA=$(grep "darwin-arm64" SHA256SUMS | awk '{print $1}') + LINUX_AMD64_SHA=$(grep "linux-amd64" SHA256SUMS | awk '{print $1}') + LINUX_ARM64_SHA=$(grep "linux-arm64" SHA256SUMS | awk '{print $1}') + echo "darwin_amd64_sha=$DARWIN_AMD64_SHA" >> $GITHUB_OUTPUT + echo "darwin_arm64_sha=$DARWIN_ARM64_SHA" >> $GITHUB_OUTPUT + echo "linux_amd64_sha=$LINUX_AMD64_SHA" >> $GITHUB_OUTPUT + echo "linux_arm64_sha=$LINUX_ARM64_SHA" >> $GITHUB_OUTPUT + id: shasums + + - name: Render formula from template + run: | + mkdir -p out/Formula + # Substitute placeholders in the formula template. Pure + # sed — no Ruby installation needed in the runner. + sed \ + -e "s/__VERSION__/${{ steps.tag.outputs.version }}/g" \ + -e "s/__TAG__/${{ steps.tag.outputs.tag }}/g" \ + -e "s/__SHA_DARWIN_AMD64__/${{ steps.shasums.outputs.darwin_amd64_sha }}/g" \ + -e "s/__SHA_DARWIN_ARM64__/${{ steps.shasums.outputs.darwin_arm64_sha }}/g" \ + -e "s/__SHA_LINUX_AMD64__/${{ steps.shasums.outputs.linux_amd64_sha }}/g" \ + -e "s/__SHA_LINUX_ARM64__/${{ steps.shasums.outputs.linux_arm64_sha }}/g" \ + scripts/homebrew-formula.rb.tmpl > out/Formula/tracebloc.rb + echo "--- Rendered formula ---" + cat out/Formula/tracebloc.rb + + - name: Push to tracebloc/homebrew-tap + env: + GH_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} + run: | + git clone https://x-access-token:${GH_TOKEN}@github.com/tracebloc/homebrew-tap.git tap + cp out/Formula/tracebloc.rb tap/Formula/tracebloc.rb + cd tap + git config user.name "tracebloc-cli-release-bot" + git config user.email "bot@tracebloc.io" + git add Formula/tracebloc.rb + git commit -m "tracebloc ${{ steps.tag.outputs.tag }}" + git push diff --git a/.gitignore b/.gitignore index 72a79d0..ac2c77c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ # Build output — populated by `go build ./cmd/tracebloc` at the repo -# root. Release artifacts go through dist/ (see Phase 5 / release -# workflow); local hacking puts a binary here. +# root, or by `go build -o ./bin/tracebloc ./cmd/tracebloc` for the +# convention some folks prefer. Release artifacts go through dist/ +# (see Phase 5 / release workflow); local hacking puts a binary here. /tracebloc +/bin/ # Cross-compiled binaries staged by the release workflow. /dist/ diff --git a/.golangci.yml b/.golangci.yml index 7763fa4..a6d52e7 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -7,17 +7,32 @@ run: timeout: 5m - go: "1.22" + # Track go.mod's `go` directive. Go's release cadence + `go mod + # tidy`'s aggressive bumping (especially when k8s.io/* deps want + # newer Go) keep dragging go.mod's minimum up; pinning a stale + # version here just produces typecheck errors instead of real + # findings. Currently 1.26 to match `go 1.26.0` in go.mod (the + # earlier "1.24" pin drifted when k8s.io/* v0.36 bumped the floor; + # Bugbot caught the drift on PR #2). + go: "1.26" linters: disable-all: true enable: - # Standard set — these catch real bugs and have very low false-positive rates. + # The set is trimmed to the linters that DON'T do full whole-program + # SSA analysis. `staticcheck` and `unused` were in the original set + # and reproducibly caused the GitHub-hosted runner to time-budget + # the job (~2 min then shutdown signal) on this module — k8s.io/* + # transitive deps inflate the analysis graph enough to OOM-or-stall + # the standard 4-CPU/16GB runner. Re-enabling them is a v0.2 + # follow-up that needs either a larger runner, a much narrower + # scope (e.g. only `./internal/...`), or a faster successor like + # govulncheck for the security-only subset. + + # Cheap, per-file checks — catch real bugs without SSA. - errcheck # unchecked error returns - govet # `go vet` - ineffassign # assignments that go nowhere - - staticcheck # SA-series checks (well-vetted) - - unused # dead code # Style / hygiene that pays for itself in code review time. - gofmt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8c1bf3f..2bd03cb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,14 +2,19 @@ ## Local development -```bash -go build -o tracebloc ./cmd/tracebloc -./tracebloc version +The `Makefile` mirrors the CI pipeline — `make ci` runs the exact same checks that PR #N's GitHub Actions run. If `make ci` passes locally, CI will too (modulo non-deterministic flakes). **Run `make ci` before pushing.** Skipping it has cost us at least one PR's worth of fix-up commits per bug class so far. -go test ./... -golangci-lint run # https://golangci-lint.run/usage/install/ +```bash +make ci # vet + test + lint + fmt-check + schema-check (run this before push) +make build # produces ./tracebloc +make fmt # fixes gofmt -s drift in place +make schema-sync # pulls latest ingest.v1.json from data-ingestors master ``` +Individual targets are also runnable in isolation — `make test`, `make lint`, etc. See the `Makefile` for the full list. + +Requires [`golangci-lint`](https://golangci-lint.run/usage/install/) (install via `brew install golangci-lint` or your platform's equivalent). + Cobra autocomplete for `bash` / `zsh` / `fish` / `powershell` is available via the `completion` subcommand. Useful while developing too: ```bash diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..bcf7328 --- /dev/null +++ b/Makefile @@ -0,0 +1,98 @@ +# Top-level Makefile for tracebloc/cli. +# +# Purpose: keep the local feedback loop the same shape as the CI +# loop. Anything that fails in `make ci` would have failed on a PR, +# and vice versa. Don't add targets here that aren't also enforced +# by .github/workflows/build.yml — divergence between local and CI +# is the bug this file exists to prevent. + +# ---- toggles ----------------------------------------------------- + +GO ?= go +GOLANGCI_LINT ?= golangci-lint +PKGS := ./... + +# ---- top-level targets ------------------------------------------- + +.PHONY: ci +ci: vet test lint fmt-check schema-check + @echo "==> ci: all green" + +.PHONY: build +build: + $(GO) build -o tracebloc ./cmd/tracebloc + +.PHONY: install +install: + $(GO) install ./cmd/tracebloc + +# ---- individual targets (also runnable in isolation) ------------- + +.PHONY: vet +vet: + $(GO) vet $(PKGS) + +.PHONY: test +test: + $(GO) test -race -cover $(PKGS) + +# Integration tests (build-tagged `integration`) run against a REAL +# cluster reachable via the ambient kubeconfig — kind in CI, or your +# own dev cluster locally. They cover the real-I/O seams the unit +# suite mocks (clientset connectivity, the SPDYExecutor tar-over-exec +# stream against a live Pod + PVC). -count=1 disables caching since +# these touch live cluster state. See .github/workflows/e2e.yml. +.PHONY: test-integration +test-integration: + $(GO) test -tags integration -count=1 -timeout 10m -v ./test/integration/... + +# Lint set matched to .github/workflows/build.yml's lint job: errcheck + +# ineffassign + misspell (gofmt -s is `fmt-check`, go vet is `vet`). +# golangci-lint-action is disabled in CI pending tracebloc/cli#6 — so +# until it's re-enabled there, `make ci` runs the SAME standalone tools +# the CI lint job runs, keeping the "make ci green => CI green" invariant +# this Makefile exists to protect. `make lint-full` keeps golangci-lint +# available for a richer local pass. +.PHONY: lint +lint: + $(GO) run github.com/kisielk/errcheck@latest ./... + $(GO) run github.com/gordonklaus/ineffassign@latest ./... + $(GO) run github.com/client9/misspell/cmd/misspell@latest -error . + +.PHONY: lint-full +lint-full: + @command -v $(GOLANGCI_LINT) >/dev/null 2>&1 || { \ + echo "==> $(GOLANGCI_LINT) not on PATH"; \ + echo " install via: brew install golangci-lint"; \ + echo " or see: https://golangci-lint.run/usage/install/"; \ + exit 1; \ + } + $(GOLANGCI_LINT) run + +.PHONY: fmt +fmt: + gofmt -s -w . + +.PHONY: fmt-check +fmt-check: + @diff="$$(gofmt -s -l . 2>/dev/null)"; \ + if [ -n "$$diff" ]; then \ + echo "==> gofmt -s needed on:"; \ + echo "$$diff" | sed 's/^/ /'; \ + echo "==> run \`make fmt\` to fix"; \ + exit 1; \ + fi + +.PHONY: schema-check +schema-check: + ./scripts/sync-schema.sh --check + +.PHONY: schema-sync +schema-sync: + ./scripts/sync-schema.sh + +# ---- cleanup ----------------------------------------------------- + +.PHONY: clean +clean: + rm -rf tracebloc dist/ coverage.out coverage.html diff --git a/README.md b/README.md index cd2746c..da3f27b 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,13 @@ The customer-facing CLI for the tracebloc declarative ingestion path. Wraps the ## Status -🚧 **Pre-alpha** — Phase 0 of the [v0.1 roadmap](https://github.com/tracebloc/client/issues/147). Today this binary implements only `version` and `completion`. Subsequent phases (#149–#153) land schema validation, cluster discovery, data staging, submission, and distribution. +**v0.1.0-alpha is published** — the first [release](https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha), a pre-release for early testers, cut from `develop`. Every phase of the [v0.1 roadmap](https://github.com/tracebloc/client/issues/147) (#148–#153) is merged. The binary implements `version`, `completion`, `ingest validate`, `cluster info`, and the full `dataset push` flow — local schema validation, cluster discovery, data staging, submission, and Job watching, end to end. -For the production ingestion flow today, use the Helm chart: `helm install tracebloc/ingestor --set-file ingestConfig=./ingest.yaml`. See the chart's [README](https://github.com/tracebloc/client/blob/develop/ingestor/README.md) for the full customer journey including the data-staging recipe. +`dataset push` covers **9 of 10 task categories**: `image_classification`, `object_detection`, `keypoint_detection`, `text_classification`, `masked_language_modeling`, `tabular_classification`, `tabular_regression`, `time_series_forecasting`, and `time_to_event_prediction`. `semantic_segmentation` is pending mask-sidecar support upstream ([data-ingestors#136](https://github.com/tracebloc/data-ingestors/issues/136)); `instance_segmentation` is not yet implemented. + +The release pipeline has shipped [`v0.1.0-alpha`](https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha): cosign-signed binaries for linux/darwin/windows (amd64/arm64), `SHA256SUMS`, and the install scripts. Install it via [Customer experience](#customer-experience) or [build from source](#building-from-source). (The `install.tracebloc.io` shortcut, the `latest` one-liner, and a Homebrew tap all wait for the first **stable** `v*` tag — see below.) + +The Helm chart remains a sibling interface for the Kubernetes-native workflow: `helm install tracebloc/ingestor --set-file ingestConfig=./ingest.yaml` (see the chart's [README](https://github.com/tracebloc/client/blob/develop/ingestor/README.md)). ## Why a CLI in addition to the chart? @@ -37,13 +41,15 @@ Customer interfaces (pick one or many): The protocol — the v1 schema + the POST endpoint — is the stable point. Everything above is interchangeable. -## v0.1 design (target customer experience, once Phase 5 ships) +## Customer experience + +> `v0.1.0-alpha` is a **pre-release**, so the `install.tracebloc.io` shortcut isn't wired up yet and GitHub's `latest` skips pre-releases. Install the alpha by explicit tag (below), or [build from source](#building-from-source). ```bash -# Install once -brew install tracebloc/tap/tracebloc # macOS -curl -fsSL https://install.tracebloc.io | sh # Linux/macOS -irm https://install.tracebloc.io/install.ps1 | iex # Windows +# Install the alpha — pins the pre-release tag (Linux/macOS) +curl -fsSL https://github.com/tracebloc/cli/releases/download/v0.1.0-alpha/install.sh \ + | sh -s -- --version v0.1.0-alpha +# Windows + manual/signed downloads: https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha # Per dataset tracebloc dataset push ./my-data \ @@ -73,7 +79,7 @@ go build -o tracebloc ./cmd/tracebloc ./tracebloc version ``` -Requires Go 1.22 or newer. The binary self-reports its build metadata; a `go build` without `-ldflags` reports `dev / unknown / unknown` for version/git-sha/build-date so support can tell a local hack apart from a release build. +Requires Go 1.26 or newer (the `k8s.io/*` dependencies set the floor; see `go.mod`). The binary self-reports its build metadata; a `go build` without `-ldflags` reports `dev / unknown / unknown` for version/git-sha/build-date so support can tell a local hack apart from a release build. ```bash # Release-style build with version metadata @@ -86,14 +92,20 @@ go build -ldflags "\ ## Roadmap -| Phase | Ticket | What | -|---|---|---| -| **0 (this PR)** | [#148](https://github.com/tracebloc/client/issues/148) | Repo bootstrap + Go module + CI + `tracebloc version` | -| 1 | [#149](https://github.com/tracebloc/client/issues/149) | Embed `ingest.v1.json` + `tracebloc ingest validate ` (local-only) | -| 2 | [#150](https://github.com/tracebloc/client/issues/150) | Cluster discovery + ingestor SA token via TokenRequest | -| 3 | [#151](https://github.com/tracebloc/client/issues/151) | Stage data into the shared PVC via ephemeral Pod | -| 4 | [#152](https://github.com/tracebloc/client/issues/152) | Submit to jobs-manager + watch ingestor Job + summary | -| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases, Homebrew tap, install.sh distribution | +All v0.1 phases are merged: + +| Phase | Ticket | What | Status | +|---|---|---|---| +| 0 | [#148](https://github.com/tracebloc/client/issues/148) | Repo bootstrap + Go module + CI + `tracebloc version` | ✅ | +| 1 | [#149](https://github.com/tracebloc/client/issues/149) | Embed `ingest.v1.json` + `tracebloc ingest validate ` (local-only) | ✅ | +| 2 | [#150](https://github.com/tracebloc/client/issues/150) | Cluster discovery + ingestor SA token via TokenRequest | ✅ | +| 3 | [#151](https://github.com/tracebloc/client/issues/151) | Stage data into the shared PVC via ephemeral Pod | ✅ | +| 4 | [#152](https://github.com/tracebloc/client/issues/152) | Submit to jobs-manager + watch ingestor Job + summary | ✅ | +| 5 | [#153](https://github.com/tracebloc/client/issues/153) | GitHub Releases + install.sh distribution (Homebrew tap deferred) | ✅ — [`v0.1.0-alpha`](https://github.com/tracebloc/cli/releases/tag/v0.1.0-alpha) published (pre-release) | + +Beyond the original phases, `dataset push` was widened from image-classification-only to 9 of 10 modalities, and the test suite gained unit-coverage wins plus a kind-based integration harness for the real-I/O seams. + +**Next (v0.2):** cloud-source ingestion (S3/GCS/HTTPS) for datasets above the 1 GiB local cap; `semantic_segmentation` ([data-ingestors#136](https://github.com/tracebloc/data-ingestors/issues/136)) and `instance_segmentation`; more `dataset` verbs (`list`, `rm`). Smaller follow-ups are tracked as [open issues](https://github.com/tracebloc/cli/issues) (#3–#7). Epic: [tracebloc/client#147](https://github.com/tracebloc/client/issues/147). diff --git a/cmd/tracebloc/main.go b/cmd/tracebloc/main.go index 071bba4..978cde4 100644 --- a/cmd/tracebloc/main.go +++ b/cmd/tracebloc/main.go @@ -18,7 +18,11 @@ package main import ( + "context" + "fmt" "os" + "os/signal" + "syscall" "github.com/tracebloc/cli/internal/cli" ) @@ -34,13 +38,59 @@ var ( ) func main() { - if err := cli.NewRootCmd(cli.BuildInfo{ + // Wire SIGINT / SIGTERM into the cobra root command's context. + // Long-running operations (e.g. push.Stage in `dataset push`) + // propagate ctx down through every k8s API call, so cancelling + // here triggers ctx.Done() → in-flight HTTP cancels → defers + // fire in normal stack unwind → orphan Pod gets cleaned up. + // + // Without this wire, Ctrl-C goes straight through Go's runtime + // signal handler and exits the process WITHOUT running defers + // — which silently broke the SIGINT-safe cleanup contract in + // push.Stage's docstring. Bugbot flagged this on PR-b. + // + // signal.NotifyContext (Go 1.16+) is the stdlib pattern for + // this. The stop func unregisters the handler so a *second* + // SIGINT does the normal hard-kill — important for the case + // where the cleanup itself hangs (e.g. API server unreachable + // even past the 30s cleanup deadline). The customer's second + // Ctrl-C should always work. + ctx, stop := signal.NotifyContext(context.Background(), + syscall.SIGINT, syscall.SIGTERM) + defer stop() + + err := cli.NewRootCmd(cli.BuildInfo{ Version: version, GitSHA: gitSHA, BuildDate: buildDate, - }).Execute(); err != nil { - // cobra has already printed the error + usage to stderr by - // the time we get here; just propagate a non-zero exit. - os.Exit(1) + }).ExecuteContext(ctx) + if err == nil { + return + } + + // Print the error to stderr before exiting. The root command + // sets SilenceErrors: true to keep cobra from prepending its + // own "Error: ..." line on top of structured handler output + // — but that puts the burden on us to surface the error + // message ourselves. Without this, every non-schema-violation + // failure (file-read errors, YAML parse errors, schema-compile + // errors) exits non-zero with NO message to the customer. + // + // Handlers that have already printed their own diagnostic + // (e.g. `ingest validate` prints per-violation lines) signal + // "silent" by returning an exitError with a nil inner — see + // cli.IsSilentError for the contract. + if !cli.IsSilentError(err) { + // Explicit-discard the writer error: if stderr itself is + // gone (closed pipe, redirected to /dev/full, etc.) we + // still need to exit non-zero — the error message is + // best-effort, the exit code is the contract. + _, _ = fmt.Fprintln(os.Stderr, "Error:", err) } + + // Map command-defined exit codes through. Handlers that want a + // specific exit code (e.g. `ingest validate` returns 2 for + // schema violations, 3 for parse errors) return a *cli.ExitError + // the package exports; everything else gets the default 1. + os.Exit(cli.ExitCodeFromError(err)) } diff --git a/go.mod b/go.mod index 796f785..56b4e7b 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,63 @@ module github.com/tracebloc/cli -go 1.22 +// Minimum Go is what `go 1.26.0` says below — this used to claim +// "1.22" but the k8s.io/* v0.36.x deps dragged the floor up to 1.26 +// (transitively via `go mod tidy`), and the stale comment misled +// Bugbot and humans alike. The lint config in .golangci.yml's `go:` +// key tracks this directive; bump both together when the minimum +// moves again. +go 1.26.0 -require github.com/spf13/cobra v1.8.1 +require ( + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 + github.com/schollz/progressbar/v3 v3.19.0 + github.com/spf13/cobra v1.8.1 + golang.org/x/term v0.39.0 + golang.org/x/text v0.33.0 + gopkg.in/yaml.v3 v3.0.1 + k8s.io/api v0.36.1 + k8s.io/apimachinery v0.36.1 + k8s.io/client-go v0.36.1 +) require ( + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/google/gnostic-models v0.7.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect + github.com/moby/spdystream v0.5.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/spf13/pflag v1.0.9 // indirect + github.com/x448/float16 v0.8.4 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + k8s.io/klog/v2 v2.140.0 // indirect + k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a // indirect + k8s.io/streaming v0.36.1 // indirect + k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect + sigs.k8s.io/randfill v1.0.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.3.2 // indirect + sigs.k8s.io/yaml v1.6.0 // indirect ) diff --git a/go.sum b/go.sum index 912390a..a49a7f6 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,144 @@ +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/chengxilo/virtualterm v1.0.4 h1:Z6IpERbRVlfB8WkOmtbHiDbBANU7cimRIof7mk9/PwM= +github.com/chengxilo/virtualterm v1.0.4/go.mod h1:DyxxBZz/x1iqJjFxTFcr6/x+jSpqN0iwWCOK1q10rlY= github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= +github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= +github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/google/gnostic-models v0.7.0 h1:qwTtogB15McXDaNqTZdzPJRHvaVJlAl+HVQnLmJEJxo= +github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7OUGxBlw57miDrQ= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 h1:JeSE6pjso5THxAzdVpqr6/geYxZytqFMBCOtn/ujyeo= +github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674/go.mod h1:r4w70xmWCQKmi1ONH4KIaBptdivuRPyosB9RmPlGEwA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= +github.com/moby/spdystream v0.5.1 h1:9sNYeYZUcci9R6/w7KDaFWEWeV4LStVG78Mpyq/Zm/Y= +github.com/moby/spdystream v0.5.1/go.mod h1:xBAYlnt/ay+11ShkdFKNAG7LsyK/tmNBVvVOwrfMgdI= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8= +github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= +github.com/schollz/progressbar/v3 v3.19.0 h1:Ea18xuIRQXLAUidVDox3AbwfUhD0/1IvohyTutOIFoc= +github.com/schollz/progressbar/v3 v3.19.0/go.mod h1:IsO3lpbaGuzh8zIMzgY3+J8l4C8GjO0Y9S69eFvNsec= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= +github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= +golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.36.1 h1:XbL/EMj8K2aJpJtePmqUyQMsM0D4QI2pvl7YKJ20FTY= +k8s.io/api v0.36.1/go.mod h1:KOWo4ey3TINlXjeHVuwB3i+tXXnu+UcwFBHlI/9dvEo= +k8s.io/apimachinery v0.36.1 h1:G63Gjx2W+q0YD+72Vo8oY0nDnePVwnuzTmmy5ENrVSA= +k8s.io/apimachinery v0.36.1/go.mod h1:ibYOR00vW/I1kzvi5SF0dRuJ52BvKtfvRdOn35GPQ+8= +k8s.io/client-go v0.36.1 h1:FN/K8QIT2CEDt+2WB2HnWrUANZ50AP5GII43/SP2JR0= +k8s.io/client-go v0.36.1/go.mod h1:s6rAnCtTGYDQnpNjEhSaISV+2O8jwruZ6m3QOYBFbtU= +k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= +k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a h1:xCeOEAOoGYl2jnJoHkC3hkbPJgdATINPMAxaynU2Ovg= +k8s.io/kube-openapi v0.0.0-20260317180543-43fb72c5454a/go.mod h1:uGBT7iTA6c6MvqUvSXIaYZo9ukscABYi2btjhvgKGZ0= +k8s.io/streaming v0.36.1 h1:L+K68n4Gg940BGNNYtUBvL1WTLL0YnKT3s+P1MNAmR4= +k8s.io/streaming v0.36.1/go.mod h1:z6fV3D+NVkoeqRMtWwlUZK6U17SY/LqNzOxWL6GyR/s= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= +sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2 h1:kwVWMx5yS1CrnFWA/2QHyRVJ8jM6dBA80uLmm0wJkk8= +sigs.k8s.io/structured-merge-diff/v6 v6.3.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= +sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/internal/cli/cluster.go b/internal/cli/cluster.go new file mode 100644 index 0000000..fd9da91 --- /dev/null +++ b/internal/cli/cluster.go @@ -0,0 +1,202 @@ +package cli + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" + + "github.com/spf13/cobra" + + "github.com/tracebloc/cli/internal/cluster" +) + +// newClusterCmd wires the `tracebloc cluster` subtree. Today it has +// a single verb — `info` — which is the customer's "is the CLI +// pointing at the right cluster?" pre-flight before running +// `dataset push`. Future verbs (e.g. `cluster doctor` for +// diagnostics, `cluster contexts` for switching) hang off this +// parent in later phases. +func newClusterCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "cluster", + Short: "Inspect the cluster the CLI is currently targeting", + Long: `Commands for inspecting the Kubernetes cluster the CLI is +configured to talk to. + +Use ` + "`cluster info`" + ` to verify which cluster, namespace, and parent +tracebloc release the next ` + "`dataset push`" + ` will target. Useful as a +pre-flight before doing anything destructive (e.g. ingesting into +the wrong cluster).`, + } + + cmd.AddCommand(newClusterInfoCmd()) + return cmd +} + +// newClusterInfoCmd implements `tracebloc cluster info`. Discovers +// the parent client release in the configured namespace, mints (or +// looks up) an ingestor SA token, and prints a single-screen +// summary the customer can sanity-check before running a real +// ingestion. +// +// Three kubeconfig flags follow kubectl conventions: +// +// --kubeconfig=PATH override KUBECONFIG / ~/.kube/config +// --context=NAME override the kubeconfig's current-context +// --namespace=NAME override the context's default namespace +// +// All three are zero-value-safe — running `tracebloc cluster info` +// with no flags uses the customer's normal kubectl defaults. +func newClusterInfoCmd() *cobra.Command { + var ( + kubeconfigPath string + contextOverride string + nsOverride string + ingestorSAName string + tokenExpiry int64 + ) + + cmd := &cobra.Command{ + Use: "info", + Short: "Show the cluster, namespace, parent release, and ingestor SA token state", + Long: `Discovers the tracebloc parent client release in the configured +cluster + namespace and prints: + + • Which kubeconfig context the CLI used + • The namespace it resolved to + • The parent release name + chart version + appVersion + • The jobs-manager Service the next dataset push would POST to + • The ingestor ServiceAccount the post-install hook would auth as + • The cluster's configured INGESTOR_IMAGE_DIGEST default + • Whether the user's kubeconfig can mint short-lived SA tokens + via TokenRequest, or has to fall back to a static + service-account-token Secret + +The actual token bytes are never printed; the diagnostic shows +SHA256(token)[:8] so the customer can verify "yes, that's the +token I expect" without leaking it to terminal scrollback. + +Exit codes: + 0 cluster discovered + token mintable; CLI is ready + 4 cluster reachable but no tracebloc parent release found + 5 cluster reachable + release found but no usable SA token`, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return runClusterInfo( + cmd.Context(), + cmd.OutOrStdout(), + kubeconfigPath, contextOverride, nsOverride, + ingestorSAName, + tokenExpiry, + ) + }, + } + + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", "", + "path to the kubeconfig file (default: $KUBECONFIG, then ~/.kube/config)") + cmd.Flags().StringVar(&contextOverride, "context", "", + "name of the kubeconfig context to use (default: kubeconfig's current-context)") + cmd.Flags().StringVarP(&nsOverride, "namespace", "n", "", + "namespace where the parent tracebloc/client release is installed (default: the context's namespace, or 'default')") + cmd.Flags().StringVar(&ingestorSAName, "ingestor-sa", "", + "override the ingestor ServiceAccount name (default: \"ingestor\", the chart default; "+ + "set this if you customized `ingestionAuthz.serviceAccountName` in the parent client chart)") + cmd.Flags().Int64Var(&tokenExpiry, "token-expiry-seconds", 600, + "requested SA token expiration in seconds (default 600 = 10 min; ignored for static-secret fallback)") + + return cmd +} + +func runClusterInfo( + ctx context.Context, + out interface{ Write([]byte) (int, error) }, + kubeconfigPath, contextOverride, nsOverride string, + ingestorSAOverride string, + tokenExpiry int64, +) error { + resolved, err := cluster.Load(cluster.KubeconfigOptions{ + Path: kubeconfigPath, + Context: contextOverride, + Namespace: nsOverride, + }) + if err != nil { + // Kubeconfig errors are exit-code-3 territory (file/parse + // problem, same conceptual class as `ingest validate`'s + // unreadable-input). + return &exitError{code: 3, err: fmt.Errorf("loading kubeconfig: %w", err)} + } + + cs, err := cluster.NewClientset(resolved) + if err != nil { + return &exitError{code: 3, err: err} + } + + // Explicit-discard the writer errors: same rationale as + // internal/cli/ingest.go — the exit code is the contract, and a + // pipe-write failure shouldn't convert success into failure. If + // the downstream consumer disappears mid-write we still finish + // cleanly. errcheck wants this acknowledged. + _, _ = fmt.Fprintf(out, "Kubeconfig:\n") + _, _ = fmt.Fprintf(out, " context: %s\n", resolved.Context) + _, _ = fmt.Fprintf(out, " server: %s\n", resolved.ServerURL) + _, _ = fmt.Fprintf(out, " namespace: %s\n", resolved.Namespace) + _, _ = fmt.Fprintln(out) + + // Discover the parent release. + release, err := cluster.DiscoverParentRelease(ctx, cs, resolved.Namespace) + if err != nil { + // 4 = "cluster reachable, but no tracebloc release here." + // Distinct from the kubeconfig error (3) so callers can + // branch: 3 means "fix your kubeconfig", 4 means "install + // the parent chart first". + return &exitError{code: 4, err: err} + } + + // Apply the SA-name override here. Discovery doesn't read the + // name from the cluster (see #7); customers with a non-default + // name pass --ingestor-sa. + if ingestorSAOverride != "" { + release.IngestorSAName = ingestorSAOverride + } + + _, _ = fmt.Fprintf(out, "Parent release:\n") + _, _ = fmt.Fprintf(out, " name: %s\n", release.ReleaseName) + _, _ = fmt.Fprintf(out, " chart version: %s\n", release.ChartVersion) + _, _ = fmt.Fprintf(out, " app version: %s\n", release.AppVersion) + _, _ = fmt.Fprintf(out, " jobs-manager: %s\n", release.JobsManagerService) + _, _ = fmt.Fprintf(out, " ingestor SA: %s/%s\n", resolved.Namespace, release.IngestorSAName) + digest := release.IngestorImageDigest + if digest == "" { + digest = "" + } + _, _ = fmt.Fprintf(out, " ingestor img: %s\n", digest) + _, _ = fmt.Fprintln(out) + + // Mint a token (or fall back). The audience is intentionally + // nil today — jobs-manager's TokenReview accepts the default + // API-server audience. A future jobs-manager audience plugs in + // here. + tok, err := cluster.MintIngestorToken(ctx, cs, resolved.Namespace, release.IngestorSAName, tokenExpiry, nil) + if err != nil { + // 5 = "release found but no usable token." Distinct from + // 4 (no release) so customers can RBAC-debug separately + // from install issues. + return &exitError{code: 5, err: err} + } + + hash := sha256.Sum256([]byte(tok.Token)) + _, _ = fmt.Fprintf(out, "Ingestor SA token:\n") + _, _ = fmt.Fprintf(out, " source: %s\n", tok.Source) + _, _ = fmt.Fprintf(out, " sha256[:8]: %s\n", hex.EncodeToString(hash[:8])) + if tok.ExpirationSeconds > 0 { + exp := time.Duration(tok.ExpirationSeconds) * time.Second + _, _ = fmt.Fprintf(out, " expires in: ~%s (server may cap shorter)\n", exp) + } else { + _, _ = fmt.Fprintf(out, " expires in: never (static-secret fallback)\n") + } + _, _ = fmt.Fprintln(out) + _, _ = fmt.Fprintln(out, "Ready for `tracebloc dataset push`.") + return nil +} diff --git a/internal/cli/coverage_test.go b/internal/cli/coverage_test.go new file mode 100644 index 0000000..0cbe474 --- /dev/null +++ b/internal/cli/coverage_test.go @@ -0,0 +1,106 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/push" +) + +// TestPrintPushPreflight_RendersKeyFacts pins that the pre-flight +// summary surfaces the facts a customer sanity-checks before a push: +// the target release, the shared PVC, and the synthesized spec +// identity. It's the customer's last look before bytes move, so the +// content (not just "it didn't panic") is worth asserting. +func TestPrintPushPreflight_RendersKeyFacts(t *testing.T) { + layout := &push.LocalLayout{ + Root: "/tmp/cats_dogs", + LabelsCSV: "/tmp/cats_dogs/labels.csv", + Images: []string{"a.jpg", "b.jpg", "c.jpg"}, + TotalBytes: 1024, + } + release := &cluster.ParentRelease{ + ReleaseName: "ingdemo", + ChartVersion: "1.4.2", + JobsManagerService: "http://jobs-manager.ingdemo.svc.cluster.local:8080", + } + pvc := &cluster.SharedPVC{ + ClaimName: "client-pvc", + MountPath: "/data/shared", + Phase: corev1.ClaimBound, + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + } + spec := map[string]any{ + "table": "cats_dogs_train", + "category": "image_classification", + "intent": "train", + "label": "label", + } + + var buf bytes.Buffer + printPushPreflight(&buf, layout, release, pvc, spec, false) + out := buf.String() + + for _, want := range []string{ + "ingdemo", "1.4.2", "client-pvc", + "cats_dogs_train", "image_classification", "train", + } { + if !strings.Contains(out, want) { + t.Errorf("pre-flight output missing %q:\n%s", want, out) + } + } +} + +// TestExitError_Methods pins the exit-code carrier: Error() surfaces +// the wrapped message (or a fallback when nil), and Code() returns the +// process exit code main() propagates. +func TestExitError_Methods(t *testing.T) { + e := &exitError{code: 7, err: errors.New("staging failed")} + if e.Error() != "staging failed" { + t.Errorf("Error() = %q, want %q", e.Error(), "staging failed") + } + if e.Code() != 7 { + t.Errorf("Code() = %d, want 7", e.Code()) + } + // err==nil: Error() falls back to a generic "exit N" string so the + // type still satisfies error without panicking. + nilErr := &exitError{code: 2} + if !strings.Contains(nilErr.Error(), "2") { + t.Errorf("Error() on nil-err exitError = %q, want it to mention the code", nilErr.Error()) + } + if nilErr.Code() != 2 { + t.Errorf("Code() = %d, want 2", nilErr.Code()) + } +} + +// TestRunClusterInfo_BadKubeconfigExitsThree: an unreadable/invalid +// kubeconfig is exit-code-3 territory (the kubeconfig/local-input +// bucket), surfaced before any cluster work. Covers the Load-error +// branch of runClusterInfo without needing a real cluster. +func TestRunClusterInfo_BadKubeconfigExitsThree(t *testing.T) { + bad := filepath.Join(t.TempDir(), "broken.yaml") + if err := os.WriteFile(bad, []byte("}{ this is not valid kubeconfig"), 0o644); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + err := runClusterInfo(context.Background(), &buf, bad, "", "", "", 600) + if err == nil { + t.Fatal("runClusterInfo with a broken kubeconfig returned nil; want an exitError") + } + var ee *exitError + if !errors.As(err, &ee) { + t.Fatalf("error is not an *exitError: %v", err) + } + if ee.Code() != 3 { + t.Errorf("exit code = %d, want 3 (kubeconfig/local-input error)", ee.Code()) + } +} diff --git a/internal/cli/dataset.go b/internal/cli/dataset.go new file mode 100644 index 0000000..095dc1a --- /dev/null +++ b/internal/cli/dataset.go @@ -0,0 +1,697 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + "path/filepath" + "strings" + + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/push" + "github.com/tracebloc/cli/internal/schema" + "github.com/tracebloc/cli/internal/submit" +) + +// newDatasetCmd wires the `tracebloc dataset` subtree. The dominant +// verb is `push`, completed in Phase 3 (tracebloc/client#151) across +// PR-a (pre-flight: spec synth, validation, layout walk, cluster +// discovery) and PR-b (this one: ephemeral stage Pod + tar-over- +// exec stream + progress bar + SIGINT-safe cleanup). Future verbs +// (`dataset list`, `dataset rm`) hang off this parent in v0.2. +func newDatasetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "dataset", + Short: "Manage datasets in the parent client release", + Long: `Commands for staging and managing datasets on the cluster's +shared PVC. + +` + "`dataset push`" + ` stages a local dataset to the cluster's shared +PVC, submits the ingestion run to jobs-manager, and watches the +ingestor Job to completion (streaming logs + the final summary). + +` + "`tracebloc cluster info`" + ` is the pre-flight you'd typically run +before the first push.`, + } + cmd.AddCommand(newDatasetPushCmd()) + return cmd +} + +// newDatasetPushCmd implements `tracebloc dataset push `. +// +// Phase 3 scope (now complete across PR-a + PR-b): +// +// - Synthesize the ingest spec from flags (`internal/push.SpecArgs.Build`) +// - Validate it against the embedded ingest.v1 schema +// - Walk the local directory + enforce v0.1 size caps +// - Discover cluster, parent release, and shared PVC +// - Print a single-screen pre-flight summary +// - Either --dry-run stop, OR create an ephemeral stage Pod +// (alpine 3.20 pinned by digest, PSA-restricted security +// context), tar local files into it via an SPDY exec stream +// with a progress bar, then defer-delete the Pod +// +// Phase 4 (`tracebloc/client#152`) hooks the submit-to-jobs-manager +// step into the bottom of this command, replacing the "manually +// kick off helm ingestor" workaround in the success message. +func newDatasetPushCmd() *cobra.Command { + var ( + // Kubeconfig flags — same conventions as `cluster info`. + // Promoting these to persistent on the root is a v0.2 + // follow-up (tracebloc/cli#3); for now they live on each + // command that needs them. + kubeconfigPath string + contextOverride string + nsOverride string + + // Ingest-spec flags. image_classification + the tabular / + // time-series family are supported today; text + detection + + // segmentation land in later increments. + table string + category string + intent string + labelColumn string + targetSize string + schemaFlag string + labelPolicy string + timeColumn string + numberOfKeypoints int + + // Operations flags. + dryRun bool + + // Ingestor SA name override. Used as the ServiceAccountName + // of the ephemeral stage Pod, so the Pod inherits whatever + // imagePullSecrets + PSA exemptions the admin already + // configured for that SA. + ingestorSAName string + + // Stage Pod image override. Defaults to the digest-pinned + // alpine that ships with the CLI; air-gapped customers + // override this to an image their registry mirror serves. + // Pin by digest in your override too — tag-only references + // drift silently and break "all my pushes worked yesterday." + stagePodImage string + + // Phase 4 flags. --detach exits immediately after the 201 + // from jobs-manager; --idempotency-key plumbs through to + // the submit body for retry-safety across CLI invocations + // (default: fresh per call); --image-digest pins the + // ingestor image (default: jobs-manager picks the + // cluster-configured one). + detach bool + idempotencyKey string + imageDigest string + ) + + cmd := &cobra.Command{ + Use: "push ", + Short: "Stage a local dataset to the cluster's shared PVC", + Long: `Stages a local dataset to the parent client release's shared PVC, +submits an ingestion run to jobs-manager, and watches the ingestor Job +to completion. Supports 9 task categories (image classification, +object/keypoint detection, text classification, masked language +modeling, and the tabular / time-series family); pick one with --category. + +Expected local layout (image_classification shown): + + / + labels.csv (required) + images/ (required) + 001.jpg + 002.jpg + ... + +Accepted image extensions: .jpg, .jpeg, .png, .webp (case-insensitive). + +v0.1 caps the dataset at 1 GiB total + 500 MiB per file. Larger +datasets need the v0.2 cloud-source story (S3/GCS/HTTPS sources) — +see tracebloc/client#147 non-goals. + +Exit codes: + 0 files staged + ingested successfully (or --detach: just staged + submitted) + 2 schema validation failed (synthesized spec rejected) or + v0.1-unsupported category passed + 3 local-layout or kubeconfig error + 4 cluster reachable but parent release / shared PVC missing + 5 ingestor SA token couldn't be obtained, or jobs-manager + rejected the token (401/403) + 7 pre-flight succeeded but staging the files failed + (Pod creation, image pull, exec stream, or remote tar error) + 8 jobs-manager rejected the submit (4xx/5xx other than auth) + 9 ingestion Job exited non-zero, or completed with row-level + failures the summary panel reports`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return runDatasetPush(cmd.Context(), cmd.OutOrStdout(), cmd.ErrOrStderr(), + runDatasetPushArgs{ + LocalPath: args[0], + Kubeconfig: kubeconfigPath, + Context: contextOverride, + Namespace: nsOverride, + Spec: push.SpecArgs{ + Table: table, Category: category, Intent: intent, + LabelColumn: labelColumn, LabelPolicy: labelPolicy, TimeColumn: timeColumn, + NumberOfKeypoints: numberOfKeypoints, + }, + TargetSizeFlag: targetSize, + SchemaFlag: schemaFlag, + DryRun: dryRun, + IngestorSAName: ingestorSAName, + StagePodImage: stagePodImage, + Detach: detach, + IdempotencyKey: idempotencyKey, + ImageDigest: imageDigest, + }) + }, + } + + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", "", + "path to the kubeconfig file (default: $KUBECONFIG, then ~/.kube/config)") + cmd.Flags().StringVar(&contextOverride, "context", "", + "name of the kubeconfig context to use (default: kubeconfig's current-context)") + cmd.Flags().StringVarP(&nsOverride, "namespace", "n", "", + "namespace where the parent tracebloc/client release is installed") + + // Required spec flags. We DON'T mark them required-at-cobra-level + // because cobra's "required flag" error message is terse and + // pre-empts our richer schema-driven diagnostic. Instead, the + // schema validator catches missing/empty values with the canonical + // JSON-pointer-anchored error. + cmd.Flags().StringVar(&table, "table", "", + "destination table name (MySQL identifier; matches /data/shared// on the PVC)") + cmd.Flags().StringVar(&category, "category", "image_classification", + "task category: image_classification, tabular_classification, tabular_regression, "+ + "time_series_forecasting, time_to_event_prediction") + cmd.Flags().StringVar(&intent, "intent", "", + "intent: train|test") + cmd.Flags().StringVar(&labelColumn, "label-column", "", + "name of the label/target column (in labels.csv for image categories, in the data CSV for tabular)") + cmd.Flags().StringVar(&targetSize, "target-size", "", + "image categories only: resolution as WxH (e.g. 512x512). Default: auto-detected from the first image. "+ + "All images must share this resolution — the ingestor validates it, it does not resize.") + cmd.Flags().StringVar(&schemaFlag, "schema", "", + "tabular/time-series only: column types as col:TYPE,col:TYPE (e.g. age:INT,price:FLOAT). "+ + "Default: inferred from the CSV (INT/FLOAT/VARCHAR).") + cmd.Flags().StringVar(&labelPolicy, "label-policy", "", + "regression-class only (tabular_regression, time_series_forecasting, time_to_event_prediction): "+ + "passthrough|bucket (default bucket — bins the target so the raw value never leaves the cluster)") + cmd.Flags().StringVar(&timeColumn, "time-column", "", + "time_to_event_prediction only: name of the time/duration column (default: a column named \"time\")") + cmd.Flags().IntVar(&numberOfKeypoints, "number-of-keypoints", 0, + "keypoint_detection only: number of keypoints per sample (required; e.g. 17 for COCO pose)") + + cmd.Flags().BoolVar(&dryRun, "dry-run", false, + "validate + discover + walk, but don't create any cluster resources") + cmd.Flags().StringVar(&ingestorSAName, "ingestor-sa", "", + "override the ingestor ServiceAccount name (default: \"ingestor\"); "+ + "set this if you customized ingestionAuthz.serviceAccountName in the parent client chart") + cmd.Flags().StringVar(&stagePodImage, "stage-pod-image", "", + "override the ephemeral stage Pod's image (default: digest-pinned alpine 3.20 baked into the CLI). "+ + "Pin by digest in your override too — tag-only refs drift silently.") + + cmd.Flags().BoolVar(&detach, "detach", false, + "exit immediately after jobs-manager accepts the run (no log streaming, no summary panel). "+ + "Use for CI scenarios; reconnect later with `kubectl logs -f -n job/`.") + cmd.Flags().StringVar(&idempotencyKey, "idempotency-key", "", + "reuse this idempotency key across retry attempts (default: fresh per invocation). "+ + "jobs-manager treats a duplicate key as a replay and attaches to the existing Job "+ + "rather than spawning a new one — useful for at-most-once-across-attempts semantics.") + cmd.Flags().StringVar(&imageDigest, "image-digest", "", + "pin the ingestor container image to a specific digest (default: jobs-manager picks the "+ + "cluster-configured `images.ingestor.digest`). Format: sha256:.") + + return cmd +} + +// runDatasetPushArgs collects every parameter runDatasetPush needs, +// so the body stays testable without going through cobra. The cobra +// RunE wrapper above is the ONLY caller in production; tests +// construct one of these directly. +type runDatasetPushArgs struct { + LocalPath string + Kubeconfig string + Context string + Namespace string + Spec push.SpecArgs + TargetSizeFlag string // raw --target-size; resolved after Discover (image) + SchemaFlag string // raw --schema; resolved or inferred after Discover (tabular) + DryRun bool + IngestorSAName string + StagePodImage string + + // Phase 4 (#152) fields. See the flag declarations for the + // per-knob rationale; all three are optional. + Detach bool + IdempotencyKey string + ImageDigest string +} + +// runDatasetPush is the full Phase 3 implementation: pre-flight +// checks, then either --dry-run stop or stage Pod + tar stream + +// cleanup. Phase 4 (#152) will hook submit-to-jobs-manager after +// the staging step. +// +// Step order is "fail fast, fail local" — every step that doesn't +// need the cluster runs before any that does, so a customer with +// a bad label-column or oversized dataset gets the diagnostic in +// milliseconds without a kubeconfig round-trip. +func runDatasetPush(ctx context.Context, out, errOut io.Writer, a runDatasetPushArgs) error { + // 1. Validate the table name BEFORE anything else. It's both + // the MySQL identifier and the /data/shared/
/ PVC + // subdirectory — an unsanitized traversal name (../../etc) + // would escape that subtree once the stage Pod writes to + // it. The embedded schema only checks minLength on `table`, + // so this CLI-side guard is the real fix. SpecArgs.Build() + // below calls StagedPrefix, which panics on an unsafe name — + // so this check MUST come first. + if err := push.ValidateTableName(a.Spec.Table); err != nil { + return &exitError{code: 2, err: err} + } + + // 2. Category gate. Runs BEFORE schema validation so an + // unsupported category gets a clear, actionable CLI message + // rather than the schema's terse enum / missing-property error. + // Supported today: image_classification + the tabular / + // time-series family. The other image categories need sidecar + // (annotation/mask) staging the CLI doesn't do yet, and the + // text family needs a texts/sequences dir — both land in later + // increments. A typo'd category also lands here with a clear + // list rather than the schema's 11-option enum dump. + switch { + case a.Spec.Category == "": + // Left empty by a caller; let the schema produce the canonical + // "category is required" error downstream. + case push.IsTabular(a.Spec.Category) || push.IsText(a.Spec.Category) || + a.Spec.Category == "image_classification" || + a.Spec.Category == "object_detection" || + a.Spec.Category == "keypoint_detection": + // supported + case push.IsImage(a.Spec.Category): + // semantic_segmentation / instance_segmentation + return &exitError{code: 2, err: fmt.Errorf( + "category %q isn't supported by the CLI yet. semantic_segmentation is "+ + "blocked on the ingestor's mask-sidecar support (data-ingestors#136), and "+ + "instance_segmentation isn't implemented. Supported image categories: "+ + "image_classification, object_detection, keypoint_detection.", a.Spec.Category)} + default: + return &exitError{code: 2, err: fmt.Errorf( + "category %q isn't a recognized task category. Supported: image_classification, "+ + "object_detection, keypoint_detection, text_classification, "+ + "masked_language_modeling, tabular_classification, tabular_regression, "+ + "time_series_forecasting, time_to_event_prediction.", a.Spec.Category)} + } + + // 3. Walk the local directory FIRST (local "fail fast"), dispatched + // by category family. Image categories expect labels.csv + + // images/; tabular / time-series categories expect a single + // data CSV. The walk also yields what the per-category + // resolution below needs (the image list for target-size, the + // CSV for schema inference). + var ( + layout *push.LocalLayout + err error + ) + switch { + case push.IsTabular(a.Spec.Category): + layout, err = push.DiscoverTabular(a.LocalPath) + case push.IsText(a.Spec.Category): + layout, err = push.DiscoverText(a.Spec.Category, a.LocalPath) + case a.Spec.Category == "object_detection": + layout, err = push.DiscoverObjectDetection(a.LocalPath) + default: + // image_classification + keypoint_detection: labels.csv + images/. + layout, err = push.Discover(a.LocalPath) + } + if err != nil { + return &exitError{code: 3, err: err} + } + + // 3a. Per-category spec resolution from the local data, so the + // synthesized spec carries the right fields before validation. + switch { + case push.IsTabular(a.Spec.Category): + // Column schema: an explicit --schema wins; otherwise infer + // INT/FLOAT/VARCHAR types from the CSV so the customer doesn't + // hand-write one for the common case. + if a.SchemaFlag != "" { + sch, perr := push.ParseSchema(a.SchemaFlag) + if perr != nil { + return &exitError{code: 2, err: perr} + } + a.Spec.Schema = sch + } else { + sch, skipped, ierr := push.InferSchema(layout.LabelsCSV) + if ierr != nil { + return &exitError{code: 3, err: fmt.Errorf("inferring schema from CSV: %w", ierr)} + } + a.Spec.Schema = sch + _, _ = fmt.Fprintf(out, + "Inferred schema for %d column(s) from %s (override with --schema).\n", + len(sch), filepath.Base(layout.LabelsCSV)) + if len(skipped) > 0 { + _, _ = fmt.Fprintf(out, + " (skipped framework-managed column(s): %s)\n", strings.Join(skipped, ", ")) + } + } + case push.IsImage(a.Spec.Category): + // keypoint_detection needs --number-of-keypoints (dataset- + // specific, no default). Catch it here with an actionable + // message rather than letting the ingestor fail mid-run. + if a.Spec.Category == "keypoint_detection" && a.Spec.NumberOfKeypoints <= 0 { + return &exitError{code: 2, err: errors.New( + "keypoint_detection requires --number-of-keypoints (e.g. " + + "--number-of-keypoints 17); it's dataset-specific and has no default")} + } + // Image target resolution: the ingestor's image_classification + // default is 512x512 and it VALIDATES (it does not resize), so + // a mismatch hard-fails. Honour an explicit --target-size; + // otherwise auto-detect from the first image so the common + // "all my images are NxN" case just works. + if a.TargetSizeFlag != "" { + w, h, perr := push.ParseTargetSize(a.TargetSizeFlag) + if perr != nil { + return &exitError{code: 2, err: perr} + } + a.Spec.TargetSize = []int{w, h} + } else if len(layout.Images) > 0 { + if w, h, derr := push.DetectImageSize(layout.Images[0]); derr == nil { + a.Spec.TargetSize = []int{w, h} + _, _ = fmt.Fprintf(out, + "Auto-detected image target size %dx%d from %s (override with --target-size).\n", + w, h, filepath.Base(layout.Images[0])) + } else { + _, _ = fmt.Fprintf(errOut, + "Note: couldn't auto-detect image size (%v); using the ingestor "+ + "default. Pass --target-size WxH if ingestion reports a "+ + "resolution mismatch.\n", derr) + } + } + default: + // Text family: no extra per-category resolution. The label (for + // text_classification) comes straight from --label-column; + // masked_language_modeling needs neither a label nor a schema. + } + + // 4. Synthesize the spec from flags + validate against schema. + // Catches "bad category", "missing intent" etc. BEFORE we + // touch the cluster. The error formatter is the same one + // ingest validate uses, so a customer who YAML'd manually + // first sees identical wording. + spec := a.Spec.Build() + specBytes, err := yaml.Marshal(spec) + if err != nil { + return &exitError{code: 3, err: fmt.Errorf("marshaling synthesized spec: %w", err)} + } + v, err := schema.NewV1Validator() + if err != nil { + return &exitError{code: 3, err: fmt.Errorf("loading embedded schema: %w", err)} + } + _, errs, parseErr := v.ValidateYAML(specBytes) + if parseErr != nil { + // "Parse" failing on a spec we marshaled ourselves is a + // programming error, not a customer error — surface it + // with the bytes so we can diagnose. Exit 3 (the + // "internal" bucket) matches the marshal-failure branch + // above. + return &exitError{code: 3, err: fmt.Errorf("internal: re-parsing synthesized spec: %w\n%s", parseErr, specBytes)} + } + if len(errs) > 0 { + // Use the SAME formatter `ingest validate` uses, so the + // experience is identical whether the customer authored + // YAML by hand or via flags. Diagnostics go to stderr + // (matching ingest validate) so a downstream pipe of + // stdout (e.g. piping the summary to jq once that's a + // JSON output mode) isn't polluted by error text. Exit 2 + // is reserved for schema violations across the CLI. + _, _ = fmt.Fprintf(errOut, "synthesized spec failed schema validation (%d issue%s):\n", + len(errs), plural(len(errs))) + _, _ = fmt.Fprintln(errOut, schema.FormatErrors(errs)) + return &exitError{code: 2, err: errors.New("synthesized spec failed schema validation; check the flag values above")} + } + + // 5. Cluster discovery — same kubeconfig path as `cluster info`. + // Errors mirror that command's exit-code contract (3 for + // kubeconfig, 4 for missing release) so behaviour is + // consistent across pre-flight commands. + resolved, err := cluster.Load(cluster.KubeconfigOptions{ + Path: a.Kubeconfig, + Context: a.Context, + Namespace: a.Namespace, + }) + if err != nil { + return &exitError{code: 3, err: fmt.Errorf("loading kubeconfig: %w", err)} + } + cs, err := cluster.NewClientset(resolved) + if err != nil { + return &exitError{code: 3, err: err} + } + release, err := cluster.DiscoverParentRelease(ctx, cs, resolved.Namespace) + if err != nil { + return &exitError{code: 4, err: err} + } + if a.IngestorSAName != "" { + release.IngestorSAName = a.IngestorSAName + } + + // 6. PVC discovery — confirms the chart's shared-data PVC is + // Bound before we waste time provisioning a Pod that can't + // mount it. + pvc, err := cluster.DiscoverSharedPVC(ctx, cs, resolved.Namespace) + if err != nil { + return &exitError{code: 4, err: err} + } + + // 7. Print the pre-flight summary. The output is the same in + // dry-run and live mode — only the "what happens next" line + // differs. Customers iterating on a bad layout see this + // every attempt, so it's worth keeping skimmable: one fact + // per line, aligned by column. + printPushPreflight(out, layout, release, pvc, spec, a.DryRun) + + // 8. Dry-run stop. Acknowledged success. + if a.DryRun { + _, _ = fmt.Fprintln(out, "Dry-run complete — no cluster resources were created.") + return nil + } + + // 9. Stage the files: create ephemeral Pod → wait Ready → tar + // stream → cleanup. The deferred cleanup inside push.Stage + // runs on success and failure (including ctx cancellation + // from a SIGINT handler), so no orphan Pod is left behind. + // + // Exit code 7 ("staging failed") is distinct from the + // pre-flight codes so customers can branch on whether the + // failure was their environment vs the actual data transfer. + progress := push.NewProgress(out, layout.TotalBytes, + fmt.Sprintf("Staging %s", a.Spec.Table)) + // Defer Finish so a failure path that returns BEFORE + // StreamLayout (e.g. CreateStagePod fails on PSA rejection, + // WaitForStagePodReady times out) still clears the TTY + // progress UI. push.StreamLayout's own deferred Finish would + // otherwise be unreachable. Calling Finish twice on the same + // schollz bar is a no-op, so the double-call on the happy + // path is safe. Bugbot flagged on PR-b round 5. + defer progress.Finish() + stageErr := push.Stage(ctx, push.StageOptions{ + Client: cs, + Executor: &push.SPDYExecutor{ + Config: resolved.RestConfig, + Client: cs, + }, + Namespace: resolved.Namespace, + IngestorSAName: release.IngestorSAName, + PVCClaimName: pvc.ClaimName, + PVCMountPath: pvc.MountPath, + Layout: layout, + Table: a.Spec.Table, + StagePodImage: a.StagePodImage, + Progress: progress, + Out: out, + }) + if stageErr != nil { + return &exitError{code: 7, err: stageErr} + } + + // 10. Mint the SA token Phase 4 uses to authenticate the POST + // to jobs-manager. Expiry is 1 hour (vs cluster info's 10 + // min) because the full Phase 4 lifecycle — submit + watch + // + log stream — can run that long for large ingestions. + // The chart's helm flow uses the same token-mint code path. + _, _ = fmt.Fprintln(out) + tok, err := cluster.MintIngestorToken(ctx, cs, resolved.Namespace, + release.IngestorSAName, 3600, nil) + if err != nil { + return &exitError{code: 5, err: err} + } + + // 11. Open a port-forward to a Pod backing the jobs-manager + // Service. The CLI runs off-cluster (on a laptop, in CI + // runners outside the cluster network), so the discovered + // *.svc.cluster.local URL isn't reachable — we tunnel + // through the kubeconfig-authenticated apiserver, same as + // `kubectl port-forward`. Bugbot PR #10 r3 caught the + // original broken-by-design direct-URL POST. + _, _ = fmt.Fprintln(out, "Opening port-forward to jobs-manager...") + pf, err := submit.PortForwardJobsManager(ctx, cs, resolved.RestConfig, + resolved.Namespace, release.JobsManagerServiceName, release.JobsManagerPort) + if err != nil { + return &exitError{code: 8, err: fmt.Errorf("setting up jobs-manager port-forward: %w", err)} + } + defer pf.Close() + + // 12. Phase 4: POST to jobs-manager via the local port, + // watch the spawned ingestor Job, render the parsed + // INGESTION SUMMARY panel. + // + // Exit-code mapping: + // SubmitError 401/403 → 5 (auth — same bucket as + // token-mint, shared + // "your SA can't do this" + // diagnostic class) + // SubmitError other 4xx/5xx → 8 (submit failed) + // WatchResult Failed → 9 (ingest failed) + // WatchResult Succeeded + + // summary.HasFailures() → 9 (some rows failed + // even though Job exited 0; + // the ingestor surfaces + // partial-failure summaries) + // WatchResult Detached → 0 (cluster keeps running) + // WatchResult Succeeded clean → 0 + localEndpoint := fmt.Sprintf("http://localhost:%d", pf.LocalPort) + submitRes, err := submit.Run(ctx, submit.Options{ + Submitter: submit.NewHTTPSubmitter(localEndpoint, tok.Token), + Client: cs, + IngestConfigYAML: string(specBytes), + IdempotencyKey: a.IdempotencyKey, + ImageDigest: a.ImageDigest, + Detach: a.Detach, + Out: out, + }) + if err != nil { + switch { + case submit.IsAuthError(err): + return &exitError{code: 5, err: err} + case submit.IsWatchError(err): + // Watch-phase failure: jobs-manager already accepted + // the run, the cluster is doing the work, the CLI + // just couldn't follow along. Exit 9 (ingest-side) + // not 8 (submit-side). Bugbot flagged the + // previously-undifferentiated mapping on PR #10. + return &exitError{code: 9, err: err} + default: + return &exitError{code: 8, err: err} + } + } + + // Detach paths (--detach flag OR SIGINT-mid-watch) are + // success — cluster keeps running; the orchestrator already + // printed the reconnect hint. + if submitRes.Watch == nil || submitRes.Watch.Outcome == submit.JobOutcomeDetached { + return nil + } + + // Watch outcomes. Both Failed and Unknown route to exit 9 + // (Unknown = finalJobStatus timed out without seeing a + // terminal condition, which we can't claim as success). + // Bugbot flagged the prior switch's missing Unknown branch + // on PR #10. + switch submitRes.Watch.Outcome { + case submit.JobOutcomeFailed: + return &exitError{code: 9, err: errors.New("ingestion Job exited non-zero — see logs above")} + case submit.JobOutcomeUnknown: + return &exitError{code: 9, err: errors.New( + "ingestion Job's final status couldn't be determined within the watch window — " + + "check `kubectl get job -n " + submitRes.Submit.Namespace + " " + submitRes.Submit.JobName + "` for the outcome")} + case submit.JobOutcomeSucceeded: + if submitRes.Watch.Summary != nil && submitRes.Watch.Summary.HasFailures() { + return &exitError{code: 9, err: errors.New( + "ingestion Job completed but the summary reports failures — see panel above")} + } + } + return nil +} + +// printPushPreflight is the customer-facing summary. Mirrors +// `cluster info`'s layout for consistency: section header, +// indented key:value rows. Kept here (not on the layout/release/pvc +// types) because the formatting is policy and lives with the CLI, +// not the data. +func printPushPreflight( + out io.Writer, + layout *push.LocalLayout, + release *cluster.ParentRelease, + pvc *cluster.SharedPVC, + spec map[string]any, + dryRun bool, +) { + // Explicit-discard the writer errors throughout — same rationale + // as cli/cluster.go and cli/ingest.go: a pipe-write failure + // shouldn't convert success into failure. The exit code is + // the contract. + cat, _ := spec["category"].(string) + + _, _ = fmt.Fprintf(out, "Local dataset:\n") + _, _ = fmt.Fprintf(out, " root: %s\n", layout.Root) + switch { + case push.IsTabular(cat): + _, _ = fmt.Fprintf(out, " data CSV: %s\n", layout.LabelsCSV) + if sch, ok := spec["schema"].(map[string]string); ok { + _, _ = fmt.Fprintf(out, " columns: %d\n", len(sch)) + } + case push.IsText(cat): + dir := push.TextSidecarDir(cat) + _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) + _, _ = fmt.Fprintf(out, " %-15s%d files\n", dir+":", len(layout.Sidecars[dir])) + if _, ok := layout.ExtraFiles["tokenizer.json"]; ok { + _, _ = fmt.Fprintf(out, " %-15s%s\n", "tokenizer:", "tokenizer.json") + } + default: + _, _ = fmt.Fprintf(out, " labels.csv: %s\n", layout.LabelsCSV) + _, _ = fmt.Fprintf(out, " images: %d files\n", len(layout.Images)) + if anns := layout.Sidecars["annotations"]; len(anns) > 0 { + _, _ = fmt.Fprintf(out, " annotations: %d files\n", len(anns)) + } + } + _, _ = fmt.Fprintf(out, " total size: %s\n", push.HumanBytes(layout.TotalBytes)) + _, _ = fmt.Fprintln(out) + + _, _ = fmt.Fprintf(out, "Target cluster:\n") + _, _ = fmt.Fprintf(out, " release: %s (chart %s)\n", release.ReleaseName, release.ChartVersion) + _, _ = fmt.Fprintf(out, " jobs-manager: %s\n", release.JobsManagerService) + _, _ = fmt.Fprintf(out, " shared PVC: %s (%s)\n", pvc.ClaimName, pvc.Phase) + if !pvc.IsReadWriteMany() { + // Warn but don't block — RWO clusters still work, the + // scheduler will co-locate the stage Pod with the existing + // mounter. Phase 3 PR-b will surface the same warning at + // pod-create time too. + _, _ = fmt.Fprintf(out, " access: %v (warn: not ReadWriteMany — stage Pod will co-locate)\n", pvc.AccessModes) + } + _, _ = fmt.Fprintln(out) + + _, _ = fmt.Fprintf(out, "Synthesized ingest spec:\n") + _, _ = fmt.Fprintf(out, " table: %s\n", spec["table"]) + _, _ = fmt.Fprintf(out, " category: %s\n", spec["category"]) + _, _ = fmt.Fprintf(out, " intent: %s\n", spec["intent"]) + switch lbl := spec["label"].(type) { + case string: + _, _ = fmt.Fprintf(out, " label column: %s\n", lbl) + case map[string]any: + _, _ = fmt.Fprintf(out, " label column: %v (policy: %v)\n", lbl["column"], lbl["policy"]) + } + if tc, ok := spec["time_column"].(string); ok && tc != "" { + _, _ = fmt.Fprintf(out, " time column: %s\n", tc) + } + _, _ = fmt.Fprintf(out, " destination: %s\n", push.FinalDestPrefix(spec["table"].(string))) + _, _ = fmt.Fprintln(out) + + if !dryRun { + _, _ = fmt.Fprintf(out, "Next: stage %d files (%s) for table %q\n", + layout.FileCount(), push.HumanBytes(layout.TotalBytes), spec["table"]) + _, _ = fmt.Fprintln(out) + } +} diff --git a/internal/cli/dataset_test.go b/internal/cli/dataset_test.go new file mode 100644 index 0000000..5d5cfcf --- /dev/null +++ b/internal/cli/dataset_test.go @@ -0,0 +1,243 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// imgcLayout drops a minimum-viable image_classification directory +// under t.TempDir() and returns its path. Mirrors push.imgcDir +// (tests can't import test helpers across packages, so we duplicate +// the few lines). +func imgcLayout(t *testing.T) string { + t.Helper() + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "001.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image: %v", err) + } + return root +} + +// execDatasetPush drives the full cobra dispatch for the push +// command and returns the exit code + captured stdout/stderr. +// Mirrors the execIngestValidate helper from ingest_test.go — same +// rationale about not sharing *cobra.Command across cases (cobra +// holds flag state on the command tree). +// +// kubeconfigPath is required because every push invocation tries +// kubeconfig load before any cluster work; tests that want to +// stop EARLIER (at schema validation or layout walk) still need a +// kubeconfig path that resolves predictably. We feed in a path +// that's guaranteed to fail os.Stat so the kubeconfig branch +// errors out consistently when reached — and tests assert on the +// EARLIER stage's exit code, which fires before kubeconfig. +func execDatasetPush(t *testing.T, args []string) (exitCode int, stdout, stderr string) { + t.Helper() + root := NewRootCmd(BuildInfo{Version: "test"}) + var so, se bytes.Buffer + root.SetOut(&so) + root.SetErr(&se) + + // Always inject a guaranteed-bad kubeconfig path so tests that + // "fall through" the local pre-checks into kubeconfig load + // get a deterministic exit 3 (not a flaky "depends on whether + // you have a real kubeconfig" outcome). + cmdArgs := append([]string{"dataset", "push", + "--kubeconfig=/tmp/tracebloc-cli-test-nonexistent-" + t.Name()}, + args...) + root.SetArgs(cmdArgs) + + err := root.Execute() + return ExitCodeFromError(err), so.String(), se.String() +} + +// TestDatasetPush_UnsupportedCategory_ExitsTwo: the CLI-side category +// gate runs before schema validation so a customer who passes a +// not-yet-supported category gets an actionable message (exit 2) +// rather than the schema's confusing missing-property error. Today's +// supported set is image_classification + the tabular / time-series +// family; the other image categories (which need annotation/mask +// sidecar staging), the text family, and nonsense values are gated +// out here. Bugbot review-on-self caught the missing gate on PR-a. +func TestDatasetPush_UnsupportedCategory_ExitsTwo(t *testing.T) { + root := imgcLayout(t) + for _, badCategory := range []string{ + "semantic_segmentation", // blocked on the ingestor (data-ingestors#136) + "instance_segmentation", // not implemented + "definitely-not-a-category", // nonsense; gate catches this too + } { + t.Run(badCategory, func(t *testing.T) { + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=" + badCategory, + "--intent=train", + "--label-column=label", + }) + if code != 2 { + t.Fatalf("expected exit 2 for unsupported category %q, got %d", badCategory, code) + } + }) + } +} + +// TestDatasetPush_TraversalTableName_ExitsTwo is the security +// regression pin at the CLI layer. --table=../../etc must be +// rejected with exit 2 BEFORE any spec synthesis or cluster work — +// the table name flows into the /data/shared/
/ PVC path, +// and a traversal value would let PR-b's stage Pod escape that +// subtree. Bugbot flagged this on PR #8 commit 4240097. +func TestDatasetPush_TraversalTableName_ExitsTwo(t *testing.T) { + root := imgcLayout(t) + for _, bad := range []string{"../../etc", "../foo", "foo/bar"} { + t.Run(bad, func(t *testing.T) { + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=" + bad, + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 2 { + t.Fatalf("expected exit 2 for traversal table name %q, got %d", bad, code) + } + }) + } +} + +// TestDatasetPush_MissingIntent_ExitsTwo: pins the "intent is +// required" diagnostic path — different schema violation but the +// same exit-code class. +func TestDatasetPush_MissingIntent_ExitsTwo(t *testing.T) { + root := imgcLayout(t) + code, _, stderr := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=image_classification", + // intent omitted + "--label-column=label", + }) + if code != 2 { + t.Fatalf("expected exit 2 for missing intent, got %d", code) + } + if !strings.Contains(stderr, "intent") { + t.Errorf("expected stderr to mention 'intent', got:\n%s", stderr) + } +} + +// TestDatasetPush_NonexistentLocalPath_ExitsThree: the layout walk +// runs AFTER schema validation, so an invalid local path with +// otherwise-valid flags surfaces at the walk stage with exit 3 +// (the "local input or kubeconfig" code). +// +// We only assert on the exit code, not the error text — cobra's +// SilenceErrors=true means root.Execute() doesn't surface the +// returned error to stderr (main.go does that). Mirrors +// ingest_test.go's TestIngestValidate_UnreadableFileExitsThree +// pattern; the error-content surface is exercised at the package +// level (internal/push.Discover's own tests). +func TestDatasetPush_NonexistentLocalPath_ExitsThree(t *testing.T) { + code, _, _ := execDatasetPush(t, []string{ + "/tmp/tracebloc-cli-test-no-such-dir-" + t.Name(), + "--table=t1", + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 3 { + t.Fatalf("expected exit 3 for missing local path, got %d", code) + } +} + +// TestDatasetPush_MissingLabelsCSV_ExitsThree: most likely "real +// world" wrong-layout case — customer has images but forgot +// labels.csv. Pins the exit-code contract for the common failure +// mode; the diagnostic-text content is covered by +// internal/push.TestDiscover_MissingLabelsCSV. +func TestDatasetPush_MissingLabelsCSV_ExitsThree(t *testing.T) { + root := t.TempDir() + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write img: %v", err) + } + + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 3 { + t.Fatalf("expected exit 3 for missing labels.csv, got %d", code) + } +} + +// TestDatasetPush_BadKubeconfig_ExitsThree: schema + layout both +// pass; kubeconfig load fails because the injected path doesn't +// exist. The exit-code contract matches `cluster info`'s — same +// class of failure (3 = local input problem) surfaces with the +// same code regardless of which command tripped it. +func TestDatasetPush_BadKubeconfig_ExitsThree(t *testing.T) { + root := imgcLayout(t) + code, _, _ := execDatasetPush(t, []string{ + root, + "--table=t1", + "--category=image_classification", + "--intent=train", + "--label-column=label", + }) + if code != 3 { + t.Fatalf("expected exit 3 for bad kubeconfig, got %d", code) + } +} + +// TestDatasetPush_RequiresExactlyOneArg: cobra-level Args check +// pins the command signature. Two positional args, or zero, should +// fail before the runner even fires. +func TestDatasetPush_RequiresExactlyOneArg(t *testing.T) { + cases := []struct { + name string + args []string + }{ + { + name: "no positional", + args: []string{ + "--table=t1", "--category=image_classification", + "--intent=train", "--label-column=label", + }, + }, + { + name: "two positionals", + args: []string{ + "./a", "./b", + "--table=t1", "--category=image_classification", + "--intent=train", "--label-column=label", + }, + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + code, _, _ := execDatasetPush(t, c.args) + if code == 0 { + t.Errorf("expected non-zero exit for %s, got 0", c.name) + } + }) + } +} diff --git a/internal/cli/exit.go b/internal/cli/exit.go new file mode 100644 index 0000000..bdbe10a --- /dev/null +++ b/internal/cli/exit.go @@ -0,0 +1,68 @@ +package cli + +// ExitCodeFromError extracts an exit code from a handler-returned +// error. Handlers that need a specific exit code wrap their return +// in an *exitError (see ingest.go); everything else defaults to 1. +// +// Public-but-package-keyed: main.go is the only intended caller, +// and the exitError type itself stays unexported so subcommand +// handlers go through the constructor. +func ExitCodeFromError(err error) int { + if err == nil { + return 0 + } + var ee *exitError + if asExitError(err, &ee) { + return ee.code + } + return 1 +} + +// IsSilentError reports whether a handler-returned error wants +// main() to suppress its own "Error: ..." stderr line on the way +// out. The contract: a handler that has already printed a +// structured diagnostic itself (e.g. the schema-validate path +// prints per-violation lines to stderr) returns +// `&exitError{code: N, err: nil}` to signal "exit non-zero but +// don't print anything more." Errors with a non-nil inner err +// (file-read failures, parse errors, schema-compile bugs) are +// NOT silent — main() prints them so the customer doesn't see a +// bare non-zero exit with no explanation. +// +// Caller pattern in main.go: +// +// if err != nil && !cli.IsSilentError(err) { +// fmt.Fprintln(os.Stderr, "Error:", err) +// } +// os.Exit(cli.ExitCodeFromError(err)) +func IsSilentError(err error) bool { + if err == nil { + return false + } + var ee *exitError + if asExitError(err, &ee) { + return ee.err == nil + } + return false +} + +// asExitError walks the wrapped-error chain looking for an +// *exitError. Same pattern as errors.As but with a typed target so +// callers don't have to import errors at every site. +func asExitError(err error, target **exitError) bool { + for cur := err; cur != nil; cur = unwrapError(cur) { + if ee, ok := cur.(*exitError); ok { + *target = ee + return true + } + } + return false +} + +func unwrapError(err error) error { + type unwrapper interface{ Unwrap() error } + if u, ok := err.(unwrapper); ok { + return u.Unwrap() + } + return nil +} diff --git a/internal/cli/ingest.go b/internal/cli/ingest.go new file mode 100644 index 0000000..b49d692 --- /dev/null +++ b/internal/cli/ingest.go @@ -0,0 +1,143 @@ +package cli + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/tracebloc/cli/internal/schema" +) + +// newIngestCmd implements the `tracebloc ingest` subtree. Today it +// has only one verb — `validate` — which runs the schema check +// locally without touching the cluster. Future verbs (status, retry, +// cancel) hang off this same parent in later phases. +func newIngestCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "ingest", + Short: "Inspect and manage ingestion configurations", + Long: `Commands for working with ingest.yaml files and ingestion runs. + +Today only ` + "`validate`" + ` is implemented — it runs the same schema check +the cluster's jobs-manager runs, but locally and instantly. Use it to +catch typos and missing fields before submitting an ingestion. + +Future verbs (status, retry, cancel) will land alongside the +push/list/show commands in later phases.`, + } + + cmd.AddCommand(newIngestValidateCmd()) + return cmd +} + +// newIngestValidateCmd implements `tracebloc ingest validate `. +// Reads a YAML file from disk, validates it against the embedded v1 +// schema, prints any violations in the same format the Python +// implementation uses, exits non-zero on any violation. +// +// The output format is deliberately matched to +// tracebloc_ingestor.cli.run._format_errors so a customer's editor +// or CI can grep both implementations' output uniformly. See +// internal/schema/validate.go for the formatting contract. +func newIngestValidateCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "validate ", + Short: "Validate an ingest.yaml against the embedded v1 schema, locally", + Long: `Reads , parses it as YAML, and validates it against the bundled +ingest.v1.json schema (synced from tracebloc/data-ingestors). Prints +violations in the same JSON-pointer-prefixed format the cluster's +jobs-manager uses, and exits non-zero if any are found. + +Useful as a pre-flight before ` + "`tracebloc dataset push`" + ` lands in a +future phase; for now, customers running the Helm chart can validate +their ` + "`ingest.yaml`" + ` before invoking ` + "`helm install`" + `, getting +millisecond local feedback instead of a multi-second cluster round +trip. + +Exit codes: + 0 YAML parses and validates cleanly + 2 YAML parses but has schema violations (printed to stderr) + 3 YAML doesn't parse or file isn't readable`, + Args: cobra.ExactArgs(1), + RunE: runIngestValidate, + } + return cmd +} + +func runIngestValidate(cmd *cobra.Command, args []string) error { + path := args[0] + + body, err := os.ReadFile(path) + if err != nil { + // fileError is exit-code 3 territory. We use a sentinel + // exit-coded error so cobra propagates the right code via + // the main()-side os.Exit mapping (in a follow-up commit + // we'll wire main.go to inspect for these). + return &exitError{code: 3, err: fmt.Errorf("reading %s: %w", path, err)} + } + + v, err := schema.NewV1Validator() + if err != nil { + // Schema-compilation failures are infrastructure-side, not + // customer-side — we bundle the schema, so this only fires + // if the build is broken. Treat as exit-code-2 so CI can + // distinguish from a customer file problem. + return &exitError{code: 2, err: fmt.Errorf("loading embedded schema: %w", err)} + } + + _, violations, parseErr := v.ValidateYAML(body) + if parseErr != nil { + return &exitError{code: 3, err: fmt.Errorf("%s: %w", path, parseErr)} + } + + if len(violations) == 0 { + // Explicit discard: Fprintf returns an error when the + // underlying writer fails (closed pipe, etc.). For the + // success summary we'd rather still exit 0 even if the + // downstream consumer dropped the connection — they got + // what they needed (the exit code), and propagating a + // pipe-write error would convert success into failure for + // reasons unrelated to validation. + _, _ = fmt.Fprintf(cmd.OutOrStdout(), "%s: ok\n", path) + return nil + } + + // Print violations to stderr so success can be piped without + // interference; error lines are diagnostic, not data. Same + // pipe-write rationale as the ok-path above: don't let a + // stderr-write failure mask the real exit-2 schema-violation + // signal. + _, _ = fmt.Fprintf(cmd.ErrOrStderr(), "%s: schema validation failed (%d issue%s)\n", + path, len(violations), plural(len(violations))) + _, _ = fmt.Fprintln(cmd.ErrOrStderr(), schema.FormatErrors(violations)) + return &exitError{code: 2, err: nil} // err==nil so cobra doesn't print "Error: ..." on top +} + +func plural(n int) string { + if n == 1 { + return "" + } + return "s" +} + +// exitError carries a process exit code alongside (or instead of) +// an error message. main.go inspects for this type before calling +// os.Exit, mapping the code through. Other handlers can opt in to +// specific exit codes by returning an exitError. +type exitError struct { + code int + err error +} + +func (e *exitError) Error() string { + if e.err == nil { + return fmt.Sprintf("exit %d", e.code) + } + return e.err.Error() +} + +func (e *exitError) Unwrap() error { return e.err } + +// Code returns the process exit code main() should propagate. +func (e *exitError) Code() int { return e.code } diff --git a/internal/cli/ingest_test.go b/internal/cli/ingest_test.go new file mode 100644 index 0000000..4e7dd08 --- /dev/null +++ b/internal/cli/ingest_test.go @@ -0,0 +1,190 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" +) + +// writeTmpYAML drops a small YAML doc into the test's t.TempDir and +// returns the path. Using TempDir guarantees cleanup on test exit; +// callers don't have to defer os.Remove themselves. +func writeTmpYAML(t *testing.T, content string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "ingest.yaml") + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("WriteFile: %v", err) + } + return path +} + +// execIngestValidate drives the full cobra dispatch for +// `tracebloc ingest validate ` and returns the exit code, the +// captured stdout, and the captured stderr. Tests should never share +// a *cobra.Command across cases — cobra holds flag state on the +// command tree and stale trees leak one test's args into the next. +func execIngestValidate(t *testing.T, path string) (exitCode int, stdout, stderr string) { + t.Helper() + root := NewRootCmd(BuildInfo{Version: "test"}) + var so, se bytes.Buffer + root.SetOut(&so) + root.SetErr(&se) + root.SetArgs([]string{"ingest", "validate", path}) + + err := root.Execute() + return ExitCodeFromError(err), so.String(), se.String() +} + +func TestIngestValidate_HappyPath(t *testing.T) { + path := writeTmpYAML(t, ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: cats_dogs_train +intent: train +category: image_classification +csv: /data/labels.csv +images: /data/images/ +label: image_label +`) + + code, stdout, stderr := execIngestValidate(t, path) + if code != 0 { + t.Fatalf("expected exit 0, got %d\nstderr:\n%s", code, stderr) + } + if !strings.Contains(stdout, "ok") { + t.Errorf("expected 'ok' on stdout, got: %q", stdout) + } + if stderr != "" { + t.Errorf("expected empty stderr on success, got: %q", stderr) + } +} + +func TestIngestValidate_SchemaFailureExitsTwo(t *testing.T) { + // Missing required fields → schema violation → exit 2. + path := writeTmpYAML(t, ` +kind: IngestConfig +category: image_classification +table: t +csv: /data/labels.csv +images: /data/images/ +label: image_label +`) + + code, _, stderr := execIngestValidate(t, path) + if code != 2 { + t.Fatalf("expected exit 2 for schema failure, got %d", code) + } + // Errors print to stderr (so success can be piped without + // interference). Both the count line + the per-error lines + // should be there. + for _, want := range []string{"schema validation failed", "apiVersion"} { + if !strings.Contains(stderr, want) { + t.Errorf("expected stderr to mention %q, got:\n%s", want, stderr) + } + } +} + +func TestIngestValidate_UnreadableFileExitsThree(t *testing.T) { + // Distinct exit code (3) for file-level problems (missing, + // permission-denied, etc.) — separates from schema violations + // (2) so callers can branch on the cause. + code, _, _ := execIngestValidate(t, "/tmp/definitely-does-not-exist-"+t.Name()) + if code != 3 { + t.Errorf("expected exit 3 for missing file, got %d", code) + } +} + +func TestIngestValidate_NonMappingExitsThree(t *testing.T) { + // A top-level YAML sequence (vs mapping) is a parse-shape + // problem, not a schema problem — surface it as the file-level + // exit code so the customer knows their file isn't an + // ingest-config at all, vs being one with the wrong fields. + path := writeTmpYAML(t, "- one\n- two\n") + + code, _, _ := execIngestValidate(t, path) + if code != 3 { + t.Errorf("expected exit 3 for non-mapping YAML, got %d", code) + } +} + +func TestIngestValidate_RequiresExactlyOneArg(t *testing.T) { + // Cobra catches arg-count violations before our RunE runs; + // confirm we exit non-zero (the specific code is cobra's + // default 1, not our exitError 2/3 — that's intentional). + root := NewRootCmd(BuildInfo{Version: "test"}) + var so, se bytes.Buffer + root.SetOut(&so) + root.SetErr(&se) + root.SetArgs([]string{"ingest", "validate"}) // no path + + err := root.Execute() + if err == nil { + t.Fatalf("expected error from missing path arg, got nil") + } + if got := ExitCodeFromError(err); got == 0 { + t.Errorf("expected non-zero exit, got %d", got) + } +} + +func TestExitCodeFromError(t *testing.T) { + // Defensive: pin the exitError dispatch behavior since this is + // the only thing main() depends on from this package. + if got := ExitCodeFromError(nil); got != 0 { + t.Errorf("nil err should map to 0, got %d", got) + } + if got := ExitCodeFromError(&exitError{code: 7}); got != 7 { + t.Errorf("explicit exit code should propagate, got %d", got) + } + // A wrapped exitError still reports its code (asExitError walks + // the unwrap chain). + wrapped := &exitError{code: 9, err: &exitError{code: 0}} + if got := ExitCodeFromError(wrapped); got != 9 { + t.Errorf("outermost exitError wins, got %d", got) + } +} + +// IsSilentError is the main()-side hook that decides whether to +// print an "Error: ..." stderr line on the way out. Pin the +// contract so main.go doesn't silently regress to swallowing +// errors (the high-severity bugbot finding that led to this +// being added in the first place). +func TestIsSilentError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil error", nil, false}, + { + "exitError with nil inner (e.g. schema-violation already-printed)", + &exitError{code: 2, err: nil}, + true, + }, + { + "exitError with non-nil inner (e.g. file read failure)", + &exitError{code: 3, err: io_eof_or_similar()}, + false, + }, + {"plain error from cobra (e.g. unknown command)", errorString("unknown command"), false}, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsSilentError(c.err); got != c.want { + t.Errorf("IsSilentError(%v) = %v, want %v", c.err, got, c.want) + } + }) + } +} + +// errorString is the simplest possible error implementation, used +// in tests that need a "plain" error without any custom Unwrap +// behavior. Same as the stdlib errors.New() result; defined inline +// to keep the test file self-contained. +type errorString string + +func (e errorString) Error() string { return string(e) } + +func io_eof_or_similar() error { return errorString("read: file does not exist") } diff --git a/internal/cli/root.go b/internal/cli/root.go index 8ce9ac3..d7d9b47 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -50,9 +50,12 @@ on the cluster's shared PVC, submitting the ingestion request, watching the resulting Job, and reporting the outcome. Customers never touch Helm, never edit YAML, never run kubectl cp manually. -Today this binary implements only ` + "`version`" + ` and ` + "`completion`" + ` — -see https://github.com/tracebloc/client/issues/147 for the v0.1 -roadmap. Subsequent phases land subcommands incrementally.`, +This binary implements the full v0.1 ingestion path: ` + "`dataset push`" + ` +(the dominant workflow above), ` + "`ingest validate`" + ` for a local +schema check, ` + "`cluster info`" + ` for discovery diagnostics, plus +` + "`version`" + ` and ` + "`completion`" + `. See +https://github.com/tracebloc/client/issues/147 for the v0.1 roadmap and +what's planned next.`, // Silence cobra's auto-printed errors + usage on every error; // we already print structured errors in handlers, and the @@ -64,6 +67,9 @@ roadmap. Subsequent phases land subcommands incrementally.`, // Subcommands. New phases append here. root.AddCommand(newVersionCmd(info)) + root.AddCommand(newIngestCmd()) + root.AddCommand(newClusterCmd()) + root.AddCommand(newDatasetCmd()) return root } diff --git a/internal/cluster/discover.go b/internal/cluster/discover.go new file mode 100644 index 0000000..b1718f9 --- /dev/null +++ b/internal/cluster/discover.go @@ -0,0 +1,225 @@ +package cluster + +import ( + "context" + "fmt" + "strings" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// ParentRelease describes the tracebloc parent client chart release +// discovered in the customer's cluster. The information comes from +// the helm-managed Deployment's labels (and a few of its env vars), +// not from the helm release secret — parsing those secrets requires +// either pulling in helm.sh/helm/v3 (massive dep) or re-implementing +// the gzip+base64+JSON unwrap. The chart's Deployment labels carry +// everything we actually need, so we use those instead. +type ParentRelease struct { + // ReleaseName is the helm release name. The chart's + // jobs-manager Deployment is named "-jobs-manager" by + // convention and labelled `app.kubernetes.io/instance=`. + ReleaseName string + + // ChartVersion is the parent chart's version, e.g. "1.3.5". + // Comes from the `helm.sh/chart=client-1.3.5` label. + ChartVersion string + + // AppVersion mirrors Chart.yaml's appVersion. Useful for + // detecting whether the customer is on a chart that supports + // the declarative ingestor flow at all. + AppVersion string + + // JobsManagerService is the in-cluster DNS name of the + // jobs-manager Service, e.g. + // "-jobs-manager..svc.cluster.local:8080". + // Used as the POST target for ingestion submissions WHEN + // the CLI runs in-cluster (e.g. CI inside the same cluster). + // For laptop / off-cluster use, the orchestrator port-forwards + // to JobsManagerServiceName + JobsManagerPort instead. + JobsManagerService string + + // JobsManagerServiceName + JobsManagerPort are the bare Service + // reference for off-cluster port-forwarding (Bugbot PR #10 r3). + // The FQDN-based JobsManagerService URL above doesn't resolve + // from a laptop; the port-forward path uses these to set up a + // localhost tunnel via the kubeconfig API server. + JobsManagerServiceName string + JobsManagerPort int + + // IngestorSAName is the name of the ServiceAccount the chart's + // hook pods run as. Today this is always the chart's default + // "ingestor". Customers who set `ingestionAuthz.serviceAccountName` + // to a non-default name in the parent client chart need to + // override via `tracebloc cluster info --ingestor-sa=` + // (and similarly for the future `dataset push` command). Reading + // the name from the cluster's ingestionAuthz ConfigMap so this + // flag becomes unnecessary is a v0.2 follow-up (see #7). + IngestorSAName string + + // IngestorImageDigest is the canonical digest the cluster's + // chart will spawn ingestor Jobs with. Comes from + // `INGESTOR_IMAGE_DIGEST` env on jobs-manager. Empty when the + // admin hasn't set images.ingestor.digest yet. + IngestorImageDigest string +} + +// DiscoverParentRelease finds the tracebloc parent client chart +// release in the given namespace by looking for a Deployment with +// the chart's hallmark labels. Returns a friendly error if none +// found, or if multiple candidates exist (so customers can pick). +// +// Why labels on a Deployment, not Helm's release secrets: +// +// - The release-secret path needs gzip + base64 + JSON parsing +// (helm v3's storage format), or pulls in helm.sh/helm/v3 as +// a dep (which transitively pulls in ~80MB of Go modules, +// including kube-runtime). +// - The Deployment is what the chart actually deploys; if it's +// not there, the chart isn't installed (or the install +// failed). Using its labels for discovery means "the +// discovered release is the running release" by construction. +// - The chart's labels are part of its public contract via +// `_helpers.tpl`; they're stable across minor versions. +// +// The chart's labels share `app.kubernetes.io/name=client` across +// EVERY resource it creates (mysql-client, jobs-manager, +// requests-proxy, etc.) — that's the helm convention where the +// chart's name is the label, not the component's. To distinguish +// jobs-manager from its siblings we filter by Deployment name +// suffix matching the chart's naming convention +// "-jobs-manager" (or plain "jobs-manager" for unprefixed +// installs). +func DiscoverParentRelease(ctx context.Context, cs kubernetes.Interface, namespace string) (*ParentRelease, error) { + deps, err := cs.AppsV1().Deployments(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "app.kubernetes.io/name=client,app.kubernetes.io/managed-by=Helm", + }) + if err != nil { + return nil, fmt.Errorf("listing chart-managed deployments in namespace %s: %w", namespace, err) + } + + // Filter to just the jobs-manager Deployment(s). The chart pins + // the name to either "-jobs-manager" or "jobs-manager" + // depending on chart version + values; either way it ends in + // "-jobs-manager" or equals "jobs-manager". + var jmDeps []appsv1.Deployment + for _, d := range deps.Items { + if d.Name == "jobs-manager" || strings.HasSuffix(d.Name, "-jobs-manager") { + jmDeps = append(jmDeps, d) + } + } + + switch len(jmDeps) { + case 0: + return nil, fmt.Errorf( + "no tracebloc parent client release found in namespace %q "+ + "(no chart-managed Deployment named *-jobs-manager). "+ + "Install with `helm install tracebloc/client --namespace %s` first, "+ + "or pass --namespace to point at the namespace where it's running.", + namespace, namespace, + ) + case 1: + // happy path + default: + names := make([]string, 0, len(jmDeps)) + for _, d := range jmDeps { + names = append(names, d.Name) + } + return nil, fmt.Errorf( + "found %d tracebloc parent releases in namespace %q (%s); "+ + "this CLI doesn't yet support disambiguating between multiple. "+ + "Pass --namespace to target a namespace with exactly one release.", + len(jmDeps), namespace, strings.Join(names, ", "), + ) + } + + d := jmDeps[0] + release := &ParentRelease{ + ReleaseName: d.Labels["app.kubernetes.io/instance"], + ChartVersion: chartVersionFromLabel(d.Labels["helm.sh/chart"]), + AppVersion: d.Labels["app.kubernetes.io/version"], + } + + // The Service shares the Deployment's release labels by + // convention. Construct the FQDN — we don't need to actually + // hit the API to find it; the chart's helper templates pin the + // service name to "-jobs-manager" (or just + // "jobs-manager" for older chart versions where the release + // prefix wasn't included). + // + // We probe both names to be liberal: if the chart pinned just + // "jobs-manager", that wins; otherwise fall back to the + // release-prefixed form. Customers can always override via the + // ingestor subchart's `jobsManager.endpoint` value. + svc := pickJobsManagerService(ctx, cs, namespace, release.ReleaseName) + const jobsManagerPort = 8080 // chart's well-known port for /internal/submit-ingestion-run + release.JobsManagerService = fmt.Sprintf("http://%s.%s.svc.cluster.local:%d", svc, namespace, jobsManagerPort) + release.JobsManagerServiceName = svc + release.JobsManagerPort = jobsManagerPort + + // Read INGESTOR_IMAGE_DIGEST from jobs-manager's pod-spec env. + // The chart pipes images.ingestor.digest through to here. + // + // SA name is NOT discovered today — the chart doesn't surface + // `ingestionAuthz.serviceAccountName` through the jobs-manager + // env, and reading the ingestionAuthz ConfigMap to learn it is a + // v0.2 follow-up (see #7). We default to "ingestor" (the chart + // default); customers who renamed it pass --ingestor-sa from + // the CLI. Bugbot caught the earlier version that incorrectly + // claimed to read the SA name from env. + release.IngestorSAName = "ingestor" + if len(d.Spec.Template.Spec.Containers) > 0 { + for _, env := range d.Spec.Template.Spec.Containers[0].Env { + if env.Name == "INGESTOR_IMAGE_DIGEST" { + release.IngestorImageDigest = env.Value + } + } + } + + return release, nil +} + +// pickJobsManagerService probes for the chart's jobs-manager +// Service. The chart's helper templates have used both names over +// chart history: +// +// - "jobs-manager" (chart 1.3.x and earlier) +// - "-jobs-manager" (some prefixing variants) +// +// We try the unprefixed name first because that's the dominant +// shipped behavior; fall back if it doesn't exist. Probe failures +// (timeouts, RBAC denials) fall through to the prefixed name too — +// the customer gets a clear DNS error later if neither resolves, +// which is more actionable than "couldn't enumerate services." +func pickJobsManagerService(ctx context.Context, cs kubernetes.Interface, namespace, release string) string { + candidates := []string{ + "jobs-manager", + release + "-jobs-manager", + } + for _, name := range candidates { + _, err := cs.CoreV1().Services(namespace).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return name + } + } + // Last-ditch: return the unprefixed candidate; the caller will + // hit a DNS error at POST time which surfaces with a clearer + // message than us guessing. + return "jobs-manager" +} + +// chartVersionFromLabel extracts "1.3.5" from helm.sh/chart="client-1.3.5". +// Labels in this format are standard Helm output; we strip the +// chart-name prefix to expose just the version. Returns the raw +// label if it doesn't match the expected pattern (defensive +// fallback for unusual chart-name formats). +func chartVersionFromLabel(label string) string { + // Expected: "-"; strip the "client-" prefix. + const prefix = "client-" + if strings.HasPrefix(label, prefix) { + return label[len(prefix):] + } + return label +} diff --git a/internal/cluster/discover_test.go b/internal/cluster/discover_test.go new file mode 100644 index 0000000..5c0921c --- /dev/null +++ b/internal/cluster/discover_test.go @@ -0,0 +1,224 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// jobsManagerDeployment builds the minimal Deployment the chart +// creates for jobs-manager — labels mirror the chart's +// _helpers.tpl output as of client 1.3.5. Tests use this to seed +// the fake clientset with realistic data; if the chart's label +// contract ever changes, these tests fail loudly and we update both +// here and the discovery logic in lockstep. +// +// Note: app.kubernetes.io/name is "client" (the chart name) on +// EVERY resource the chart creates, not "jobs-manager". This is +// the helm convention. The discovery logic distinguishes +// jobs-manager from its mysql / requests-proxy siblings by name +// suffix matching ("-jobs-manager") since the chart-level labels +// don't disambiguate. +func jobsManagerDeployment(release, namespace, chartLabel, appVersion, digest string) *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: release + "-jobs-manager", + Namespace: namespace, + Labels: map[string]string{ + "app.kubernetes.io/name": "client", + "app.kubernetes.io/instance": release, + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/version": appVersion, + "helm.sh/chart": chartLabel, + }, + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "api", + Env: []corev1.EnvVar{ + {Name: "INGESTOR_IMAGE_DIGEST", Value: digest}, + }, + }}, + }, + }, + }, + } +} + +func jobsManagerService(name, namespace string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } +} + +func TestDiscoverParentRelease_HappyPath(t *testing.T) { + const ns = "tracebloc" + cs := fake.NewClientset( + jobsManagerDeployment("tracebloc", ns, + "client-1.3.5", "1.3.5", + "sha256:463e236748708a5e3564569eec9173ea8cb3bcf515992d4939c5b610f3807a4a"), + jobsManagerService("jobs-manager", ns), + ) + + release, err := DiscoverParentRelease(context.Background(), cs, ns) + if err != nil { + t.Fatalf("DiscoverParentRelease: %v", err) + } + + want := ParentRelease{ + ReleaseName: "tracebloc", + ChartVersion: "1.3.5", + AppVersion: "1.3.5", + JobsManagerService: "http://jobs-manager." + ns + ".svc.cluster.local:8080", + JobsManagerServiceName: "jobs-manager", + JobsManagerPort: 8080, + IngestorSAName: "ingestor", + IngestorImageDigest: "sha256:463e236748708a5e3564569eec9173ea8cb3bcf515992d4939c5b610f3807a4a", + } + if *release != want { + t.Errorf("mismatch.\ngot: %+v\nwant: %+v", *release, want) + } +} + +func TestDiscoverParentRelease_NoReleaseFound(t *testing.T) { + cs := fake.NewClientset() // empty cluster + + _, err := DiscoverParentRelease(context.Background(), cs, "tracebloc") + if err == nil { + t.Fatal("expected error when no jobs-manager deployment exists") + } + // The error message has to be customer-actionable. Pin the + // key remediation phrase so a future refactor that loses it + // (or worse, replaces it with a stack trace) fails this test. + for _, want := range []string{"no tracebloc parent client release found", "helm install"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("expected error to mention %q, got: %s", want, err) + } + } +} + +func TestDiscoverParentRelease_MultipleReleases(t *testing.T) { + const ns = "tracebloc" + cs := fake.NewClientset( + jobsManagerDeployment("rel-a", ns, "client-1.3.5", "1.3.5", ""), + jobsManagerDeployment("rel-b", ns, "client-1.3.4", "1.3.4", ""), + ) + + _, err := DiscoverParentRelease(context.Background(), cs, ns) + if err == nil { + t.Fatal("expected error when multiple releases exist") + } + // Customer needs to see which releases the CLI found so they + // can pick (or clean up). + for _, want := range []string{"found 2", "rel-a", "rel-b"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("expected error to mention %q, got: %s", want, err) + } + } +} + +// Regression for the real-cluster discovery bug. Pre-fix, the +// selector was `app.kubernetes.io/name=jobs-manager` which never +// matches because the chart's convention is `name=` +// — so the real-cluster smoke test returned "no parent release +// found" even though one was clearly running. The fix is two-part: +// match on `name=client` (the chart name) AND filter the result +// set by Deployment-name suffix to pick out jobs-manager from its +// mysql/requests-proxy siblings, all of which share the +// chart-level labels. +func TestDiscoverParentRelease_FiltersOutSiblingDeployments(t *testing.T) { + const ns = "tracebloc" + // All three deployments share the chart-level labels — only + // the name suffix distinguishes them. + mysqlClient := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mysql-client", + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/name": "client", + "app.kubernetes.io/instance": "tracebloc", + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "client-1.3.5", + }, + }, + } + requestsProxy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tracebloc-requests-proxy", + Namespace: ns, + Labels: map[string]string{ + "app.kubernetes.io/name": "client", + "app.kubernetes.io/instance": "tracebloc", + "app.kubernetes.io/managed-by": "Helm", + "helm.sh/chart": "client-1.3.5", + }, + }, + } + + cs := fake.NewClientset( + mysqlClient, + requestsProxy, + jobsManagerDeployment("tracebloc", ns, "client-1.3.5", "1.3.5", ""), + jobsManagerService("jobs-manager", ns), + ) + + release, err := DiscoverParentRelease(context.Background(), cs, ns) + if err != nil { + t.Fatalf("DiscoverParentRelease: %v", err) + } + // Should pick jobs-manager, NOT mysql-client or requests-proxy. + if release.ReleaseName != "tracebloc" { + t.Errorf("ReleaseName = %q, want %q (siblings should be filtered out by name suffix)", + release.ReleaseName, "tracebloc") + } +} + +func TestDiscoverParentRelease_FallsBackToReleasePrefixedService(t *testing.T) { + // Some older chart versions exposed the Service as + // "-jobs-manager" rather than the unprefixed + // "jobs-manager". The discover code probes both names and + // picks whichever resolves. This test seeds ONLY the + // release-prefixed Service and asserts the FQDN reflects it. + const ns = "tracebloc" + cs := fake.NewClientset( + jobsManagerDeployment("custom", ns, "client-1.3.5", "1.3.5", ""), + jobsManagerService("custom-jobs-manager", ns), + // NOTE: no "jobs-manager" service + ) + + release, err := DiscoverParentRelease(context.Background(), cs, ns) + if err != nil { + t.Fatalf("DiscoverParentRelease: %v", err) + } + wantFQDN := "http://custom-jobs-manager." + ns + ".svc.cluster.local:8080" + if release.JobsManagerService != wantFQDN { + t.Errorf("JobsManagerService = %q, want %q", release.JobsManagerService, wantFQDN) + } +} + +func TestChartVersionFromLabel(t *testing.T) { + cases := map[string]string{ + "client-1.3.5": "1.3.5", + "client-2.0.0-rc": "2.0.0-rc", + "client": "client", // unexpected shape — return as-is + "other-1.0.0": "other-1.0.0", // non-client chart — return as-is + "": "", + } + for in, want := range cases { + t.Run(in, func(t *testing.T) { + if got := chartVersionFromLabel(in); got != want { + t.Errorf("chartVersionFromLabel(%q) = %q, want %q", in, got, want) + } + }) + } +} diff --git a/internal/cluster/kubeconfig.go b/internal/cluster/kubeconfig.go new file mode 100644 index 0000000..2a94380 --- /dev/null +++ b/internal/cluster/kubeconfig.go @@ -0,0 +1,164 @@ +// Package cluster handles everything the CLI needs to talk to a +// Kubernetes cluster running the tracebloc parent client chart: +// +// - Locating the customer's kubeconfig + picking a context/namespace +// - Discovering the parent release (jobs-manager Deployment + its +// labels) +// - Minting an ingestor ServiceAccount token via TokenRequest +// +// Everything in this package is exercise-able without a real cluster +// thanks to client-go's fake clientset; see *_test.go. +package cluster + +import ( + "fmt" + "os" + "path/filepath" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// KubeconfigOptions captures every knob the customer can turn when +// telling the CLI which cluster + namespace to talk to. Defaults +// mirror kubectl: $KUBECONFIG > ~/.kube/config; current-context if +// --context unset; the context's default namespace if --namespace +// unset; "default" if even the context doesn't pin one. +// +// All fields are zero-value-safe — an empty Options{} produces the +// same behaviour as kubectl run with no flags. +type KubeconfigOptions struct { + // Path overrides $KUBECONFIG and ~/.kube/config when set. + // Honors path expansion (e.g. "~/.kube/config" → absolute). + Path string + + // Context picks a non-current-context from the loaded + // kubeconfig. Empty = use whatever current-context points at. + Context string + + // Namespace overrides the context's default. Empty = pull from + // the context's namespace, falling back to "default". + Namespace string +} + +// ResolvedConfig is what Load() returns: a usable rest.Config plus +// the source-of-truth metadata that the rest of the CLI needs (the +// namespace it resolved to, the context name it used). The +// metadata is separate from rest.Config because rest.Config doesn't +// carry the context name out of the box — it's discarded after the +// merge — and we want to show the customer "you're talking to +// context X" in diagnostic output. +type ResolvedConfig struct { + RestConfig *rest.Config + Context string + Namespace string + + // ServerURL is the cluster's API server URL (the value from + // the context's cluster.server field). Exposed for `tracebloc + // cluster info` to show "you're targeting " so the + // customer can sanity-check before doing anything destructive. + ServerURL string +} + +// Load resolves the kubeconfig + flags into a rest.Config ready for +// kubernetes.NewForConfig(). It does NOT make any network calls — +// pure parsing of local files + env. Network errors only happen +// when the caller subsequently uses the returned RestConfig. +func Load(opts KubeconfigOptions) (*ResolvedConfig, error) { + // NewDefaultClientConfigLoadingRules() wires up the full + // kubectl-style lookup chain: ExplicitPath > $KUBECONFIG + // (multi-file merging) > ~/.kube/config. Constructing + // ClientConfigLoadingRules{} directly with an empty + // ExplicitPath does NOT fall back to those defaults — it + // just refuses to load anything ("no configuration has been + // provided" was the symptom). Surfaced during the first + // real-cluster smoke test of `tracebloc cluster info`. + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + if explicit := expandPath(opts.Path); explicit != "" { + // Setting ExplicitPath bypasses the env-var merge + // (intentional — when a customer says "use THIS file", + // we don't mix it with their env-var-listed others). + loadingRules.ExplicitPath = explicit + } + + overrides := &clientcmd.ConfigOverrides{} + if opts.Context != "" { + overrides.CurrentContext = opts.Context + } + if opts.Namespace != "" { + // Setting Context.Namespace overrides the kubeconfig's + // per-context default. This matches `kubectl --namespace` + // semantics. + overrides.Context.Namespace = opts.Namespace + } + + loader := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(loadingRules, overrides) + + restConfig, err := loader.ClientConfig() + if err != nil { + return nil, fmt.Errorf("building rest config from kubeconfig: %w", err) + } + + // Pull the namespace the way kubectl does: explicit override + // wins; otherwise the context's namespace; otherwise "default". + // loader.Namespace() handles that fallback chain. + ns, _, err := loader.Namespace() + if err != nil { + return nil, fmt.Errorf("resolving namespace from kubeconfig: %w", err) + } + + // rest.Config doesn't carry the context name out of the box — + // fish it back out of the raw kubeconfig so diagnostics can + // show it. RawConfig() does a fresh read each call; cache if + // this ever becomes hot (it isn't). + rawCfg, err := loader.RawConfig() + if err != nil { + return nil, fmt.Errorf("reading raw kubeconfig: %w", err) + } + contextName := overrides.CurrentContext + if contextName == "" { + contextName = rawCfg.CurrentContext + } + + return &ResolvedConfig{ + RestConfig: restConfig, + Context: contextName, + Namespace: ns, + ServerURL: restConfig.Host, + }, nil +} + +// NewClientset is a tiny wrapper that constructs a kubernetes +// clientset from a ResolvedConfig. Exists so callers don't have to +// import k8s.io/client-go/kubernetes themselves for the common case +// of "I have a ResolvedConfig, I want a clientset." +func NewClientset(rc *ResolvedConfig) (kubernetes.Interface, error) { + cs, err := kubernetes.NewForConfig(rc.RestConfig) + if err != nil { + return nil, fmt.Errorf("constructing kubernetes clientset: %w", err) + } + return cs, nil +} + +// expandPath handles `~/.kube/config` → `/home/user/.kube/config` +// since clientcmd's ExplicitPath wants an absolute path. Empty +// strings pass through unchanged (they signal "use defaults" to +// clientcmd). +func expandPath(p string) string { + if p == "" { + return "" + } + if p[0] != '~' { + return p + } + home, err := os.UserHomeDir() + if err != nil { + // Best-effort: return the unexpanded path and let + // clientcmd's error surface mention it. Failing here would + // hide the more useful "tried to read ~/.kube/config and + // got X" error downstream. + return p + } + return filepath.Join(home, p[1:]) +} diff --git a/internal/cluster/kubeconfig_test.go b/internal/cluster/kubeconfig_test.go new file mode 100644 index 0000000..39709e2 --- /dev/null +++ b/internal/cluster/kubeconfig_test.go @@ -0,0 +1,84 @@ +package cluster + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestExpandPath(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Skipf("os.UserHomeDir failed in this environment, skipping: %v", err) + } + + cases := []struct { + in string + wantFunc func(string) bool + }{ + { + in: "", + wantFunc: func(got string) bool { return got == "" }, + }, + { + in: "/absolute/path/to/kubeconfig", + wantFunc: func(got string) bool { return got == "/absolute/path/to/kubeconfig" }, + }, + { + in: "relative/path/kubeconfig", + wantFunc: func(got string) bool { return got == "relative/path/kubeconfig" }, + }, + { + in: "~/.kube/config", + wantFunc: func(got string) bool { + want := filepath.Join(home, ".kube/config") + return got == want + }, + }, + { + in: "~/some/file", + wantFunc: func(got string) bool { + want := filepath.Join(home, "some/file") + return got == want + }, + }, + } + + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + got := expandPath(c.in) + if !c.wantFunc(got) { + t.Errorf("expandPath(%q) = %q (failed predicate)", c.in, got) + } + }) + } +} + +// Load() involves real filesystem + clientcmd parsing; it's covered +// indirectly by the cluster-info integration path. A unit test +// would require writing a temp kubeconfig and feeding it back — +// useful but lower-priority than the discover/token tests since +// the clientcmd library is well-trodden. Pin the smallest contract: +// an empty Options{} doesn't panic. +func TestLoad_EmptyOptionsDoesNotPanic(t *testing.T) { + // May succeed or fail depending on whether the test runner has + // a kubeconfig available; either is fine. The point of this + // test is to ensure the function returns rather than crashes. + _, err := Load(KubeconfigOptions{}) + if err != nil && !looksLikeKubeconfigError(err.Error()) { + // Defense-in-depth: if Load() ever starts wrapping + // errors in a way that loses the "kubeconfig" context, + // flag it so the customer-facing diagnostic stays clear. + t.Logf("Load() failed with unfamiliar error: %v", err) + } +} + +func looksLikeKubeconfigError(s string) bool { + for _, marker := range []string{"kubeconfig", "config", "context", "namespace"} { + if strings.Contains(strings.ToLower(s), marker) { + return true + } + } + return false +} diff --git a/internal/cluster/pvc.go b/internal/cluster/pvc.go new file mode 100644 index 0000000..fd744d0 --- /dev/null +++ b/internal/cluster/pvc.go @@ -0,0 +1,146 @@ +package cluster + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// SharedPVCClaimName is the chart's hardcoded shared-data PVC name. +// +// From tracebloc/client's templates/_helpers.tpl: +// +// {{- define "tracebloc.clientDataPvc" -}} +// client-pvc +// {{- end }} +// +// The helper isn't parameterized by release name (yet) — every +// installation of the chart creates a PVC literally named +// "client-pvc". We probe by claim name rather than by labels here +// because the chart's labels include the helm release name and we +// already discovered the release via DiscoverParentRelease — once +// we have the namespace, the name is unambiguous. +// +// If a customer renames the PVC (out-of-band patch, mostly), they +// need the v0.2 follow-up that reads the name from +// jobs-manager's volume-mount spec instead of hardcoding. Tracked +// as a future ticket alongside #7 (the ingestor-SA-name discovery). +const SharedPVCClaimName = "client-pvc" + +// SharedPVCMountPath is where jobs-manager mounts the shared PVC +// inside its container. The CLI's staging Pod (Phase 3 PR-b) uses +// the same mount path so any tooling that introspects "where files +// live" in either context agrees. From jobs-manager-deployment.yaml: +// +// volumeMounts: +// - name: shared-volume +// mountPath: "/data/shared" +const SharedPVCMountPath = "/data/shared" + +// SharedPVC describes the chart's shared-data PVC after discovery. +// Carries enough metadata for Phase 3 PR-b to construct a stage Pod +// that can mount the same claim. +type SharedPVC struct { + // ClaimName is the metadata.name of the PVC, always + // SharedPVCClaimName today. Wrapped in a field rather than + // re-using the constant so a future "discovered via labels" + // implementation can vary the name without changing callers. + ClaimName string + + // MountPath is the in-container directory where the chart's + // pods mount this claim — SharedPVCMountPath today. Same + // rationale as ClaimName for being a field. + MountPath string + + // AccessModes is the resolved access-mode list on the PVC's + // spec. Surfaces in `tracebloc cluster info` (eventually) and + // drives the stage-Pod scheduling story: + // + // - ReadWriteMany: stage Pod can schedule on any node + // - ReadWriteOnce: stage Pod must land on the same node + // as whatever else is using the PVC (jobs-manager, + // mysql-client). PR-b will surface a warning here so + // RWO clusters get diagnostic guidance up front. + AccessModes []corev1.PersistentVolumeAccessMode + + // Phase is the PVC's current bind phase. We check Bound — an + // Unbound PVC means the cluster never provisioned storage + // (e.g. missing StorageClass), and the stage Pod would hang + // indefinitely waiting for a volume. + Phase corev1.PersistentVolumeClaimPhase +} + +// DiscoverSharedPVC verifies the chart's shared-data PVC exists in +// the given namespace and returns its metadata. Returns a friendly +// error if the PVC isn't there or isn't Bound — both are real +// situations Phase 3's pre-flight catches before we waste time +// constructing a Pod that can't mount anything. +func DiscoverSharedPVC(ctx context.Context, cs kubernetes.Interface, namespace string) (*SharedPVC, error) { + pvc, err := cs.CoreV1().PersistentVolumeClaims(namespace). + Get(ctx, SharedPVCClaimName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + // The 80% case for "the CLI's pre-flight failed against + // a cluster that has SOME tracebloc release installed + // but not the parent client chart" — the parent-release + // check (DiscoverParentRelease) catches the no-release + // case first, so reaching here usually means the chart + // was installed with a customized PVC name. + return nil, fmt.Errorf( + "no PersistentVolumeClaim named %q found in namespace %q. "+ + "The chart's _helpers.tpl pins this name; if your install "+ + "renamed it out-of-band, the CLI doesn't yet support that "+ + "(read-name-from-jobs-manager is a v0.2 follow-up). "+ + "Verify with: kubectl get pvc -n %s", + SharedPVCClaimName, namespace, namespace) + } + // Forbidden / network / other — surface as-is so the + // customer can RBAC-debug. Wrapping rather than substituting + // because the underlying %w already carries the useful info. + return nil, fmt.Errorf("reading PVC %s/%s: %w", + namespace, SharedPVCClaimName, err) + } + + if pvc.Status.Phase != corev1.ClaimBound { + // Unbound PVCs are a real cluster-config issue — pre-flight + // surfacing of this is much more helpful than the stage Pod + // silently pending forever. The most common cause is a + // missing StorageClass (the chart's default is the cluster + // default, which may not exist on EKS without + // gp2/gp3-csi configured). + return nil, fmt.Errorf( + "PVC %s/%s is in phase %q, not Bound. "+ + "The shared volume hasn't been provisioned — "+ + "check that the cluster has a usable StorageClass "+ + "(kubectl get sc) and that the PVC's storageClassName matches.", + namespace, SharedPVCClaimName, pvc.Status.Phase) + } + + return &SharedPVC{ + ClaimName: pvc.Name, + MountPath: SharedPVCMountPath, + AccessModes: pvc.Spec.AccessModes, + Phase: pvc.Status.Phase, + }, nil +} + +// IsReadWriteMany reports whether the PVC accepts simultaneous +// mounts from multiple nodes. The Phase 3 stage Pod cares about +// this because RWO claims force same-node scheduling — and if the +// existing mounter (jobs-manager) is on a different node than +// where the scheduler wants to put our stage Pod, the Pod will +// pend indefinitely. PR-b will surface a pre-flight warning when +// this returns false; the dataset push still proceeds (eventually +// succeeds when the scheduler co-locates), it just takes longer. +func (p *SharedPVC) IsReadWriteMany() bool { + for _, m := range p.AccessModes { + if m == corev1.ReadWriteMany { + return true + } + } + return false +} diff --git a/internal/cluster/pvc_test.go b/internal/cluster/pvc_test.go new file mode 100644 index 0000000..3260ed7 --- /dev/null +++ b/internal/cluster/pvc_test.go @@ -0,0 +1,117 @@ +package cluster + +import ( + "context" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// seedPVC returns a corev1.PersistentVolumeClaim wired to look like +// the chart's shared-data claim. Lets each test mutate one field +// (phase, access mode) to exercise its specific branch. +func seedPVC(phase corev1.PersistentVolumeClaimPhase, modes ...corev1.PersistentVolumeAccessMode) *corev1.PersistentVolumeClaim { + if len(modes) == 0 { + modes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany} + } + return &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: SharedPVCClaimName, + Namespace: "tracebloc", + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: modes, + }, + Status: corev1.PersistentVolumeClaimStatus{ + Phase: phase, + }, + } +} + +func TestDiscoverSharedPVC_HappyPath(t *testing.T) { + cs := fake.NewClientset(seedPVC(corev1.ClaimBound)) + got, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("DiscoverSharedPVC: %v", err) + } + if got.ClaimName != SharedPVCClaimName { + t.Errorf("ClaimName = %q, want %q", got.ClaimName, SharedPVCClaimName) + } + if got.MountPath != SharedPVCMountPath { + t.Errorf("MountPath = %q, want %q", got.MountPath, SharedPVCMountPath) + } + if got.Phase != corev1.ClaimBound { + t.Errorf("Phase = %v, want Bound", got.Phase) + } + if !got.IsReadWriteMany() { + t.Errorf("IsReadWriteMany() = false on a RWX-seeded PVC") + } +} + +func TestDiscoverSharedPVC_NotFound(t *testing.T) { + // Empty cluster — the parent-release check should normally + // have failed first, but we still pin the diagnostic for the + // case where customers renamed the PVC out-of-band. + cs := fake.NewClientset() + _, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err == nil { + t.Fatal("DiscoverSharedPVC returned nil error on empty cluster") + } + // Diagnostic must point at the v0.2 follow-up and `kubectl + // get pvc` command — those are the actionable bits. + for _, want := range []string{SharedPVCClaimName, "v0.2", "kubectl get pvc"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %s", want, err) + } + } +} + +func TestDiscoverSharedPVC_UnboundPhase(t *testing.T) { + // Most common cause: missing StorageClass on EKS where the + // admin removed gp2-default before installing the chart. The + // pre-flight diagnostic must call out StorageClass explicitly + // so customers know what to fix. + cs := fake.NewClientset(seedPVC(corev1.ClaimPending)) + _, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err == nil { + t.Fatal("DiscoverSharedPVC returned nil error on Pending PVC") + } + for _, want := range []string{"Pending", "not Bound", "StorageClass"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %s", want, err) + } + } +} + +func TestIsReadWriteMany_RWO(t *testing.T) { + // ReadWriteOnce — common on cheap-storage clusters (single-node + // minikube, single-zone EBS). Phase 3 still works against RWO + // (the scheduler co-locates), but PR-b will print a warning. + // Pin the API surface that the warning logic will key off. + cs := fake.NewClientset(seedPVC(corev1.ClaimBound, corev1.ReadWriteOnce)) + got, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("DiscoverSharedPVC: %v", err) + } + if got.IsReadWriteMany() { + t.Errorf("IsReadWriteMany() = true on a RWO-seeded PVC") + } +} + +func TestIsReadWriteMany_RWXMixedWithOther(t *testing.T) { + // A few cloud providers (specifically EFS-on-EKS) advertise + // both RWX and ROX. As long as RWX is in the list, our stage + // Pod can schedule freely. + cs := fake.NewClientset(seedPVC(corev1.ClaimBound, + corev1.ReadOnlyMany, corev1.ReadWriteMany)) + got, err := DiscoverSharedPVC(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("DiscoverSharedPVC: %v", err) + } + if !got.IsReadWriteMany() { + t.Errorf("IsReadWriteMany() = false on a mixed-mode PVC including RWX") + } +} diff --git a/internal/cluster/token.go b/internal/cluster/token.go new file mode 100644 index 0000000..0c9a962 --- /dev/null +++ b/internal/cluster/token.go @@ -0,0 +1,214 @@ +package cluster + +import ( + "context" + "fmt" + + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// IngestorToken is what the customer (well, the CLI) authenticates +// to jobs-manager with. Bearer-token over HTTPS to +// /internal/submit-ingestion-run. The token is short-lived (10 +// minutes default) so it can be regenerated per `tracebloc dataset +// push` invocation without long-term-credential concerns. +type IngestorToken struct { + // Token is the raw bearer token string. Treat as sensitive; + // don't log it. Diagnostics print SHA256(Token)[:8] instead. + Token string + + // ExpirationSeconds matches the request — actual server-side + // expiration may be capped by cluster policy + // (--service-account-max-token-expiration on kube-apiserver). + // We don't try to parse the JWT to read its `exp` claim + // because the customer's diagnostic ("token expires in ~N + // min") is accurate enough using the requested value. + ExpirationSeconds int64 + + // Source records how the token was obtained — TokenRequest + // (the modern path) or a static secret (the fallback for + // clusters where the user can't call TokenRequest). Surfaced + // in `tracebloc cluster info` so the customer can see which + // path their RBAC allowed. + Source TokenSource +} + +// TokenSource enumerates the two ways the CLI can obtain an +// ingestor SA token. Adding a third (e.g. a customer-managed +// long-lived secret reference) is a v0.2 extension. +type TokenSource int + +const ( + // TokenSourceTokenRequest means we called the + // /serviceaccounts/{name}/token subresource. The modern, + // rotation-friendly path. Requires `create` permission on + // "serviceaccounts/token" for the user's kubeconfig. + TokenSourceTokenRequest TokenSource = iota + + // TokenSourceStaticSecret means we fell back to reading a + // pre-existing Secret of type + // kubernetes.io/service-account-token that references the + // ingestor SA. Older k8s (pre-1.24 default) auto-created + // these; on 1.24+ admins must create them by hand. Tokens + // from this source are long-lived (no expiration), which is + // less ideal but unblocks customers whose RBAC denies + // TokenRequest. + TokenSourceStaticSecret +) + +func (s TokenSource) String() string { + switch s { + case TokenSourceTokenRequest: + return "TokenRequest" + case TokenSourceStaticSecret: + return "static secret" + default: + return "unknown" + } +} + +// MintIngestorToken obtains a bearer token for the ingestor +// ServiceAccount in the given namespace. Tries TokenRequest first; +// if that's denied (RBAC) or unavailable (older API server) falls +// back to looking for a pre-existing service-account-token Secret. +// Returns an error with a clear remediation message if neither +// works. +// +// audiences may be nil/empty for the default API-server audience. +// Today jobs-manager's TokenReview-based validation accepts the +// default audience, so callers usually pass nil; future audiences +// (e.g. a forthcoming jobs-manager-specific audience) plug in here +// without API breakage. +func MintIngestorToken( + ctx context.Context, + cs kubernetes.Interface, + namespace, saName string, + expirationSeconds int64, + audiences []string, +) (*IngestorToken, error) { + // Try TokenRequest first. + req := &authenticationv1.TokenRequest{ + Spec: authenticationv1.TokenRequestSpec{ + ExpirationSeconds: &expirationSeconds, + Audiences: audiences, + }, + } + tr, err := cs.CoreV1().ServiceAccounts(namespace). + CreateToken(ctx, saName, req, metav1.CreateOptions{}) + + if err == nil { + return &IngestorToken{ + Token: tr.Status.Token, + ExpirationSeconds: expirationSeconds, + Source: TokenSourceTokenRequest, + }, nil + } + + // TokenRequest didn't work. Distinguish the cases we can fall + // back from (permission denied / not-found / unsupported) + // from cases we can't (e.g. context cancelled, network down). + if !isTokenRequestRecoverable(err) { + return nil, fmt.Errorf( + "minting token for ServiceAccount %s/%s via TokenRequest: %w", + namespace, saName, err, + ) + } + + // Fallback: look for a Secret of type + // kubernetes.io/service-account-token in the same namespace + // whose annotations point at our SA. + tok, fallbackErr := findStaticTokenSecret(ctx, cs, namespace, saName) + if fallbackErr != nil { + // Surface BOTH the original TokenRequest failure and the + // fallback failure — they give the customer different + // remediation options. + return nil, fmt.Errorf( + "no usable ingestor token. TokenRequest failed: %v. "+ + "Fallback to static secret also failed: %w. "+ + "Remediation: either grant your user the `create` verb on "+ + "`serviceaccounts/token` (RBAC), or have an admin create a "+ + "long-lived Secret of type kubernetes.io/service-account-token "+ + "that references the %s ServiceAccount in namespace %s.", + err, fallbackErr, saName, namespace, + ) + } + return tok, nil +} + +// isTokenRequestRecoverable reports whether a failed TokenRequest +// call is one we can fall back from (permission denied, API not +// available, SA missing) vs one we can't (context cancelled, +// network errors, etc.). +// +// Falling back on a network error would mask the network problem; +// the customer would see a less useful "static secret not found" +// instead of "couldn't reach the API server." +// +// Defers to k8s.io/apimachinery/pkg/api/errors for the classification +// rather than re-implementing the HTTP-status checks. The stdlib +// helpers key off Status.Reason (the typed enum) — that's the +// canonical interpretation, and our test file already constructs its +// fake errors via apierrors.NewForbidden, which sets both Code and +// Reason. Earlier versions of this file rolled their own helpers +// that read Status.Code numerically; Bugbot correctly flagged the +// divergence as a future-bug-waiting-to-happen for non-standard +// status errors. +func isTokenRequestRecoverable(err error) bool { + if err == nil { + return false + } + // Permission denied (StatusForbidden), SA missing + // (StatusNotFound), API not available on this cluster — typically + // pre-1.22 (StatusMethodNotAllowed/MethodNotSupported). + return apierrors.IsForbidden(err) || + apierrors.IsNotFound(err) || + apierrors.IsMethodNotSupported(err) +} + +// findStaticTokenSecret looks for a long-lived +// service-account-token Secret bound to the given SA. The legacy +// pre-k8s-1.24 way of authenticating an SA: kubectl auto-created +// these on SA creation. On modern clusters admins create them by +// hand when needed. +func findStaticTokenSecret(ctx context.Context, cs kubernetes.Interface, namespace, saName string) (*IngestorToken, error) { + secrets, err := cs.CoreV1().Secrets(namespace).List(ctx, metav1.ListOptions{ + FieldSelector: "type=kubernetes.io/service-account-token", + }) + if err != nil { + // If the user can't even list Secrets, surface that + // directly — they may be missing more RBAC than just + // TokenRequest, and "static secret fallback failed" is + // less informative than the underlying error. + return nil, fmt.Errorf("listing service-account-token secrets in %s: %w", namespace, err) + } + + for _, s := range secrets.Items { + // The Secret is bound to an SA via the + // `kubernetes.io/service-account.name` annotation. (The + // `.uid` annotation pins it to that exact SA instance — + // we don't check that because we want any historically + // created token-secret for the named SA to work.) + if s.Annotations[corev1.ServiceAccountNameKey] != saName { + continue + } + token, ok := s.Data["token"] + if !ok || len(token) == 0 { + continue // empty token field — skip + } + return &IngestorToken{ + Token: string(token), + ExpirationSeconds: 0, // static secrets don't expire client-side + Source: TokenSourceStaticSecret, + }, nil + } + + return nil, fmt.Errorf( + "no Secret of type kubernetes.io/service-account-token bound to "+ + "ServiceAccount %s found in namespace %s", + saName, namespace, + ) +} diff --git a/internal/cluster/token_test.go b/internal/cluster/token_test.go new file mode 100644 index 0000000..64e1c60 --- /dev/null +++ b/internal/cluster/token_test.go @@ -0,0 +1,186 @@ +package cluster + +import ( + "context" + "errors" + "strings" + "testing" + + authenticationv1 "k8s.io/api/authentication/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +func TestMintIngestorToken_TokenRequest_HappyPath(t *testing.T) { + const ns = "tracebloc" + cs := fake.NewClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "ingestor", Namespace: ns}, + }) + + // The fake clientset doesn't implement the TokenRequest + // subresource by default — we have to inject a reactor. This + // mirrors what the real API server does: returns a stamped + // Status.Token. + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + ca, ok := action.(k8stesting.CreateAction) + if !ok { + return false, nil, nil + } + if ca.GetSubresource() != "token" { + return false, nil, nil + } + tr := ca.GetObject().(*authenticationv1.TokenRequest) + tr.Status.Token = "fake-token-deadbeef" + return true, tr, nil + }) + + tok, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err != nil { + t.Fatalf("MintIngestorToken: %v", err) + } + if tok.Token != "fake-token-deadbeef" { + t.Errorf("Token = %q, want %q", tok.Token, "fake-token-deadbeef") + } + if tok.Source != TokenSourceTokenRequest { + t.Errorf("Source = %v, want TokenSourceTokenRequest", tok.Source) + } + if tok.ExpirationSeconds != 600 { + t.Errorf("ExpirationSeconds = %d, want 600", tok.ExpirationSeconds) + } +} + +func TestMintIngestorToken_FallsBackToStaticSecret(t *testing.T) { + const ns = "tracebloc" + + // Pre-seed a service-account-token Secret bound to the + // ingestor SA. This is what older clusters auto-created on SA + // creation; modern clusters require admins to create it by + // hand. + staticSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ingestor-token-abc123", + Namespace: ns, + Annotations: map[string]string{ + corev1.ServiceAccountNameKey: "ingestor", + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{ + "token": []byte("static-token-cafebabe"), + }, + } + cs := fake.NewClientset( + &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "ingestor", Namespace: ns}, + }, + staticSecret, + ) + + // Make TokenRequest fail with a recoverable error + // (Forbidden = the user lacks RBAC, which is the dominant + // reason customers fall through to the static-secret path). + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + if ca, ok := action.(k8stesting.CreateAction); ok && ca.GetSubresource() == "token" { + return true, nil, apierrors.NewForbidden( + corev1.Resource("serviceaccounts/token"), + "ingestor", + errors.New("user cannot create serviceaccounts/token"), + ) + } + return false, nil, nil + }) + + tok, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err != nil { + t.Fatalf("MintIngestorToken: %v", err) + } + if tok.Token != "static-token-cafebabe" { + t.Errorf("Token = %q, want fallback static-token-cafebabe", tok.Token) + } + if tok.Source != TokenSourceStaticSecret { + t.Errorf("Source = %v, want TokenSourceStaticSecret", tok.Source) + } + if tok.ExpirationSeconds != 0 { + t.Errorf("ExpirationSeconds = %d, want 0 (static secrets don't expire client-side)", tok.ExpirationSeconds) + } +} + +func TestMintIngestorToken_NoFallbackOnUnrecoverableError(t *testing.T) { + // Network errors (or any non-403/404/405 error) shouldn't + // trigger the static-secret fallback — they'd mask the real + // underlying problem. Pin that behavior. + const ns = "tracebloc" + cs := fake.NewClientset() + + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + if ca, ok := action.(k8stesting.CreateAction); ok && ca.GetSubresource() == "token" { + return true, nil, errors.New("simulated network failure: i/o timeout") + } + return false, nil, nil + }) + + _, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err == nil { + t.Fatal("expected non-recoverable error to propagate, got nil") + } + if !strings.Contains(err.Error(), "i/o timeout") { + t.Errorf("expected the network-error message to surface, got: %v", err) + } +} + +func TestMintIngestorToken_BothPathsFailWithRemediation(t *testing.T) { + // Worst case: TokenRequest forbidden AND no static secret. The + // error must surface both failures plus an actionable + // remediation hint. + const ns = "tracebloc" + cs := fake.NewClientset(&corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{Name: "ingestor", Namespace: ns}, + }) + + cs.PrependReactor("create", "serviceaccounts", + func(action k8stesting.Action) (handled bool, ret runtime.Object, err error) { + if ca, ok := action.(k8stesting.CreateAction); ok && ca.GetSubresource() == "token" { + return true, nil, apierrors.NewForbidden( + corev1.Resource("serviceaccounts/token"), + "ingestor", + errors.New("forbidden"), + ) + } + return false, nil, nil + }) + + _, err := MintIngestorToken(context.Background(), cs, ns, "ingestor", 600, nil) + if err == nil { + t.Fatal("expected error when both TokenRequest + static fallback fail") + } + for _, want := range []string{ + "TokenRequest failed", + "static secret also failed", + "Remediation", + "create", // remediation mentions granting the verb + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("expected combined error to mention %q, got: %s", want, err) + } + } +} + +func TestTokenSource_String(t *testing.T) { + cases := map[TokenSource]string{ + TokenSourceTokenRequest: "TokenRequest", + TokenSourceStaticSecret: "static secret", + TokenSource(99): "unknown", + } + for s, want := range cases { + if got := s.String(); got != want { + t.Errorf("TokenSource(%d).String() = %q, want %q", s, got, want) + } + } +} diff --git a/internal/push/category.go b/internal/push/category.go new file mode 100644 index 0000000..c20728a --- /dev/null +++ b/internal/push/category.go @@ -0,0 +1,75 @@ +package push + +// Category families. These mirror data-ingestors' +// tracebloc_ingestor/cli/conventions.py groupings so the CLI's +// per-category behaviour (which flags are required, which local +// layout to expect, which spec fields to emit) stays in lock-step +// with what the ingestor actually resolves. +// +// Kept as a single source of truth here rather than scattered +// string comparisons across spec.go / dataset.go. + +// imageCategories take a labels CSV + an images/ directory (plus, +// for some, extra sidecar dirs handled in later increments). +var imageCategories = map[string]bool{ + "image_classification": true, + "object_detection": true, + "keypoint_detection": true, + "semantic_segmentation": true, + "instance_segmentation": true, +} + +// tabularCategories take a single CSV whose columns are described by +// a `schema` (column → SQL type) map. No sidecar files. +var tabularCategories = map[string]bool{ + "tabular_classification": true, + "tabular_regression": true, + "time_series_forecasting": true, + "time_to_event_prediction": true, +} + +// regressionClassCategories predict a numeric target rather than a +// class. The schema requires the label in object form with an +// explicit `policy` so the raw target never ships to the central +// backend by default (policy=bucket bins it first). +var regressionClassCategories = map[string]bool{ + "tabular_regression": true, + "time_series_forecasting": true, + "time_to_event_prediction": true, +} + +// textCategories take a labels CSV + a directory of text files +// (texts/ for classification, sequences/ for masked language +// modeling). masked_language_modeling additionally needs a +// tokenizer.json at the dataset root and has NO label. +var textCategories = map[string]bool{ + "text_classification": true, + "masked_language_modeling": true, +} + +// IsImage reports whether category uses the labels.csv + images/ +// local layout. +func IsImage(category string) bool { return imageCategories[category] } + +// IsTabular reports whether category uses the single-CSV + schema +// local layout (no sidecar files). +func IsTabular(category string) bool { return tabularCategories[category] } + +// IsRegressionClass reports whether category predicts a numeric +// target and therefore needs label.policy (object label form). +func IsRegressionClass(category string) bool { return regressionClassCategories[category] } + +// IsText reports whether category uses the labels.csv + text-file +// directory (texts/ or sequences/) local layout. +func IsText(category string) bool { return textCategories[category] } + +// TextSidecarDir returns the sidecar directory name a text category +// expects: "sequences" for masked_language_modeling, "texts" for +// text_classification. (Used both as the local subdir to stage and +// the spec field to emit.) +func TextSidecarDir(category string) string { + if category == "masked_language_modeling" { + return "sequences" + } + return "texts" +} diff --git a/internal/push/detect.go b/internal/push/detect.go new file mode 100644 index 0000000..9a2b838 --- /dev/null +++ b/internal/push/detect.go @@ -0,0 +1,70 @@ +package push + +import ( + "fmt" + "image" + "os" + "strconv" + "strings" + + // Register the stdlib image decoders so image.DecodeConfig can + // read the headers of the formats the image_classification layout + // accepts. webp is NOT in the stdlib — DetectImageSize returns an + // error for it and the caller falls back to requiring + // --target-size. (.jpg/.jpeg both decode via image/jpeg.) + _ "image/gif" + _ "image/jpeg" + _ "image/png" +) + +// DetectImageSize returns the pixel width and height of the image at +// path by decoding only its header (image.DecodeConfig — it does not +// read the pixel data, so it's cheap even for large images). +// +// Supports the stdlib-registered formats (jpeg, png, gif). Returns an +// error for formats without a registered decoder (notably webp); the +// caller treats that as "couldn't auto-detect" and falls back to the +// ingestor default, advising --target-size. +func DetectImageSize(path string) (width, height int, err error) { + f, err := os.Open(path) + if err != nil { + return 0, 0, err + } + defer func() { _ = f.Close() }() + + cfg, _, err := image.DecodeConfig(f) + if err != nil { + return 0, 0, fmt.Errorf("decoding image header %q: %w", path, err) + } + return cfg.Width, cfg.Height, nil +} + +// ParseTargetSize parses a --target-size flag value into [width, +// height]. Accepts "WxH" (the documented form, e.g. "512x512") and +// "W,H" as a convenience. Both dimensions must be positive integers. +func ParseTargetSize(s string) (width, height int, err error) { + sep := "x" + if strings.Contains(s, ",") { + sep = "," + } + parts := strings.Split(s, sep) + if len(parts) != 2 { + return 0, 0, fmt.Errorf( + "target size %q must be WxH (e.g. 512x512)", s) + } + width, err = strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + return 0, 0, fmt.Errorf( + "target size %q: width is not an integer: %w", s, err) + } + height, err = strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return 0, 0, fmt.Errorf( + "target size %q: height is not an integer: %w", s, err) + } + if width <= 0 || height <= 0 { + return 0, 0, fmt.Errorf( + "target size %q: width and height must both be positive", s) + } + return width, height, nil +} diff --git a/internal/push/detect_test.go b/internal/push/detect_test.go new file mode 100644 index 0000000..14348bf --- /dev/null +++ b/internal/push/detect_test.go @@ -0,0 +1,88 @@ +package push + +import ( + "image" + "image/png" + "os" + "path/filepath" + "testing" +) + +// TestParseTargetSize covers the --target-size flag parser: the +// documented WxH form, the W,H convenience form, and the rejection +// cases (missing dimension, non-integer, non-positive, wrong arity). +func TestParseTargetSize(t *testing.T) { + cases := []struct { + in string + w, h int + wantErr bool + }{ + {"512x512", 512, 512, false}, + {"640x480", 640, 480, false}, + {"512,512", 512, 512, false}, + {"1x1", 1, 1, false}, + {"512", 0, 0, true}, + {"512x", 0, 0, true}, + {"x512", 0, 0, true}, + {"0x512", 0, 0, true}, + {"-4x512", 0, 0, true}, + {"512x512x512", 0, 0, true}, + {"abcxdef", 0, 0, true}, + {"", 0, 0, true}, + } + for _, c := range cases { + w, h, err := ParseTargetSize(c.in) + if c.wantErr { + if err == nil { + t.Errorf("ParseTargetSize(%q) = (%d,%d,nil), want error", c.in, w, h) + } + continue + } + if err != nil { + t.Errorf("ParseTargetSize(%q) unexpected error: %v", c.in, err) + continue + } + if w != c.w || h != c.h { + t.Errorf("ParseTargetSize(%q) = (%d,%d), want (%d,%d)", c.in, w, h, c.w, c.h) + } + } +} + +// TestDetectImageSize_PNG: a real (generated) PNG's header is decoded +// to its true dimensions. Pins the auto-detect path used when the +// customer doesn't pass --target-size. +func TestDetectImageSize_PNG(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "img.png") + f, err := os.Create(p) + if err != nil { + t.Fatal(err) + } + if err := png.Encode(f, image.NewRGBA(image.Rect(0, 0, 320, 200))); err != nil { + t.Fatal(err) + } + _ = f.Close() + + w, h, err := DetectImageSize(p) + if err != nil { + t.Fatalf("DetectImageSize: %v", err) + } + if w != 320 || h != 200 { + t.Errorf("DetectImageSize = (%d,%d), want (320,200)", w, h) + } +} + +// TestDetectImageSize_Unsupported: a non-image (or unregistered +// format) returns an error so the caller falls back to the ingestor +// default + advises --target-size, rather than silently using a +// bogus size. +func TestDetectImageSize_Unsupported(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "note.txt") + if err := os.WriteFile(p, []byte("not an image"), 0o644); err != nil { + t.Fatal(err) + } + if _, _, err := DetectImageSize(p); err == nil { + t.Error("DetectImageSize on non-image returned nil error; want a decode error") + } +} diff --git a/internal/push/image_extras.go b/internal/push/image_extras.go new file mode 100644 index 0000000..5e1e0f1 --- /dev/null +++ b/internal/push/image_extras.go @@ -0,0 +1,51 @@ +package push + +import ( + "fmt" + "path/filepath" +) + +// xmlExtensions are the annotation file types object_detection reads +// (Pascal VOC XML). +var xmlExtensions = map[string]struct{}{".xml": {}} + +// DiscoverObjectDetection validates a local object_detection dataset: +// +// - /labels.csv (required) +// - /images/* (required) +// - /annotations/*.xml (required; Pascal VOC) +// +// It builds on the image-classification layout (labels.csv + images/) +// and adds the annotations/ sidecar via the shared sidecar walker, so +// the existing tar/stream machinery stages annotations under +// "annotations/". +func DiscoverObjectDetection(rootDir string) (*LocalLayout, error) { + layout, err := Discover(rootDir) // labels.csv + images/ (+ caps + symlink guards) + if err != nil { + return nil, err + } + + annotations, annoBytes, err := discoverSidecarFiles(layout.Root, "annotations", xmlExtensions) + if err != nil { + return nil, err + } + if len(annotations) == 0 { + return nil, fmt.Errorf( + "no .xml annotation files found in %q. object_detection expects "+ + "/annotations/*.xml (Pascal VOC).", + filepath.Join(layout.Root, "annotations")) + } + if layout.Sidecars == nil { + layout.Sidecars = map[string][]string{} + } + layout.Sidecars["annotations"] = annotations + layout.TotalBytes += annoBytes + + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. For larger datasets, the "+ + "cloud-source path is on the v0.2 roadmap (tracebloc/client#147).", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + return layout, nil +} diff --git a/internal/push/image_extras_test.go b/internal/push/image_extras_test.go new file mode 100644 index 0000000..8ed8d71 --- /dev/null +++ b/internal/push/image_extras_test.go @@ -0,0 +1,59 @@ +package push + +import ( + "os" + "path/filepath" + "testing" +) + +// mkODDir builds an object_detection dataset dir: labels.csv + +// images/001.jpg, plus annotations/001.xml when withAnnotations. +func mkODDir(t *testing.T, withAnnotations bool) string { + t.Helper() + dir := t.TempDir() + writeFile(t, dir, "labels.csv", "image_label,filename\ncat,001.jpg\n") + imgs := filepath.Join(dir, "images") + if err := os.MkdirAll(imgs, 0o755); err != nil { + t.Fatal(err) + } + // JPEG magic bytes; Discover checks extension + size, not decode. + if err := os.WriteFile(filepath.Join(imgs, "001.jpg"), []byte("\xff\xd8\xff\xe0"), 0o644); err != nil { + t.Fatal(err) + } + if withAnnotations { + ann := filepath.Join(dir, "annotations") + if err := os.MkdirAll(ann, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(ann, "001.xml"), []byte(""), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +// TestDiscoverObjectDetection: a valid OD layout yields images + +// the annotations sidecar, staged together. +func TestDiscoverObjectDetection(t *testing.T) { + layout, err := DiscoverObjectDetection(mkODDir(t, true)) + if err != nil { + t.Fatalf("DiscoverObjectDetection: %v", err) + } + if len(layout.Images) != 1 { + t.Errorf("images = %d, want 1", len(layout.Images)) + } + if len(layout.Sidecars["annotations"]) != 1 { + t.Errorf("annotations = %d, want 1", len(layout.Sidecars["annotations"])) + } + if got := layout.FileCount(); got != 3 { // labels.csv + image + xml + t.Errorf("FileCount = %d, want 3", got) + } +} + +// TestDiscoverObjectDetection_MissingAnnotations: OD without an +// annotations/ directory is a clear error. +func TestDiscoverObjectDetection_MissingAnnotations(t *testing.T) { + if _, err := DiscoverObjectDetection(mkODDir(t, false)); err == nil { + t.Error("DiscoverObjectDetection without annotations/ returned nil error") + } +} diff --git a/internal/push/orphan.go b/internal/push/orphan.go new file mode 100644 index 0000000..65ab4c0 --- /dev/null +++ b/internal/push/orphan.go @@ -0,0 +1,196 @@ +package push + +import ( + "context" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// OrphanGracePeriod is how long a non-Running stage Pod has to live +// before we flag it as an orphan. Running Pods are NEVER flagged +// regardless of age (see FindOrphanStagePods's Phase=Running skip, +// Bugbot r7) — Pods past pod.go's StagePodActiveDeadline of 30 min +// are still legitimate "in-progress active push" candidates, and +// any per-age cap would produce false positives for near-cap pushes. +// +// 5 minutes targets the genuinely-stuck shapes: +// +// - Phase=Pending past 5 min: image pull failure that didn't +// resolve, scheduling stuck on missing nodes, PSA rejection +// that took kubelet a while to surface +// - Phase=Failed past 5 min: container crashed at startup or +// during stream, CLI didn't get a chance to clean up +// - Phase=Unknown past 5 min: node went away (network partition, +// kubelet crash) and the Pod metadata is stranded +// +// All three shapes are genuinely orphan — no still-running push +// will ever progress past Pending/Failed/Unknown into Ready, so +// 5 min is comfortable for "stuck enough to surface as a warning." +const OrphanGracePeriod = 5 * time.Minute + +// Orphan describes a stage Pod found by the orphan scan. Carries +// enough metadata for the warning surface to render an actionable +// "delete with: kubectl delete pod X -n Y" hint. +type Orphan struct { + // Name is the metadata.name. + Name string + + // Namespace is the resolved namespace (not the chart's + // release-name, the actual API namespace). + Namespace string + + // Table is the destination table the orphan was staging for — + // read from the StagePodTableLabel set in BuildStagePodSpec. + // May be empty for very old orphans (pre-label-convention), + // but every Pod the CLI ever created carries this label. + Table string + + // Age is how long the Pod has existed (now - creationTimestamp). + // Surfaces in the warning so customers can spot stale Pods at + // a glance ("3 hours old" → almost certainly safe to delete; + // "6 minutes old" → maybe wait, might be from a slow parallel + // push). + Age time.Duration +} + +// FindOrphanStagePods lists every stage Pod the CLI has ever +// created in the namespace and returns the ones past OrphanGrace +// Period. Pods younger than that are silently filtered — they +// might be the CURRENT invocation's own Pod (the orphan scan runs +// before CreateStagePod, but in a tight enough loop the previous +// invocation's just-created Pod could plausibly still be < grace). +// +// Returns nil + nil if there are no orphans. Doesn't return an +// error for the API call failing — orphan scanning is best-effort +// (an RBAC denial on `list pods` shouldn't block the push) — the +// caller logs the scan failure but proceeds. +// +// Hence the signature returns ([]Orphan, error): the slice is +// empty on success-with-no-orphans, the error is non-nil only on +// API failures the caller might want to surface for debugging. +func FindOrphanStagePods(ctx context.Context, cs kubernetes.Interface, namespace string) ([]Orphan, error) { + selector := fmt.Sprintf("%s=%s,%s=%s", + StagePodManagedByLabel, StagePodManagedByValue, + StagePodComponentLabel, StagePodComponentValue) + + pods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + // Don't wrap — the caller (cli.runDatasetPush) will either + // log this for diagnostic value and proceed, or ignore it + // entirely. Either way, the orphan scan isn't on the + // critical path. + return nil, fmt.Errorf("listing stage Pods in %s: %w", namespace, err) + } + + now := time.Now() + var orphans []Orphan + for i := range pods.Items { + p := &pods.Items[i] + + // Phase=Running means an active push (this workstation or + // another's) is still doing real work. activeDeadlineSeconds + // is its safety net at the cluster level. Flagging it as + // orphan would produce false positives for legitimate + // slow/near-cap pushes — pod.go budgets ~8.5 minutes for + // the 1 GiB-cap transfer alone, which exceeds the 5-min + // grace below. Bugbot flagged the false-positive risk on + // PR-b round 7. + // + // Pods in non-Running phases (Pending/Failed/Unknown) past + // the grace period are still flagged — those are the + // genuine orphan shapes (stuck on image pull, crashed + // during stream, network-partitioned). + if p.Status.Phase == corev1.PodRunning { + continue + } + + age := now.Sub(p.CreationTimestamp.Time) + if age < OrphanGracePeriod { + continue + } + orphans = append(orphans, Orphan{ + Name: p.Name, + Namespace: p.Namespace, + Table: p.Labels[StagePodTableLabel], + Age: age, + }) + } + return orphans, nil +} + +// FormatOrphansWarning renders the orphan list as a multi-line +// customer-facing warning. Returns empty string when orphans is +// empty — caller can blind-print without an "if len > 0" check. +// +// The "delete with" hint is the actionable part. v0.1 doesn't +// auto-delete because: +// +// 1. We can't tell from labels alone whether the Pod is stuck +// mid-transfer for a still-live parallel push from another +// workstation, vs truly orphaned from a crash. +// 2. Deleting someone else's in-progress push silently is bad +// UX. A warn-only approach respects the principle of least +// surprise. +// +// v0.2 can layer on `--cleanup-orphans` for ops folks who want it +// automated. +func FormatOrphansWarning(orphans []Orphan) string { + if len(orphans) == 0 { + return "" + } + var s string + s += fmt.Sprintf("WARNING: %d orphan stage Pod%s detected in this namespace — likely "+ + "leftover from a previously crashed `dataset push`:\n", + len(orphans), pluralS(len(orphans))) + names := make([]string, 0, len(orphans)) + for _, o := range orphans { + tableHint := "" + if o.Table != "" { + tableHint = fmt.Sprintf(" (table: %s)", o.Table) + } + s += fmt.Sprintf(" - %s, age %s%s\n", o.Name, humanDuration(o.Age), tableHint) + names = append(names, o.Name) + } + // Use SPECIFIC Pod names in the delete command, not the + // label selector. The selector would match every stage Pod + // in the namespace, including legitimate running ones from + // parallel pushes (this workstation or another's) — copy- + // pasting a label-based delete could silently kill someone + // else's in-progress push. Bugbot flagged the over-broad + // delete on PR-b round 7. + s += "Delete with: kubectl delete pod -n " + orphans[0].Namespace + " " + + strings.Join(names, " ") + "\n" + return s +} + +// pluralS returns "s" for n != 1, else "". Local helper to keep +// internal/push self-contained (the cli package has its own copy +// for the same reason — see cli/ingest.go's plural()). +func pluralS(n int) string { + if n == 1 { + return "" + } + return "s" +} + +// humanDuration is a coarse "X hours" / "X minutes" / "X seconds" +// formatter for orphan warnings. time.Duration.String() returns +// "2h13m45s" which is technically more precise but harder to read +// in a one-line warning. +func humanDuration(d time.Duration) string { + switch { + case d >= time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + case d >= time.Minute: + return fmt.Sprintf("%dm", int(d.Minutes())) + default: + return fmt.Sprintf("%ds", int(d.Seconds())) + } +} diff --git a/internal/push/orphan_test.go b/internal/push/orphan_test.go new file mode 100644 index 0000000..271f086 --- /dev/null +++ b/internal/push/orphan_test.go @@ -0,0 +1,220 @@ +package push + +import ( + "context" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// stagePodWithAge constructs a stage-labeled Pod whose +// creationTimestamp is `age` ago. Used to seed the fake clientset +// for each orphan-test variant. Default phase is "" (unscheduled) — +// helpers below construct Running/etc explicitly. +func stagePodWithAge(name, table string, age time.Duration) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "tracebloc", + CreationTimestamp: metav1.NewTime(time.Now().Add(-age)), + Labels: map[string]string{ + StagePodManagedByLabel: StagePodManagedByValue, + StagePodComponentLabel: StagePodComponentValue, + StagePodTableLabel: table, + }, + }, + } +} + +// runningStagePodWithAge constructs a stage-labeled Pod in +// Phase=Running. Used to assert the Bugbot-r7 false-positive fix: +// a Running Pod is presumed to be an active push (this or +// another workstation's) and must NOT be flagged as an orphan +// regardless of age. +func runningStagePodWithAge(name, table string, age time.Duration) *corev1.Pod { + p := stagePodWithAge(name, table, age) + p.Status.Phase = corev1.PodRunning + return p +} + +// TestFindOrphanStagePods_NoOrphans: empty cluster returns empty, +// no error. +func TestFindOrphanStagePods_NoOrphans(t *testing.T) { + cs := fake.NewClientset() + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("len(orphans) = %d, want 0", len(got)) + } +} + +// TestFindOrphanStagePods_RecentPodFiltered: a Pod just created +// (well within OrphanGracePeriod) is silently filtered out — it +// might be the current invocation's own Pod from a parallel +// workstation push. +func TestFindOrphanStagePods_RecentPodFiltered(t *testing.T) { + cs := fake.NewClientset(stagePodWithAge("fresh", "t1", 30*time.Second)) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("recent Pod surfaced as orphan; got %d, want 0", len(got)) + } +} + +// TestFindOrphanStagePods_StaleSurfaces: a Pod past the grace +// period surfaces with its metadata intact (name, table, age). +func TestFindOrphanStagePods_StaleSurfaces(t *testing.T) { + cs := fake.NewClientset(stagePodWithAge("stale-cats", "cats_dogs", 30*time.Minute)) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 1 { + t.Fatalf("len(orphans) = %d, want 1", len(got)) + } + o := got[0] + if o.Name != "stale-cats" { + t.Errorf("Name = %q, want stale-cats", o.Name) + } + if o.Table != "cats_dogs" { + t.Errorf("Table = %q, want cats_dogs", o.Table) + } + if o.Age < 29*time.Minute { + t.Errorf("Age = %v, want at least 29m", o.Age) + } +} + +// TestFindOrphanStagePods_IgnoresNonStagePods: the chart's own +// Pods (managed-by=Helm) and arbitrary user Pods must NOT show +// up — the label selector is the safety boundary. +func TestFindOrphanStagePods_IgnoresNonStagePods(t *testing.T) { + chartPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "jobs-manager-abc", + Namespace: "tracebloc", + CreationTimestamp: metav1.NewTime(time.Now().Add(-1 * time.Hour)), + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "Helm", + "app.kubernetes.io/name": "client", + }, + }, + } + customerPod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "their-app", + Namespace: "tracebloc", + CreationTimestamp: metav1.NewTime(time.Now().Add(-1 * time.Hour)), + }, + } + cs := fake.NewClientset(chartPod, customerPod) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("non-stage Pods surfaced as orphans; got %d, want 0; names=%v", len(got), got) + } +} + +// TestFormatOrphansWarning_Empty returns the empty string so the +// caller can blind-print without conditionals. +func TestFormatOrphansWarning_Empty(t *testing.T) { + if got := FormatOrphansWarning(nil); got != "" { + t.Errorf("FormatOrphansWarning(nil) = %q, want empty", got) + } +} + +// TestFormatOrphansWarning_ActionableHint: the warning must contain +// (a) the count, (b) the per-pod name + age + table, (c) a +// kubectl delete command using SPECIFIC POD NAMES (not the label +// selector — Bugbot r7 flagged that the label selector would +// match every stage Pod including parallel-push's still-running +// ones). +func TestFormatOrphansWarning_ActionableHint(t *testing.T) { + orphans := []Orphan{ + {Name: "stage-a", Namespace: "tracebloc", Table: "cats_dogs", Age: 30 * time.Minute}, + {Name: "stage-b", Namespace: "tracebloc", Table: "xrays", Age: 2 * time.Hour}, + } + got := FormatOrphansWarning(orphans) + for _, want := range []string{ + "2 orphan stage Pods", + "stage-a", "cats_dogs", "30m", + "stage-b", "xrays", "2h", + // Targeted delete — specific names, not the label + // selector. The label-selector version would nuke + // running pods from parallel pushes. + "kubectl delete pod -n tracebloc stage-a stage-b", + } { + if !strings.Contains(got, want) { + t.Errorf("warning missing %q in:\n%s", want, got) + } + } + // Regression check: the label selector MUST NOT appear in + // the delete command. If a refactor reintroduces it, + // customers could accidentally nuke their parallel-push Pod. + if strings.Contains(got, "kubectl delete pod -n tracebloc -l ") { + t.Errorf("warning uses label-selector delete (would catch parallel pushes):\n%s", got) + } +} + +// TestFindOrphanStagePods_SkipsRunningPods is the Bugbot-r7 +// false-positive fix: a Pod in Phase=Running is presumed to be an +// active push (this workstation or another's still doing work), +// and must NOT be flagged as orphan regardless of age. Near-cap +// 1 GiB pushes legitimately take >5 min (pod.go budgets ~8.5 min +// for the stream alone), so without this guard the next concurrent +// push would warn about Pods that are still actively transferring. +func TestFindOrphanStagePods_SkipsRunningPods(t *testing.T) { + // 30-minute-old Running Pod — well past OrphanGracePeriod, + // but Running means it's an active push. + cs := fake.NewClientset(runningStagePodWithAge("active-slow-push", + "big_table", 30*time.Minute)) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 0 { + t.Errorf("Running Pod flagged as orphan; got %d, want 0", len(got)) + } +} + +// TestFindOrphanStagePods_FlagsNonRunningPastGrace: complement to +// the Running-skip — a Failed or Pending Pod past the grace period +// IS still an orphan (crashed-at-startup case, stuck-on-image-pull +// case). The Running-skip should narrow the false positives, not +// eliminate the warning entirely. +func TestFindOrphanStagePods_FlagsNonRunningPastGrace(t *testing.T) { + failedPod := stagePodWithAge("crashed", "t1", 30*time.Minute) + failedPod.Status.Phase = corev1.PodFailed + cs := fake.NewClientset(failedPod) + got, err := FindOrphanStagePods(context.Background(), cs, "tracebloc") + if err != nil { + t.Fatalf("FindOrphanStagePods: %v", err) + } + if len(got) != 1 { + t.Errorf("Failed Pod past grace not flagged; got %d, want 1", len(got)) + } +} + +// TestFormatOrphansWarning_NoTableLabel: very old orphans from +// before we added StagePodTableLabel might not have it. The +// formatter must not crash or render "(table: )". +func TestFormatOrphansWarning_NoTableLabel(t *testing.T) { + got := FormatOrphansWarning([]Orphan{ + {Name: "old-orphan", Namespace: "tracebloc", Age: 1 * time.Hour}, + }) + if strings.Contains(got, "(table: )") { + t.Errorf("warning rendered empty table parenthetical:\n%s", got) + } + if !strings.Contains(got, "old-orphan") { + t.Errorf("warning missing Pod name:\n%s", got) + } +} diff --git a/internal/push/pod.go b/internal/push/pod.go new file mode 100644 index 0000000..fb28e6d --- /dev/null +++ b/internal/push/pod.go @@ -0,0 +1,528 @@ +package push + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +// DefaultStagePodImage is the alpine image the ephemeral stage Pod +// runs. Pinned by digest at CLI build time so a customer pulling +// v0.1.x at any future date gets bit-for-bit identical behavior — +// no "alpine just shipped a glibc-on-musl regression and now my +// stage Pod crashloops" surprises. +// +// alpine:3.20 was the current 3.x stable when Phase 3 PR-b landed +// (May 2026). The image is tiny (~8 MiB), has tar/sh/busybox built +// in, and is one of the most-mirrored images on the planet — air- +// gapped customers can almost always pull it from their internal +// mirror without extra config. +// +// Override via `tracebloc dataset push --stage-pod-image=...` for: +// +// - Customers on registries that don't proxy docker.io +// - Customers running a curated base image with extra audit +// instrumentation +// - Custom forks that need a specific busybox build +// +// Bumping the pinned digest is a v0.2 task; track it alongside +// the kube-deps refresh. +const DefaultStagePodImage = "alpine:3.20@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc" + +// StagePodLabel{Key,Value} mark every Pod the CLI creates so the +// orphan-scan logic (orphan.go) can find leftover Pods from a +// previously crashed `dataset push` invocation. Using the +// kubernetes.io/managed-by label means the chart's own resources +// (managed-by=Helm) don't get caught in our scan. +const ( + StagePodManagedByLabel = "app.kubernetes.io/managed-by" + StagePodManagedByValue = "tracebloc-cli" + + // StagePodComponentLabel narrows the orphan scan to just stage + // Pods (not, e.g., a future `tracebloc cluster doctor` Pod). + StagePodComponentLabel = "tracebloc.io/component" + StagePodComponentValue = "stage-pod" + + // StagePodTableLabel records which table the Pod was staging + // for. Useful in the orphan-warning output so the customer + // knows which `dataset push` invocation it came from. + StagePodTableLabel = "tracebloc.io/table" +) + +// StagePodActiveDeadline is the in-cluster self-kill timer. The CLI +// also deletes the Pod via defer + SIGINT handler, but those only +// fire if the CLI process is still alive — a hard kill (`kill -9`, +// OS crash, network partition between laptop and cluster) leaves +// the Pod stranded. activeDeadlineSeconds means the kubelet kills +// it even if the CLI never gets the chance. +// +// 30 minutes covers the full lifecycle, not just the stream: +// +// ~60s image pull on a fresh node + scheduler back-pressure +// ~60s WaitForStagePodReady ceiling (StagePodReadyTimeout) +// ~8.5m 1 GiB transfer at a conservative 2 MB/s +// + comfortable margin for variance +// +// activeDeadlineSeconds starts the clock at Pod CREATION (not at +// streaming start), so we have to budget for the pre-stream +// readiness portion too — the earlier 10-minute value cut it too +// fine for near-cap customers on slow uplinks (kubelet would +// terminate mid-transfer). Bugbot flagged the squeeze as High on +// PR-b round 6. +// +// 30 min is generous but cheap — an idle alpine Pod with `sleep` +// consumes ~5 MiB RAM and zero CPU on the cluster. v0.2 should +// make this configurable per push. +const StagePodActiveDeadline = 1800 + +// StagePodReadyTimeout is how long we wait for the Pod to become +// Running + Ready after CREATE. Most clusters spawn an alpine Pod +// in 5-15 seconds; 60 seconds covers image-pull from a slow mirror +// + scheduler back-pressure on busy clusters. Beyond that, something +// is wrong (no image-pull access, no schedulable node, PSP/PSA +// rejection) and the customer wants the diagnostic, not a longer +// wait. +const StagePodReadyTimeout = 60 * time.Second + +// PodSpecOptions controls the ephemeral stage Pod construction. +// Fields are intentionally narrow — every knob is one a customer +// could plausibly need to turn for an air-gapped or hardened- +// security setup. Adding fields should require a real use case. +type PodSpecOptions struct { + // Namespace is where the Pod gets created — always the + // discovered parent-release namespace. + Namespace string + + // PVCClaimName is the shared PVC to mount at /data/shared. + // Discovered by cluster.DiscoverSharedPVC (always "client-pvc" + // today, but routed through a field so a future per-customer + // override doesn't require touching this signature). + PVCClaimName string + + // PVCMountPath is where to mount the PVC inside the Pod — + // "/data/shared" by chart convention. + PVCMountPath string + + // Table is the destination table name. Used to compose the Pod + // name and the on-PVC subdirectory. MUST have already passed + // ValidateTableName; pod-name composition relies on the same + // character-class restrictions. + Table string + + // Image overrides DefaultStagePodImage. Empty = use default. + Image string + + // ServiceAccountName is the SA the Pod runs as. Phase 2's + // discovery surfaces this as `ingestor` (the chart default) or + // whatever the customer's `--ingestor-sa` flag overrides to. + // Using the chart's existing SA means the Pod inherits any + // imagePullSecrets and PSA exemptions the admin already + // configured for it. + ServiceAccountName string +} + +// BuildStagePodSpec produces the corev1.Pod for an ephemeral stage +// Pod, fully parametrized but with no cluster side-effects. +// Separated from CreateStagePod so unit tests can assert the spec +// shape without needing a fake clientset for every assertion. +// +// Returns an error only when crypto/rand fails — which is rare but +// possible on systems with exhausted entropy. Earlier versions +// swallowed that error and produced a Pod name ending with a bare +// trailing hyphen (DNS-1123 violation → opaque API server +// rejection). Bugbot flagged that as Low on PR-b; surfacing the +// error here turns "weird API error message" into "clear local +// diagnostic at the call site." +// +// Security context follows the Kubernetes Pod Security Standards +// "restricted" profile — the strictest preset, accepted on every +// PSA-enabled namespace including the chart's recommended config. +// This is intentional: the stage Pod runs in the customer's +// namespace and writes to their PVC, so being a model citizen for +// PSA defaults reduces "the Pod won't even start on my cluster" +// surface area. +func BuildStagePodSpec(opts PodSpecOptions) (*corev1.Pod, error) { + image := opts.Image + if image == "" { + image = DefaultStagePodImage + } + + suffix, err := randomSuffix(4) // 4 bytes → 8 hex chars + if err != nil { + return nil, fmt.Errorf("generating Pod-name random suffix: %w", err) + } + // Transform the table name into a DNS-1123 subdomain-safe + // segment for the Pod name. ValidateTableName accepts + // [A-Za-z0-9_]+ (MySQL identifier rules), but Kubernetes Pod + // names follow DNS-1123 — lowercase + alphanumeric + hyphen + // only, must start/end with alphanumeric. Without this + // transform, the dominant canonical example (cats_dogs_train, + // snake_case throughout the tracebloc docs) would fail Pod + // creation post-pre-flight, which is a worst-of-both-worlds + // UX (the pre-flight summary says "we're good!" then the + // create fails). Bugbot flagged the gap as High on PR-b. + // + // The original (un-transformed) table name is preserved + // verbatim in the tracebloc.io/table label below, so orphan + // warnings still surface the customer-facing identifier. + podName := fmt.Sprintf("tracebloc-stage-%s-%s", + dns1123SafeTableSegment(opts.Table), suffix) + + // Pod-level security context: runAsNonRoot is the only field + // PSA's restricted profile *requires* at the Pod level (the + // rest are container-level). Setting it here means every + // future container (if we ever add one) inherits the + // non-root constraint. + runAsNonRoot := true + runAsUser := int64(65532) // distroless's "nonroot" UID; works on any cluster + + // Container-level security context: every PSA-restricted + // requirement. Reads top-to-bottom: capabilities dropped, + // read-only root FS, no privilege escalation, no privileged, + // seccomp default. None of these prevent tar from working — + // tar reads stdin + writes to the PVC mount (which is RW), + // and doesn't need any caps to run as a non-root user. + allowPrivEsc := false + privileged := false + readOnlyRootFS := true + + activeDeadline := int64(StagePodActiveDeadline) + + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: podName, + Namespace: opts.Namespace, + Labels: map[string]string{ + StagePodManagedByLabel: StagePodManagedByValue, + StagePodComponentLabel: StagePodComponentValue, + StagePodTableLabel: opts.Table, + }, + Annotations: map[string]string{ + // Annotations are searchable via `kubectl describe` + // but don't constrain scheduling — perfect for + // breadcrumbs that help post-mortem an orphan. + "tracebloc.io/created-at": time.Now().UTC().Format(time.RFC3339), + "tracebloc.io/created-by": "tracebloc-cli", + }, + }, + Spec: corev1.PodSpec{ + ServiceAccountName: opts.ServiceAccountName, + RestartPolicy: corev1.RestartPolicyNever, + + // 10-min in-cluster self-kill — see comment on the const. + ActiveDeadlineSeconds: &activeDeadline, + + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: &runAsNonRoot, + RunAsUser: &runAsUser, + FSGroup: &runAsUser, // PVC writes need this group + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + + Containers: []corev1.Container{{ + Name: "stage", + Image: image, + // `sleep` keeps the Pod alive long enough for the + // CLI to open an exec stream and tar files in. + // activeDeadlineSeconds caps the worst case; the + // CLI deletes the Pod the moment the stream + // finishes (or fails). + // + // Why not the busybox `sleep infinity` idiom: it + // causes some PSA configurations to flag the Pod + // because the container holds the SIGTERM until + // activeDeadlineSeconds fires (sleep traps signals + // imperfectly). A finite sleep matched to the + // deadline is gentler. + Command: []string{"/bin/sleep", fmt.Sprintf("%d", StagePodActiveDeadline)}, + + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &allowPrivEsc, + Privileged: &privileged, + ReadOnlyRootFilesystem: &readOnlyRootFS, + RunAsNonRoot: &runAsNonRoot, + RunAsUser: &runAsUser, + Capabilities: &corev1.Capabilities{ + // PSA restricted requires ALL capabilities + // dropped, then optionally add back + // NET_BIND_SERVICE. tar doesn't need it. + Drop: []corev1.Capability{"ALL"}, + }, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + }, + + VolumeMounts: []corev1.VolumeMount{{ + Name: "shared", + MountPath: opts.PVCMountPath, + }, { + // tar needs a writable working dir for its + // temporary state; with ReadOnlyRootFilesystem + // it can't use /tmp on the root FS. An + // emptyDir at /tmp is the standard pattern. + Name: "tmp", + MountPath: "/tmp", + }}, + + // Conservative resource requests: tar of typical + // image_classification data uses <100 MiB RAM and + // negligible CPU. Setting requests lets the + // scheduler place us; setting limits keeps a + // runaway tar (e.g. corrupted archive triggering + // a busy loop) from impacting the cluster. + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + }, + }}, + + Volumes: []corev1.Volume{{ + Name: "shared", + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: opts.PVCClaimName, + }, + }, + }, { + Name: "tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }}, + }, + }, nil +} + +// CreateStagePod creates the Pod in the cluster and returns the +// metadata.name the API server assigned. The returned name is what +// callers pass to WaitForStagePodReady + DeleteStagePod. +// +// Take-name-from-API-response is deliberate: even though +// BuildStagePodSpec pre-computes a name, the API server can in +// principle rewrite it (e.g. via a mutating admission webhook +// renaming with a prefix). Reading back from the response is the +// safe contract. +func CreateStagePod(ctx context.Context, cs kubernetes.Interface, opts PodSpecOptions) (string, error) { + pod, err := BuildStagePodSpec(opts) + if err != nil { + return "", err + } + created, err := cs.CoreV1().Pods(opts.Namespace).Create(ctx, pod, metav1.CreateOptions{}) + if err != nil { + // Most-common failure path: PSA rejects the Pod because + // the customer's namespace policy is stricter than even + // "restricted." We surface the raw API error so the customer + // sees the PSA violation list verbatim — that's the most + // actionable thing. + return "", fmt.Errorf("creating stage Pod in namespace %q: %w", opts.Namespace, err) + } + return created.Name, nil +} + +// WaitForStagePodReady polls until the Pod's status reports +// Ready=True (the canonical "containers started and not yet +// terminated" signal). Times out per StagePodReadyTimeout with a +// diagnostic that tries to help — image-pull failures and +// scheduler back-pressure are the dominant slow paths and have +// distinct symptom strings. +// +// Returns nil on Ready, the pod object on success so callers don't +// have to re-Get for it. +func WaitForStagePodReady(ctx context.Context, cs kubernetes.Interface, namespace, podName string) (*corev1.Pod, error) { + var lastObserved *corev1.Pod + + // Poll every 1s — image-pull failures surface in seconds, and + // the Ready transition itself is instant once kubelet has + // pulled+started. Tighter polling burns API server CPU for no + // benefit; looser polling delays the user. + err := wait.PollUntilContextTimeout(ctx, 1*time.Second, StagePodReadyTimeout, true, + func(ctx context.Context) (bool, error) { + p, err := cs.CoreV1().Pods(namespace).Get(ctx, podName, metav1.GetOptions{}) + if err != nil { + // Distinguish terminal vs transient. Terminal + // errors (Pod deleted out-of-band, RBAC revoked + // to read Pods) MUST short-circuit the poll — + // otherwise the customer waits the full 60s + // timeout for a condition that won't change. + // Bugbot flagged the prior "everything transient" + // version as Medium on PR-b. + if apierrors.IsNotFound(err) || apierrors.IsForbidden(err) { + return false, err + } + // Transient (network blip, brief API unavail). + // Keep polling; last-observed context survives. + return false, nil + } + lastObserved = p + // Positive terminal: Ready=True. + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodReady && c.Status == corev1.ConditionTrue { + return true, nil + } + } + // Negative terminal: Phase=Failed (container crashed + // at startup — PSA rejection, image crashloop, OOM) + // or Phase=Succeeded (sleep exited unexpectedly — the + // stage container shouldn't terminate by itself before + // we exec into it, so this is also a failure mode). + // Without these checks the poll waits the full 60s + // for a Pod that will never become Ready. Bugbot + // flagged on PR-b round 4 as Medium; this is the + // counterpart to the NotFound/Forbidden short-circuit + // landed in the previous commit. + if p.Status.Phase == corev1.PodFailed || p.Status.Phase == corev1.PodSucceeded { + return false, fmt.Errorf( + "stage Pod %s/%s terminated in phase %q before becoming Ready%s", + namespace, podName, p.Status.Phase, podReadyTimeoutHint(p)) + } + return false, nil + }) + + if err == nil { + return lastObserved, nil + } + + // Differentiate "actual timeout expired" from "poll terminated + // early" (NotFound, Forbidden, Failed phase, ctx-canceled). + // The earlier blanket "did not become Ready within 60s" + // wording was misleading — Bugbot flagged on PR-b round 7. + hint := podReadyTimeoutHint(lastObserved) + if errors.Is(err, context.DeadlineExceeded) { + return nil, fmt.Errorf( + "stage Pod %s/%s did not become Ready within %s%s", + namespace, podName, StagePodReadyTimeout, hint) + } + // Early-exit error (terminal API error, terminal phase, + // SIGINT). Surface it as-is so the customer sees the actual + // cause without the wrong "ran out the timer" framing. + return nil, fmt.Errorf( + "stage Pod %s/%s did not reach Ready state: %w%s", + namespace, podName, err, hint) +} + +// podReadyTimeoutHint extracts the most useful diagnostic from a +// last-observed Pod status. Designed to match against the two +// dominant slow-path scenarios: +// +// 1. Image pull is slow or failed (ImagePullBackOff, +// ErrImagePull) — surface the registry message. +// 2. Pod is unschedulable (PodScheduled=False with reason) — +// surface the scheduler's message. +func podReadyTimeoutHint(p *corev1.Pod) string { + if p == nil { + return " (no Pod status observed — check API server connectivity)" + } + for _, cs := range p.Status.ContainerStatuses { + if cs.State.Waiting != nil && cs.State.Waiting.Reason != "" { + return fmt.Sprintf(" (last container state: %s — %s)", + cs.State.Waiting.Reason, cs.State.Waiting.Message) + } + } + for _, c := range p.Status.Conditions { + if c.Type == corev1.PodScheduled && c.Status == corev1.ConditionFalse { + return fmt.Sprintf(" (scheduling: %s — %s)", c.Reason, c.Message) + } + } + return fmt.Sprintf(" (Pod phase: %s)", p.Status.Phase) +} + +// DeleteStagePod removes the Pod. Called via defer in the orchestrator +// so it runs on both success and failure (including SIGINT, which +// is wired up to cancel the parent context and let the defer fire +// in normal stack unwind). +// +// Uses background propagation + a tiny grace period because we +// don't care about giving sleep a graceful shutdown — we just want +// the Pod gone so the next push doesn't see an orphan. +func DeleteStagePod(ctx context.Context, cs kubernetes.Interface, namespace, podName string) error { + gracePeriod := int64(0) + propagation := metav1.DeletePropagationBackground + err := cs.CoreV1().Pods(namespace).Delete(ctx, podName, metav1.DeleteOptions{ + GracePeriodSeconds: &gracePeriod, + PropagationPolicy: &propagation, + }) + if err != nil && !apierrors.IsNotFound(err) { + // Not-found is fine: the Pod might have already self-killed + // via activeDeadlineSeconds, or a parallel `kubectl delete` + // got there first. Either way, our goal (no orphan) is + // achieved. Returning the error here would mask the + // upstream cause of failure that triggered the defer. + return fmt.Errorf("deleting stage Pod %s/%s: %w", namespace, podName, err) + } + return nil +} + +// dns1123SafeTableSegment transforms a ValidateTableName-passed +// table name into a Kubernetes-Pod-name-compatible path segment. +// The Pod's full name is then `tracebloc-stage--<8hex>`, +// which must satisfy DNS-1123 subdomain rules: +// +// [a-z0-9]([-a-z0-9]*[a-z0-9])? +// +// (lowercase + digit + hyphen, start/end alphanumeric). +// +// Our input alphabet is [A-Za-z0-9_], so the transform is: +// +// 1. Lowercase (DNS-1123 forbids uppercase) +// 2. Replace '_' with '-' (DNS-1123 forbids underscore) +// 3. Strip leading/trailing hyphens (a name like "_leading" +// would otherwise become "-leading" → tracebloc-stage--leading +// which is OK in the middle but ugly) +// 4. Cap length at 30 chars — Pod names are bounded at 63 total, +// and "tracebloc-stage-" + 8-hex-suffix already consumes ~25 +// of those, leaving ~38 for the segment. 30 gives margin. +// 5. Fallback to "tbl" if the transform leaves an empty string +// (the pathological "_"-only-name case). +func dns1123SafeTableSegment(table string) string { + s := strings.ToLower(strings.ReplaceAll(table, "_", "-")) + s = strings.Trim(s, "-") + if len(s) > 30 { + s = s[:30] + // Truncation could leave a trailing hyphen — re-trim. + s = strings.TrimRight(s, "-") + } + if s == "" { + // Pathological all-underscore name. The label still + // carries the original, so customers can still trace + // orphan-Pod warnings back to their push. + s = "tbl" + } + return s +} + +// randomSuffix returns a hex string of length 2*n. Used to make +// stage Pod names unique across parallel `dataset push` invocations +// for the same table — without this, two CLI runs would race on the +// same Pod name and one would fail the Create with AlreadyExists. +// +// Cryptographic randomness is overkill for collision avoidance +// (8 hex chars = 32 bits of entropy is way more than needed) but +// crypto/rand is the simpler import compared to math/rand which +// needs explicit seeding. +func randomSuffix(n int) (string, error) { + b := make([]byte, n) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil +} diff --git a/internal/push/pod_test.go b/internal/push/pod_test.go new file mode 100644 index 0000000..6e58715 --- /dev/null +++ b/internal/push/pod_test.go @@ -0,0 +1,560 @@ +package push + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// stageOpts is the minimal PodSpecOptions for happy-path tests — +// individual cases mutate one field to exercise their specific +// branch. +func stageOpts() PodSpecOptions { + return PodSpecOptions{ + Namespace: "tracebloc", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Table: "cats_dogs", + ServiceAccountName: "ingestor", + } +} + +// mustBuildStagePod wraps BuildStagePodSpec for tests that don't +// care about the (rare) crypto/rand failure path — failing the +// test if it ever fires is the right escape from "if err != nil" +// noise in every assertion. The dedicated TestBuildStagePodSpec_ +// PropagatesRandError below covers the error path directly. +func mustBuildStagePod(t *testing.T, opts PodSpecOptions) *corev1.Pod { + t.Helper() + p, err := BuildStagePodSpec(opts) + if err != nil { + t.Fatalf("BuildStagePodSpec: %v", err) + } + return p +} + +// TestDNS1123SafeTableSegment is the High-severity regression pin +// for the Bugbot finding on PR-b. Every realistic table name in +// tracebloc docs uses snake_case (cats_dogs_train, chest_xrays_train); +// without the transform these would produce Pod names K8s rejects +// post-pre-flight, which is the worst-of-both-worlds UX. The +// transform's job: take any ValidateTableName-passed name and +// produce a DNS-1123 subdomain-safe segment for the Pod name. +func TestDNS1123SafeTableSegment(t *testing.T) { + cases := []struct { + in, want string + }{ + // Canonical happy path: snake_case → kebab-case. + {"cats_dogs", "cats-dogs"}, + {"chest_xrays_train", "chest-xrays-train"}, + // Uppercase: must lowercase. + {"MyTable", "mytable"}, + {"ABC", "abc"}, + // Already lowercase no-underscore: identity. + {"single", "single"}, + // Mixed: lowercase + underscore → hyphen. + {"Table_123", "table-123"}, + // Edge: leading underscore must be stripped (would + // otherwise produce "-leading", and while the full Pod + // name `tracebloc-stage--leading-` is valid DNS-1123 + // the consecutive hyphens are ugly). + {"_leading_underscore", "leading-underscore"}, + // Edge: trailing underscore stripped. + {"trailing_", "trailing"}, + // Edge: digit-led is valid (DNS-1123 allows it). + {"9starts_with_digit", "9starts-with-digit"}, + // Truncation: input > 30 chars caps at 30, then trims + // any trailing hyphen the cut might have left. + {strings.Repeat("a", 50), strings.Repeat("a", 30)}, + // Pathological: all-underscores → fallback "tbl". + {"_", "tbl"}, + {"___", "tbl"}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + got := dns1123SafeTableSegment(c.in) + if got != c.want { + t.Errorf("dns1123SafeTableSegment(%q) = %q, want %q", c.in, got, c.want) + } + // Defensive: the result must satisfy DNS-1123 + // subdomain rules for a Pod-name segment. Cheap + // regex check covers the contract. + if !isDNS1123SafeSegment(got) { + t.Errorf("dns1123SafeTableSegment(%q) = %q, which violates DNS-1123", + c.in, got) + } + }) + } +} + +// isDNS1123SafeSegment is a test-only helper: returns true iff s +// matches the DNS-1123 subdomain segment regex (lowercase alnum + +// hyphen, must start AND end with alnum). Matches what the K8s +// validation library checks for Pod names. +func isDNS1123SafeSegment(s string) bool { + if s == "" { + return false + } + for i, r := range s { + switch { + case r >= 'a' && r <= 'z': + case r >= '0' && r <= '9': + case r == '-': + // Hyphen forbidden at boundaries. + if i == 0 || i == len(s)-1 { + return false + } + default: + return false + } + } + return true +} + +// TestBuildStagePodSpec_Defaults pins the spec fields PR-b's +// post-create logic depends on: the SA name, the PVC mount, the +// activeDeadline, and the labels orphan.go keys off. +func TestBuildStagePodSpec_Defaults(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + + if p.Namespace != "tracebloc" { + t.Errorf("Namespace = %q, want tracebloc", p.Namespace) + } + // Pod name uses the DNS-1123-safe transform of the table name: + // "cats_dogs" → "cats-dogs". Bugbot flagged the raw-name version + // on PR-b as High severity (snake_case is the canonical naming + // style in tracebloc docs but K8s Pod names reject underscores). + if !strings.HasPrefix(p.Name, "tracebloc-stage-cats-dogs-") { + t.Errorf("Name = %q, want prefix tracebloc-stage-cats-dogs-", p.Name) + } + if got, wantLen := len(p.Name), len("tracebloc-stage-cats-dogs-")+8; got != wantLen { + t.Errorf("len(Name) = %d, want %d (8-hex-char random suffix)", got, wantLen) + } + if p.Spec.ServiceAccountName != "ingestor" { + t.Errorf("ServiceAccountName = %q, want ingestor", p.Spec.ServiceAccountName) + } + if p.Spec.RestartPolicy != corev1.RestartPolicyNever { + t.Errorf("RestartPolicy = %q, want Never", p.Spec.RestartPolicy) + } + if p.Spec.ActiveDeadlineSeconds == nil || *p.Spec.ActiveDeadlineSeconds != int64(StagePodActiveDeadline) { + t.Errorf("ActiveDeadlineSeconds = %v, want %d", p.Spec.ActiveDeadlineSeconds, StagePodActiveDeadline) + } +} + +// TestStagePodActiveDeadline_CoversFullLifecycle pins the floor on +// activeDeadlineSeconds. The earlier 600s value was too tight: the +// timer starts at Pod CREATION, but image pull (up to 60s) + +// readiness wait (up to 60s) + the worst-case stream (1 GiB at +// 2 MB/s = ~8.5 min) leaves no margin, so the kubelet could +// terminate near-cap pushes mid-transfer. Bugbot flagged as High +// on PR-b round 6. This test pins the floor at 1500s (25 min) so +// a future regression bumping it back near 600 gets caught. +func TestStagePodActiveDeadline_CoversFullLifecycle(t *testing.T) { + const minDeadline = 1500 + if StagePodActiveDeadline < minDeadline { + t.Errorf("StagePodActiveDeadline = %d, want at least %d "+ + "(must cover image-pull + readiness + worst-case 1 GiB stream "+ + "+ margin; activeDeadline starts at Pod creation not stream start)", + StagePodActiveDeadline, minDeadline) + } +} + +// TestBuildStagePodSpec_DefaultImage pins the digest-pinned image — +// if this ever drifts to a tag-only reference, air-gapped customers +// would silently get whatever alpine:3.20 resolved to that day. +func TestBuildStagePodSpec_DefaultImage(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + got := p.Spec.Containers[0].Image + if got != DefaultStagePodImage { + t.Errorf("Image = %q, want %q (default)", got, DefaultStagePodImage) + } + if !strings.Contains(got, "@sha256:") { + t.Errorf("Image = %q, want digest-pinned (@sha256:...); air-gapped customers depend on this", got) + } +} + +// TestBuildStagePodSpec_OverrideImage pins the --stage-pod-image +// flag's contract end-to-end at the spec layer. +func TestBuildStagePodSpec_OverrideImage(t *testing.T) { + opts := stageOpts() + opts.Image = "internal-mirror.example.com/alpine:3.20@sha256:abc123" + p := mustBuildStagePod(t, opts) + if got := p.Spec.Containers[0].Image; got != opts.Image { + t.Errorf("Image = %q, want override %q", got, opts.Image) + } +} + +// TestBuildStagePodSpec_LabelsForOrphanScan pins the labels orphan.go +// will key off. If these ever drift, orphan-pod detection silently +// misses leftover Pods from crashed pushes. +func TestBuildStagePodSpec_LabelsForOrphanScan(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + wantLabels := map[string]string{ + StagePodManagedByLabel: StagePodManagedByValue, + StagePodComponentLabel: StagePodComponentValue, + StagePodTableLabel: "cats_dogs", + } + for k, want := range wantLabels { + if got := p.Labels[k]; got != want { + t.Errorf("Label %s = %q, want %q", k, got, want) + } + } +} + +// TestBuildStagePodSpec_RestrictedPSA is the security-regression +// pin: every PSA-restricted requirement must hold, otherwise the +// stage Pod gets rejected on hardened namespaces (which is the +// majority of production tracebloc deployments). +func TestBuildStagePodSpec_RestrictedPSA(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + + // Pod-level: runAsNonRoot, seccomp RuntimeDefault. + psc := p.Spec.SecurityContext + if psc == nil { + t.Fatal("Pod has no SecurityContext") + } + if psc.RunAsNonRoot == nil || !*psc.RunAsNonRoot { + t.Errorf("Pod.SecurityContext.RunAsNonRoot = %v, want true", psc.RunAsNonRoot) + } + if psc.SeccompProfile == nil || psc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Errorf("Pod.SeccompProfile = %v, want RuntimeDefault", psc.SeccompProfile) + } + + // Container-level: every PSA restricted constraint. + if len(p.Spec.Containers) != 1 { + t.Fatalf("len(Containers) = %d, want 1", len(p.Spec.Containers)) + } + c := p.Spec.Containers[0] + sc := c.SecurityContext + if sc == nil { + t.Fatal("Container has no SecurityContext") + } + if sc.AllowPrivilegeEscalation == nil || *sc.AllowPrivilegeEscalation { + t.Errorf("AllowPrivilegeEscalation = %v, want false", sc.AllowPrivilegeEscalation) + } + if sc.Privileged == nil || *sc.Privileged { + t.Errorf("Privileged = %v, want false", sc.Privileged) + } + if sc.ReadOnlyRootFilesystem == nil || !*sc.ReadOnlyRootFilesystem { + t.Errorf("ReadOnlyRootFilesystem = %v, want true", sc.ReadOnlyRootFilesystem) + } + if sc.Capabilities == nil || len(sc.Capabilities.Drop) == 0 { + t.Fatalf("Capabilities.Drop = %v, want [ALL]", sc.Capabilities) + } + if sc.Capabilities.Drop[0] != "ALL" { + t.Errorf("Capabilities.Drop = %v, want [ALL]", sc.Capabilities.Drop) + } + if sc.SeccompProfile == nil || sc.SeccompProfile.Type != corev1.SeccompProfileTypeRuntimeDefault { + t.Errorf("Container SeccompProfile = %v, want RuntimeDefault", sc.SeccompProfile) + } +} + +// TestBuildStagePodSpec_PVCMount pins the volume + mountPath that +// the tar stream writes into. If the mount path here ever drifts +// from cluster.SharedPVCMountPath, the tar would write to the +// wrong location and Phase 4's ingestor Job would see "no files." +func TestBuildStagePodSpec_PVCMount(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + + // Volume side: the PVC reference. + var foundVol bool + for _, v := range p.Spec.Volumes { + if v.Name == "shared" { + foundVol = true + if v.PersistentVolumeClaim == nil { + t.Errorf("shared volume has no PVC source") + } else if v.PersistentVolumeClaim.ClaimName != "client-pvc" { + t.Errorf("PVC ClaimName = %q, want client-pvc", v.PersistentVolumeClaim.ClaimName) + } + } + } + if !foundVol { + t.Error("Pod has no shared volume") + } + + // Mount side: where the container sees it. + var foundMount bool + for _, m := range p.Spec.Containers[0].VolumeMounts { + if m.Name == "shared" { + foundMount = true + if m.MountPath != "/data/shared" { + t.Errorf("MountPath = %q, want /data/shared", m.MountPath) + } + } + } + if !foundMount { + t.Error("stage container has no shared mount") + } +} + +// TestBuildStagePodSpec_TmpEmptyDir pins the writable /tmp emptyDir +// that tar needs (since the root FS is read-only). Without this, +// tar would fail to create its working state and the stream would +// die mid-transfer. +func TestBuildStagePodSpec_TmpEmptyDir(t *testing.T) { + p := mustBuildStagePod(t, stageOpts()) + var foundEmptyDir, foundMount bool + for _, v := range p.Spec.Volumes { + if v.Name == "tmp" { + foundEmptyDir = true + if v.EmptyDir == nil { + t.Error("tmp volume is not an EmptyDir") + } + } + } + for _, m := range p.Spec.Containers[0].VolumeMounts { + if m.Name == "tmp" && m.MountPath == "/tmp" { + foundMount = true + } + } + if !foundEmptyDir || !foundMount { + t.Errorf("tmp emptyDir mount missing (vol=%v, mount=%v)", foundEmptyDir, foundMount) + } +} + +// TestBuildStagePodSpec_RandomSuffixCollisionAvoidance: two specs +// built back-to-back must have distinct names so parallel pushes +// don't race on Create. +func TestBuildStagePodSpec_RandomSuffixCollisionAvoidance(t *testing.T) { + a := mustBuildStagePod(t, stageOpts()) + b := mustBuildStagePod(t, stageOpts()) + if a.Name == b.Name { + t.Errorf("back-to-back BuildStagePodSpec produced identical name %q; "+ + "random-suffix collision avoidance is broken", a.Name) + } +} + +// TestCreateStagePod_HappyPath: the fake clientset accepts a Create +// and returns the created object. We pin that CreateStagePod +// surfaces the assigned name back to the caller. +func TestCreateStagePod_HappyPath(t *testing.T) { + cs := fake.NewClientset() + name, err := CreateStagePod(context.Background(), cs, stageOpts()) + if err != nil { + t.Fatalf("CreateStagePod: %v", err) + } + if !strings.HasPrefix(name, "tracebloc-stage-cats-dogs-") { + t.Errorf("returned name = %q, want prefix tracebloc-stage-cats-dogs-", name) + } + // Cross-check: the Pod actually exists in the fake cluster. + if _, err := cs.CoreV1().Pods("tracebloc").Get(context.Background(), name, metav1.GetOptions{}); err != nil { + t.Errorf("Pod not found after CreateStagePod: %v", err) + } +} + +// TestCreateStagePod_APIErrorSurfaces: PSA rejections, RBAC denials, +// etc. surface verbatim so the customer sees the actionable cluster- +// side message. +func TestCreateStagePod_APIErrorSurfaces(t *testing.T) { + cs := fake.NewClientset() + cs.PrependReactor("create", "pods", + func(_ k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + corev1.Resource("pods"), "", + errors.New("user cannot create pods")) + }) + + _, err := CreateStagePod(context.Background(), cs, stageOpts()) + if err == nil { + t.Fatal("CreateStagePod returned nil error on forbidden Create") + } + if !strings.Contains(err.Error(), "creating stage Pod") { + t.Errorf("error missing CLI framing: %v", err) + } +} + +// TestWaitForStagePodReady_HappyPath: a Pod that immediately reports +// Ready=True passes through. +func TestWaitForStagePodReady_HappyPath(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tracebloc-stage-cats_dogs-abc12345", + Namespace: "tracebloc", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodRunning, + Conditions: []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }}, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + p, err := WaitForStagePodReady(ctx, cs, "tracebloc", "tracebloc-stage-cats_dogs-abc12345") + if err != nil { + t.Fatalf("WaitForStagePodReady: %v", err) + } + if p == nil { + t.Fatal("returned nil Pod on success") + } +} + +// TestWaitForStagePodReady_TimeoutHint: when the Pod is stuck (e.g. +// ImagePullBackOff), the timeout error should surface the waiting- +// state reason from container statuses. This is the dominant slow +// path — air-gapped customers without the right pull secret. +func TestWaitForStagePodReady_TimeoutHint(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "stuck-pod", + Namespace: "tracebloc", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodPending, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "stage", + State: corev1.ContainerState{ + Waiting: &corev1.ContainerStateWaiting{ + Reason: "ImagePullBackOff", + Message: "Back-off pulling image \"alpine:3.20@sha256:...\"", + }, + }, + }}, + }, + }) + + // Tight ctx timeout so the test doesn't actually wait 60s. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "stuck-pod") + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on stuck Pod") + } + if !strings.Contains(err.Error(), "ImagePullBackOff") { + t.Errorf("error missing ImagePullBackOff hint: %v", err) + } +} + +// TestWaitForStagePodReady_NotFoundIsTerminal: if the Pod gets +// deleted out-of-band mid-wait (admin cleanup, parallel test +// teardown, etc.), the poll must short-circuit instead of waiting +// the full timeout. Bugbot flagged the prior "everything transient" +// behavior as Medium on PR-b. +func TestWaitForStagePodReady_NotFoundIsTerminal(t *testing.T) { + cs := fake.NewClientset() // empty — Get returns NotFound + + // Generous overall ctx; the test should return WAY before + // StagePodReadyTimeout (60s) because NotFound terminates the + // poll immediately. If the fix regresses, this test takes the + // full 60s and gets caught by go test's -timeout. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "ghost-pod") + elapsed := time.Since(start) + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on NotFound") + } + if elapsed > 3*time.Second { + t.Errorf("WaitForStagePodReady waited %s for NotFound; expected immediate return", elapsed) + } +} + +// TestWaitForStagePodReady_ForbiddenIsTerminal: same as NotFound but +// for RBAC denial — the customer's kubeconfig might lose `get pods` +// permission mid-push (token rotation, RBAC change). The poll must +// surface that immediately, not spin until timeout. +func TestWaitForStagePodReady_ForbiddenIsTerminal(t *testing.T) { + cs := fake.NewClientset() + cs.PrependReactor("get", "pods", + func(_ k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + corev1.Resource("pods"), "test-pod", + errors.New("user cannot get pods")) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "test-pod") + elapsed := time.Since(start) + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on Forbidden") + } + if elapsed > 3*time.Second { + t.Errorf("WaitForStagePodReady waited %s for Forbidden; expected immediate return", elapsed) + } +} + +// TestWaitForStagePodReady_FailedPhaseIsTerminal: a Pod that +// crashes at startup (PSA rejection, ImagePullBackOff that escalates +// to ErrImagePull, OOMKilled during container start) lands in +// Phase=Failed and will never become Ready. The poll must +// short-circuit instead of waiting the full 60s. Bugbot flagged +// the missing check as Medium on PR-b round 4. +func TestWaitForStagePodReady_FailedPhaseIsTerminal(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "crashed-pod", + Namespace: "tracebloc", + }, + Status: corev1.PodStatus{ + Phase: corev1.PodFailed, + ContainerStatuses: []corev1.ContainerStatus{{ + Name: "stage", + State: corev1.ContainerState{ + Terminated: &corev1.ContainerStateTerminated{ + Reason: "OOMKilled", + ExitCode: 137, + }, + }, + }}, + }, + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := WaitForStagePodReady(ctx, cs, "tracebloc", "crashed-pod") + elapsed := time.Since(start) + if err == nil { + t.Fatal("WaitForStagePodReady returned nil on Phase=Failed") + } + if elapsed > 3*time.Second { + t.Errorf("WaitForStagePodReady waited %s for Phase=Failed; expected immediate return", elapsed) + } + if !strings.Contains(err.Error(), "Failed") { + t.Errorf("error missing Phase=Failed signal: %v", err) + } +} + +// TestDeleteStagePod_NotFoundIsOK: the Pod might be gone already +// (activeDeadlineSeconds fired, or someone kubectl-deleted it). +// Not-found shouldn't error — our goal is "Pod doesn't exist," which +// is satisfied. +func TestDeleteStagePod_NotFoundIsOK(t *testing.T) { + cs := fake.NewClientset() // empty + if err := DeleteStagePod(context.Background(), cs, "tracebloc", "nope"); err != nil { + t.Errorf("DeleteStagePod on absent Pod = %v, want nil", err) + } +} + +// TestDeleteStagePod_HappyPath: delete a real Pod, confirm it's gone. +func TestDeleteStagePod_HappyPath(t *testing.T) { + cs := fake.NewClientset(&corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{Name: "to-delete", Namespace: "tracebloc"}, + }) + if err := DeleteStagePod(context.Background(), cs, "tracebloc", "to-delete"); err != nil { + t.Fatalf("DeleteStagePod: %v", err) + } + _, err := cs.CoreV1().Pods("tracebloc").Get(context.Background(), "to-delete", metav1.GetOptions{}) + if !apierrors.IsNotFound(err) { + t.Errorf("Pod still exists after delete (err=%v)", err) + } +} diff --git a/internal/push/progress.go b/internal/push/progress.go new file mode 100644 index 0000000..f5df7b3 --- /dev/null +++ b/internal/push/progress.go @@ -0,0 +1,132 @@ +package push + +import ( + "io" + "os" + + "github.com/schollz/progressbar/v3" + "golang.org/x/term" +) + +// Progress is the narrow interface push.Stream + the stage +// orchestrator use to surface per-byte transfer progress to the +// customer. The real implementation wraps schollz/progressbar/v3 +// (NewTTYProgress); tests pass a no-op (NoOpProgress) since CI +// runs aren't TTYs and decoupling the test layer from any +// progressbar internals keeps test output clean. +// +// The interface is deliberately byte-oriented (not file-count +// oriented). Tar header bytes flow through too — they're a few +// hundred bytes per file, which makes the progress bar slightly +// over-count the "useful" bytes, but they're real bytes on the +// wire that the customer pays for anyway. The alternative +// (counting only file-body bytes) requires the progress sink to +// know what a tar header looks like, which couples it to the +// streaming format. +type Progress interface { + // Add records that n bytes have been transferred. Safe to + // call from any goroutine. + Add(n int) + + // Finish marks the transfer done. Calling Finish before all + // bytes have been added is fine — the bar just won't fill to + // 100%, which is a useful visual signal that something cut + // the stream short. + Finish() +} + +// NewProgress returns a Progress sink appropriate for the given +// output writer: +// +// - If out is a TTY (interactive terminal), return a +// progressbar/v3 that renders an animated bar with bytes/sec +// - ETA + percentage. +// - Otherwise return a no-op (CI, redirected output, piped to +// a file). A spinning bar in a CI log is noise, not a +// feature. +// +// totalBytes is the expected transfer size (from LocalLayout's +// TotalBytes plus a small tar-overhead margin). The bar normalizes +// against this; if actual transferred bytes overshoot, the bar +// caps at 100% rather than wrapping. +// +// `description` shows up to the left of the bar; "Staging cats_dogs" +// is a typical value. +func NewProgress(out io.Writer, totalBytes int64, description string) Progress { + if !isTTY(out) { + return NoOpProgress{} + } + return &ttyProgress{ + bar: progressbar.NewOptions64(totalBytes, + progressbar.OptionSetWriter(out), + progressbar.OptionSetDescription(description), + progressbar.OptionShowBytes(true), + progressbar.OptionShowCount(), + progressbar.OptionThrottle(100), // ms — keep CPU low + progressbar.OptionSetRenderBlankState(true), + progressbar.OptionSetWidth(40), + progressbar.OptionClearOnFinish(), + ), + } +} + +// isTTY reports whether out points at a real terminal. We test for +// *os.File AND golang.org/x/term IsTerminal — a *bytes.Buffer in +// tests isn't *os.File so we return false fast. Wrappers like +// io.MultiWriter that aren't *os.File also return false, which is +// the conservative choice. +func isTTY(out io.Writer) bool { + f, ok := out.(*os.File) + if !ok { + return false + } + return term.IsTerminal(int(f.Fd())) +} + +// ttyProgress is the real-bar implementation. The progressbar +// library is itself concurrency-safe under Add(), so the wrapper +// can be a thin pass-through. +type ttyProgress struct { + bar *progressbar.ProgressBar +} + +func (p *ttyProgress) Add(n int) { + // Explicit-discard the error — progressbar's Add returns an + // error if the write to the underlying writer fails, which we + // can't do anything about mid-stream (the customer's terminal + // going away during a push doesn't affect the staging + // outcome). Same rationale as the Fprintf discards elsewhere. + _ = p.bar.Add(n) +} + +func (p *ttyProgress) Finish() { + _ = p.bar.Finish() +} + +// NoOpProgress satisfies Progress without doing anything. Used in +// tests and when output is non-TTY. Exported so tests in OTHER +// packages (internal/cli) can also use it. +type NoOpProgress struct{} + +func (NoOpProgress) Add(_ int) {} +func (NoOpProgress) Finish() {} + +// progressWriter is an io.Writer that funnels write counts into a +// Progress. Used to wrap the stdin pipe-writer so every byte +// piped to the exec stream counts toward the bar. +// +// Not exported — it's an implementation detail of Stream. Tests +// validate the contract end-to-end (transferred bytes match +// expected) rather than poking the wrapper directly. +type progressWriter struct { + w io.Writer + p Progress +} + +func (pw *progressWriter) Write(b []byte) (int, error) { + n, err := pw.w.Write(b) + if n > 0 { + pw.p.Add(n) + } + return n, err +} diff --git a/internal/push/progress_test.go b/internal/push/progress_test.go new file mode 100644 index 0000000..425e210 --- /dev/null +++ b/internal/push/progress_test.go @@ -0,0 +1,44 @@ +package push + +import ( + "bytes" + "os" + "testing" +) + +// TestNewProgress_NonTTYReturnsNoOp: a non-terminal writer (a buffer, +// a redirected file, a CI log) must get the no-op sink — an animated +// bar in non-interactive output is noise. Covers NewProgress's +// dominant branch + NoOpProgress's methods. +func TestNewProgress_NonTTYReturnsNoOp(t *testing.T) { + var buf bytes.Buffer + p := NewProgress(&buf, 1000, "Staging x") + if _, ok := p.(NoOpProgress); !ok { + t.Fatalf("NewProgress(non-TTY) = %T, want NoOpProgress", p) + } + // Must be safe no-ops that write nothing. + p.Add(500) + p.Finish() + if buf.Len() != 0 { + t.Errorf("NoOpProgress wrote %d bytes to a non-TTY; want 0", buf.Len()) + } +} + +// TestIsTTY_False: neither a bytes.Buffer nor a pipe (a non-terminal +// *os.File) is a TTY. Pins the conservative detection that keeps CI +// output clean. +func TestIsTTY_False(t *testing.T) { + var buf bytes.Buffer + if isTTY(&buf) { + t.Error("isTTY(*bytes.Buffer) = true, want false") + } + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + defer func() { _ = r.Close() }() + defer func() { _ = w.Close() }() + if isTTY(w) { + t.Error("isTTY(os.Pipe writer) = true, want false (not a terminal)") + } +} diff --git a/internal/push/spec.go b/internal/push/spec.go new file mode 100644 index 0000000..a28d7e8 --- /dev/null +++ b/internal/push/spec.go @@ -0,0 +1,380 @@ +// Package push owns the `tracebloc dataset push` flow: synthesizing +// an ingest spec from CLI flags, walking the customer's local data +// directory, and (in a follow-up PR) staging files into the cluster's +// shared PVC via an ephemeral Pod + tar-over-exec stream. +// +// Phase 3 lands in two PRs (per tracebloc/client#151): +// +// - PR-a (this one): the no-op-safe path — spec synthesis, local +// layout discovery, PVC discovery, --dry-run. Everything up to +// "ready to copy files" without actually copying. +// - PR-b (next): the novel-engineering core — ephemeral stage Pod, +// client-go remotecommand executor, tar stream, progress bar, +// SIGINT-safe cleanup. +// +// The split keeps each diff reviewable. PR-a's purpose is "fail fast +// before we touch the cluster"; PR-b is "now actually push the bytes". +package push + +import ( + "fmt" + "path" + "regexp" +) + +// tableNamePattern is the safe character set for a table name. It +// must satisfy TWO independent constraints simultaneously: +// +// 1. A valid unquoted MySQL identifier — the chart's ingestor +// CREATEs a table with this exact name. +// 2. A safe single path segment — the name becomes the +// /data/shared/
/ subdirectory on the PVC. +// +// The intersection of "MySQL identifier" and "single safe path +// component" is [A-Za-z0-9_]: letters, digits, underscore. No +// slashes, no dots — which is what closes the path-traversal hole +// (see ValidateTableName). +// +// All the real-world example tables (chest_xrays_train, +// cats_dogs_train) match this; it's the conventional snake_case +// table-naming style anyway. +var tableNamePattern = regexp.MustCompile(`^[A-Za-z0-9_]+$`) + +// MaxTableNameLength caps `--table` at 63 chars. Two hard limits +// agree on this: +// +// 1. MySQL identifier limit: 64 chars (MySQL Reference 9.2 Schema +// Object Names). We use 63 to leave one char of headroom for +// any future "ingestion_run_id" suffix the chart might want to +// append. +// 2. Kubernetes label value limit: 63 chars (DNS-1123 label rules). +// The stage Pod's tracebloc.io/table label carries the +// untransformed table name verbatim — a longer value fails Pod +// creation post-pre-flight, which Bugbot flagged on PR-b +// round 5. +// +// 63 is a coincidence that lets both checks share the same constant. +const MaxTableNameLength = 63 + +// ValidateTableName rejects table names that aren't safe as both a +// MySQL identifier and a single PVC path segment. +// +// Why a CLI-side check rather than the schema: the embedded +// ingest.v1.json only enforces `minLength: 1` on `table` — no +// `pattern`, no `maxLength`. Without this guard, --table=../../etc +// would flow into the /data/shared/
/ PVC path, and a 100-char +// name would make the Pod-label assignment fail post-pre-flight. +// Tightening the upstream schema is the proper long-term fix +// (it would protect the helm flow + jobs-manager too) but needs +// a change to tracebloc/data-ingestors' schema — filed as +// tracebloc/data-ingestors#116. Once that lands and we re-sync, +// this guard can collapse to a thin "schema says so" wrapper. +// +// Callers MUST run this before SpecArgs.Build() or StagedPrefix(), +// both of which assume a validated name. +func ValidateTableName(table string) error { + if table == "" { + return fmt.Errorf("table name is required (set --table)") + } + if len(table) > MaxTableNameLength { + return fmt.Errorf( + "table name is %d characters; the max is %d "+ + "(matches both the MySQL identifier limit and the "+ + "Kubernetes label-value limit, which the stage Pod's "+ + "tracebloc.io/table label is bound by). Use a shorter name.", + len(table), MaxTableNameLength) + } + if !tableNamePattern.MatchString(table) { + return fmt.Errorf( + "table name %q is invalid: must match [A-Za-z0-9_]+ "+ + "(letters, digits, underscore only). The table name is "+ + "used both as the MySQL table identifier and as the "+ + "/data/shared/
/ subdirectory on the cluster PVC, "+ + "so slashes, dots, and path-traversal sequences are "+ + "rejected.", + table) + } + return nil +} + +// SpecArgs is the user-facing flag set for `tracebloc dataset push`. +// +// It's intentionally narrower than the full ingest.v1.json schema — +// for v0.1, only the image_classification minimum is exposed. The +// epic (#147) defers all other categories to v0.2 as one-PR +// additions; the flag set will grow when those land. +// +// Validation is NOT enforced here. Build() produces an +// ingest.v1.json-conforming map, and the caller pipes it through +// internal/schema's V1 validator. Duplicating "intent must be +// train|test" or similar in Go-side code would drift from the +// embedded schema — the schema is the single source of truth. +type SpecArgs struct { + // Table is the destination MySQL table name in the cluster. Also + // used as the per-table subdirectory under /data/shared/ so + // pushes of multiple tables don't collide on the PVC. + Table string + + // Category pins the task family. For v0.1, only + // "image_classification" is supported end-to-end — the epic + // non-goals defer other categories to v0.2. The flag accepts + // other values so the schema's enum check produces the canonical + // error message (rather than a CLI-side "unknown category" + // that drifts from the schema). + Category string + + // Intent is "train" or "test" per the schema's enum. Same + // rationale as Category for not pre-validating here. + Intent string + + // LabelColumn is the column name in labels.csv that holds the + // label. The schema accepts either a shorthand string (this + // field) or a {column, policy} object; v0.1 only emits the + // shorthand because passthrough is the only policy + // image_classification cares about. + LabelColumn string + + // TargetSize, when len==2, holds the resolution as [W, H] — the + // order produced by --target-size "WxH" and by DetectImageSize. + // The ingestor's image_classification default is 512x512 and it + // VALIDATES (it does not resize), so a dataset whose images don't + // match the default hard-fails. Setting this emits + // spec.file_options.target_size so the customer's actual + // resolution wins. Empty (len 0) ⇒ omit and let the ingestor + // default apply. Populated by the CLI from --target-size or by + // auto-detecting the first image. (Image categories only.) + // + // NOTE: stored [W, H] here, but EMITTED as [height, width] by + // Build — that's the order the ingest.v1 schema documents and the + // ingestor reads. buildImage does the swap. (Bugbot, PR #22.) + TargetSize []int + + // Schema is the column→SQL-type map for tabular / time-series + // categories (required by the schema for those). Populated by the + // CLI from --schema or by inferring types from the CSV. Ignored + // for image categories. + Schema map[string]string + + // LabelPolicy is "passthrough" or "bucket". Regression-class + // categories (tabular_regression, time_series_forecasting, + // time_to_event_prediction) require the object label form with a + // policy; the CLI defaults it to "bucket" so the raw numeric + // target never ships to the central backend. Ignored for + // classification categories (which emit the string label form). + LabelPolicy string + + // TimeColumn names the time column for time_to_event_prediction. + // Emitted as the top-level `time_column` field. Empty ⇒ the + // ingestor falls back to a column named "time". + TimeColumn string + + // NumberOfKeypoints is the keypoints-per-sample count for + // keypoint_detection (required there; no convention default). + // Emitted under spec.file_options.number_of_keypoints — the schema + // allows arbitrary file_options keys, and conventions.resolve reads + // it from there, so this needs no top-level schema field. 0 ⇒ + // unset (ignored for non-keypoint categories). + NumberOfKeypoints int +} + +// Build produces the ingest.v1.json-conforming spec map. The +// returned map is YAML-marshalable and ready to feed to +// internal/schema's V1 validator. +// +// PVC paths (csv, images) are constructed under +// /data/shared/
/ to match the chart's mount convention +// surfaced by Phase 2's cluster discovery — jobs-manager mounts +// client-pvc at /data/shared, and the per-table subdir prevents +// cross-table collisions when a customer pushes multiple datasets +// to the same release. +// +// File-name conventions inside that subdir (labels.csv, +// images/) are dictated by the local layout that internal/push.Discover +// requires. Keeping the layout convention in lock-step on both +// sides — the CLI's view of "what local files we expect" and the +// spec's view of "where they'll live in the cluster" — means a +// successful Discover guarantees a runnable spec. +// +// PRECONDITION: a.Table must have passed ValidateTableName. Build +// calls StagedPrefix, which panics on an unsafe name. +func (a SpecArgs) Build() map[string]any { + prefix := StagedPrefix(a.Table) + spec := map[string]any{ + "apiVersion": "tracebloc.io/v1", + "kind": "IngestConfig", + "category": a.Category, + "table": a.Table, + "intent": a.Intent, + "csv": path.Join(prefix, "labels.csv"), + } + switch { + case IsTabular(a.Category): + a.buildTabular(spec) + case IsText(a.Category): + a.buildText(spec, prefix) + default: + // Image categories (and any not-yet-special-cased category — + // the schema validator produces the canonical error for those). + a.buildImage(spec, prefix) + } + return spec +} + +// buildText fills in the text-family fields: the text-file sidecar +// directory (texts/ for text_classification, sequences/ for +// masked_language_modeling) and the label. masked_language_modeling +// has NO label (the schema doesn't require one for it). +func (a SpecArgs) buildText(spec map[string]any, prefix string) { + dir := TextSidecarDir(a.Category) + // Trailing slash matches the directory-glob convention the + // ingestor uses for sidecar dirs. + spec[dir] = path.Join(prefix, dir) + "/" + if a.Category == "text_classification" { + spec["label"] = a.LabelColumn + } +} + +// buildImage fills in the image-family fields for image_classification, +// object_detection, and keypoint_detection: the images/ dir, the label, +// object_detection's annotations/ dir, and the resolution overrides. +// +// keypoint_detection emits target_size + number_of_keypoints as +// TOP-LEVEL fields — the schema's keypoint conditional requires them +// there (both are dataset-specific, no convention defaults), and the +// ingestor validates against that conditional. image_classification +// and object_detection emit target_size under spec.file_options (the +// override key conventions.resolve reads); without it, +// image_classification defaults to 512x512 and the Image Resolution +// Validator rejects other sizes. +func (a SpecArgs) buildImage(spec map[string]any, prefix string) { + // Trailing slash on the dir fields matches the schema example + // (data-ingestors/examples/yaml/image_classification.yaml); the + // ingestor treats them as directory globs. + spec["images"] = path.Join(prefix, "images") + "/" + spec["label"] = a.LabelColumn + if a.Category == "object_detection" { + spec["annotations"] = path.Join(prefix, "annotations") + "/" + } + + if a.Category == "keypoint_detection" { + if len(a.TargetSize) == 2 { + // Schema documents target_size as [height, width]; TargetSize + // is stored [W, H], so swap on emit. (Bugbot, PR #22.) + spec["target_size"] = []int{a.TargetSize[1], a.TargetSize[0]} + } + if a.NumberOfKeypoints > 0 { + spec["number_of_keypoints"] = a.NumberOfKeypoints + } + return + } + + if len(a.TargetSize) == 2 { + spec["spec"] = map[string]any{ + "file_options": map[string]any{ + // [height, width] per the schema; TargetSize is [W, H]. + "target_size": []int{a.TargetSize[1], a.TargetSize[0]}, + }, + } + } +} + +// buildTabular fills in the tabular / time-series fields: the column +// schema, the label (string form for classification, object form with +// a policy for regression-class), and time_column for +// time_to_event_prediction. +func (a SpecArgs) buildTabular(spec map[string]any) { + spec["schema"] = a.Schema + if IsRegressionClass(a.Category) { + // Regression-class tasks require the object label form with an + // explicit policy (the schema enforces this). Default to + // `bucket` so the raw numeric target never ships to the central + // backend unless the customer opts into passthrough. + policy := a.LabelPolicy + if policy == "" { + policy = "bucket" + } + spec["label"] = map[string]any{"column": a.LabelColumn, "policy": policy} + } else { + // tabular_classification: plain column-name label. + spec["label"] = a.LabelColumn + } + if a.Category == "time_to_event_prediction" && a.TimeColumn != "" { + spec["time_column"] = a.TimeColumn + } +} + +// SharedRoot is the in-cluster mount path of the chart's shared PVC +// (cluster.SharedPVCMountPath). Both the ephemeral stage Pod and the +// ingestor Job mount client-pvc here, so any path under it is visible +// to both — which is why the CLI's staging area lives under it. +const SharedRoot = "/data/shared" + +// stagingDirName is the hidden directory under SharedRoot where the +// CLI lands a run's SOURCE files. It is deliberately SEPARATE from +// the ingestor's destination (SharedRoot/
): +// +// data-ingestors computes DEST_PATH = STORAGE_PATH/TABLE_NAME = +// SharedRoot/
, and its DuplicateValidator FAILS if that path +// already exists non-empty. If the CLI staged straight into +// SharedRoot/
(as it did originally), its own staging would +// create exactly the non-empty destination the validator rejects — +// so every push failed the duplicate check. Staging under +// SharedRoot/.tracebloc-staging/
keeps the destination fresh +// while remaining on the same PVC the ingestor reads. +const stagingDirName = ".tracebloc-staging" + +// StagedPrefix returns the in-cluster directory the CLI streams a +// dataset's SOURCE files into for a given table. The synthesized +// spec's csv/images point here; the ingestor reads from here and +// writes the processed table to FinalDestPrefix(table). +// +// Two call sites MUST agree on this value: +// +// 1. The ephemeral stage Pod's tar target (StreamLayout). +// 2. The csv/images fields in Build() above, which jobs-manager +// hands to the ingestor Job so it reads what we just staged. +// +// It is intentionally NOT SharedRoot/
: that is the ingestor's +// DEST_PATH, whose DuplicateValidator rejects a pre-existing, +// non-empty directory. See stagingDirName. +// +// PRECONDITION: table must already have passed ValidateTableName. +// This function panics on an unsafe name rather than returning an +// escape path — a name that escapes SharedRoot is a caller bug +// (validation was skipped), and a panic surfaces it loudly in tests +// instead of silently letting the stage Pod write to, say, /etc. +// Every production call path runs ValidateTableName first (see +// cli.runDatasetPush), so the panic is unreachable in correct code. +func StagedPrefix(table string) string { + // Deliberately NOT path.Join here: path.Join cleans ".." + // segments, which is exactly the silent traversal we're + // guarding against. Plain concatenation keeps the name as a + // literal segment so the assertion below can detect a bad one. + if !tableNamePattern.MatchString(table) { + panic(fmt.Sprintf( + "push.StagedPrefix: unsafe table name %q — caller must "+ + "ValidateTableName before constructing a PVC path", + table)) + } + return SharedRoot + "/" + stagingDirName + "/" + table +} + +// FinalDestPrefix returns where the ingestor writes the processed +// table: SharedRoot/
, matching data-ingestors' config.DEST_PATH +// (STORAGE_PATH/TABLE_NAME). This is what the training side reads and +// what the CLI shows the customer as the destination. The CLI never +// writes here directly — doing so would trip the ingestor's +// DuplicateValidator; it stages to StagedPrefix(table) and the +// ingestor produces this path. +// +// PRECONDITION: table must already have passed ValidateTableName. +// Panics on an unsafe name, same rationale as StagedPrefix. +func FinalDestPrefix(table string) string { + if !tableNamePattern.MatchString(table) { + panic(fmt.Sprintf( + "push.FinalDestPrefix: unsafe table name %q — caller must "+ + "ValidateTableName before constructing a PVC path", + table)) + } + return SharedRoot + "/" + table +} diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go new file mode 100644 index 0000000..09c6a4c --- /dev/null +++ b/internal/push/spec_test.go @@ -0,0 +1,448 @@ +package push + +import ( + "strings" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/tracebloc/cli/internal/schema" +) + +// TestBuild_ImageClassificationMinimum_PassesSchema is the contract +// test that pins Phase 3's flag → spec synthesis to the embedded +// schema. The whole point of Build() is "produce something the +// canonical validator accepts" — if a refactor breaks that, every +// `dataset push` invocation fails after kubeconfig load but before +// the user sees a useful error, so we want this caught in CI. +func TestBuild_ImageClassificationMinimum_PassesSchema(t *testing.T) { + args := SpecArgs{ + Table: "chest_xrays_train", + Category: "image_classification", + Intent: "train", + LabelColumn: "image_label", + } + spec := args.Build() + + // Round-trip via YAML because the validator's public API is + // YAML-input. The map → JSON → YAML → parse-back chain is a + // microscopic cost per `dataset push` invocation; not worth + // adding a Validate(parsed) method to internal/schema for v0.1. + specBytes, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("yaml.Marshal: %v", err) + } + + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + _, errs, parseErr := v.ValidateYAML(specBytes) + if parseErr != nil { + t.Fatalf("ValidateYAML returned parse error on our own output: %v\n%s", parseErr, specBytes) + } + if len(errs) != 0 { + t.Fatalf("synthesized spec failed schema validation: %s\nspec:\n%s", + schema.FormatErrors(errs), specBytes) + } +} + +// TestBuild_PathsMatchStagedPrefix pins the contract between Phase 3 +// (where the CLI puts files on the PVC) and Phase 4 (what paths the +// submitted spec tells jobs-manager to look at). If these ever +// drift, the ingestor Job spawned by jobs-manager won't find the +// files we just staged — a silent "0 rows ingested" outcome that's +// hard to debug. +func TestBuild_PathsMatchStagedPrefix(t *testing.T) { + const table = "cats_dogs" + spec := SpecArgs{ + Table: table, + Category: "image_classification", + Intent: "train", + LabelColumn: "label", + }.Build() + + prefix := StagedPrefix(table) + wantCSV := prefix + "/labels.csv" + wantImages := prefix + "/images/" + + if got := spec["csv"].(string); got != wantCSV { + t.Errorf("spec.csv = %q, want %q", got, wantCSV) + } + if got := spec["images"].(string); got != wantImages { + t.Errorf("spec.images = %q, want %q", got, wantImages) + } +} + +// TestBuild_LeavesValidationToSchema asserts that Build() does NOT +// pre-validate. A garbage category goes through unchanged so the +// schema's enum check produces the canonical error message. The +// alternative — duplicating the schema's enum in Go — would drift +// the moment data-ingestors adds a new category and we forget to +// mirror it here. +func TestBuild_LeavesValidationToSchema(t *testing.T) { + spec := SpecArgs{ + Table: "x", + Category: "definitely-not-a-real-category", + Intent: "train", + LabelColumn: "label", + }.Build() + + if got := spec["category"].(string); got != "definitely-not-a-real-category" { + t.Errorf("Build() pre-validated category; want raw passthrough, got %q", got) + } +} + +// TestStagedPrefix_PerTableIsolation pins the contract that two +// concurrent pushes for different tables don't write to the same +// PVC subdirectory. If this ever returns the same path for +// different tables, parallel `dataset push` calls would race on +// labels.csv overwrites. +func TestStagedPrefix_PerTableIsolation(t *testing.T) { + if a, b := StagedPrefix("cats"), StagedPrefix("dogs"); a == b { + t.Errorf("StagedPrefix(%q) == StagedPrefix(%q) = %q, want distinct", "cats", "dogs", a) + } + if got := StagedPrefix("table_a"); got != "/data/shared/.tracebloc-staging/table_a" { + t.Errorf("StagedPrefix(%q) = %q, want /data/shared/.tracebloc-staging/table_a", "table_a", got) + } +} + +// TestStagedPrefix_DoesNotCollideWithDest is the regression pin for +// the live-discovered blocker (#26): the CLI must stage SOURCE files +// somewhere the ingestor's DEST_PATH (= FinalDestPrefix = /data/ +// shared/
) does NOT contain. If staging ever lands at or under +// the destination again, the ingestor's DuplicateValidator will +// reject the CLI's own staging as a pre-existing non-empty +// destination, and every push fails the duplicate check. +func TestStagedPrefix_DoesNotCollideWithDest(t *testing.T) { + const table = "cats_dogs_train" + staged := StagedPrefix(table) + dest := FinalDestPrefix(table) + + if staged == dest { + t.Fatalf("StagedPrefix == FinalDestPrefix == %q; the CLI would stage "+ + "into the ingestor's DEST_PATH and trip DuplicateValidator", staged) + } + // The destination must not contain the staging dir either — + // otherwise DEST is non-empty (it holds the staging subtree) and + // the validator still fails. + if strings.HasPrefix(staged, dest+"/") { + t.Errorf("StagedPrefix %q is under FinalDestPrefix %q; DEST would be "+ + "non-empty at ingest time", staged, dest) + } + if got := FinalDestPrefix(table); got != "/data/shared/"+table { + t.Errorf("FinalDestPrefix(%q) = %q, want /data/shared/%s", table, got, table) + } +} + +// TestBuild_WithTargetSize_PassesSchema pins the #27 plumbing: when +// the CLI resolves an image resolution (via --target-size or +// auto-detect), Build emits spec.file_options.target_size and the +// result still validates against the embedded v1 schema. A drift here +// would make every push that sets a target size fail validation. +func TestBuild_WithTargetSize_PassesSchema(t *testing.T) { + spec := SpecArgs{ + Table: "cats_dogs_train", + Category: "image_classification", + Intent: "train", + LabelColumn: "label", + TargetSize: []int{256, 256}, + }.Build() + + // The nested override must be present and well-shaped. + specBlock, ok := spec["spec"].(map[string]any) + if !ok { + t.Fatalf("Build() with TargetSize didn't emit a spec block: %#v", spec["spec"]) + } + fo, ok := specBlock["file_options"].(map[string]any) + if !ok { + t.Fatalf("spec.file_options missing/wrong type: %#v", specBlock["file_options"]) + } + ts, ok := fo["target_size"].([]int) + if !ok || len(ts) != 2 || ts[0] != 256 || ts[1] != 256 { + t.Fatalf("spec.file_options.target_size = %#v, want [256 256]", fo["target_size"]) + } + + specBytes, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("yaml.Marshal: %v", err) + } + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + _, errs, parseErr := v.ValidateYAML(specBytes) + if parseErr != nil { + t.Fatalf("ValidateYAML parse error on our own output: %v\n%s", parseErr, specBytes) + } + if len(errs) != 0 { + t.Fatalf("spec with target_size failed schema validation: %s\nspec:\n%s", + schema.FormatErrors(errs), specBytes) + } +} + +// TestBuild_NoTargetSize_OmitsSpecBlock: when no resolution is set, +// Build must NOT emit a spec block (the ingestor's per-category +// default applies). Asserting the omission keeps the minimal-spec +// contract that the original schema test relies on. +func TestBuild_NoTargetSize_OmitsSpecBlock(t *testing.T) { + spec := SpecArgs{ + Table: "t", + Category: "image_classification", + Intent: "train", + LabelColumn: "label", + }.Build() + if _, present := spec["spec"]; present { + t.Errorf("Build() with no TargetSize emitted a spec block; want omitted") + } +} + +// TestBuild_TargetSize_EmittedHeightWidth locks the [height, width] +// emit order (Bugbot, PR #22). TargetSize is stored [W, H] but the +// ingest.v1 schema + ingestor read target_size as [height, width]. +// Square sizes can't catch a swap, so this uses a NON-square +// resolution: [W, H] = [640, 480] must emit [480, 640]. +func TestBuild_TargetSize_EmittedHeightWidth(t *testing.T) { + // image_classification → under spec.file_options + icSpec := SpecArgs{ + Table: "t", Category: "image_classification", Intent: "train", + LabelColumn: "label", TargetSize: []int{640, 480}, // [W, H] + }.Build() + sb, ok := icSpec["spec"].(map[string]any) + if !ok { + t.Fatalf("image: no spec block: %#v", icSpec["spec"]) + } + fo, ok := sb["file_options"].(map[string]any) + if !ok { + t.Fatalf("image: no file_options: %#v", sb["file_options"]) + } + if ts, ok := fo["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 480 || ts[1] != 640 { + t.Errorf("image target_size = %#v, want [480 640] (height, width)", fo["target_size"]) + } + + // keypoint_detection → top-level + kpSpec := SpecArgs{ + Table: "t", Category: "keypoint_detection", Intent: "train", + LabelColumn: "image_label", TargetSize: []int{640, 480}, NumberOfKeypoints: 9, + }.Build() + if ts, ok := kpSpec["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 480 || ts[1] != 640 { + t.Errorf("keypoint top-level target_size = %#v, want [480 640] (height, width)", kpSpec["target_size"]) + } +} + +// TestBuild_Tabular_PassesSchema pins the tabular Build branch: it +// emits schema-valid specs for the three label shapes — a string +// label (tabular_classification), an object label+policy +// (tabular_regression), and an object label + time_column +// (time_to_event_prediction) — and never emits an images field. +func TestBuild_Tabular_PassesSchema(t *testing.T) { + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + check := func(name string, a SpecArgs, wantLabelObject bool) { + t.Run(name, func(t *testing.T) { + spec := a.Build() + if _, hasImages := spec["images"]; hasImages { + t.Errorf("tabular spec emitted an images field: %v", spec) + } + if _, hasSchema := spec["schema"]; !hasSchema { + t.Errorf("tabular spec missing schema: %v", spec) + } + if wantLabelObject { + if _, ok := spec["label"].(map[string]any); !ok { + t.Errorf("label = %#v, want object form {column, policy}", spec["label"]) + } + } else { + if _, ok := spec["label"].(string); !ok { + t.Errorf("label = %#v, want string form", spec["label"]) + } + } + b, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + _, errs, parseErr := v.ValidateYAML(b) + if parseErr != nil { + t.Fatalf("parse: %v\n%s", parseErr, b) + } + if len(errs) != 0 { + t.Fatalf("schema validation failed: %s\n%s", schema.FormatErrors(errs), b) + } + }) + } + + check("tabular_classification", SpecArgs{ + Table: "t_clf", Category: "tabular_classification", Intent: "train", + LabelColumn: "label", + Schema: map[string]string{"f0": "FLOAT", "f1": "FLOAT", "label": "INT"}, + }, false) + + check("tabular_regression", SpecArgs{ + Table: "t_reg", Category: "tabular_regression", Intent: "train", + LabelColumn: "price", + Schema: map[string]string{"sqft": "FLOAT", "price": "FLOAT"}, + }, true) + + check("time_to_event_prediction", SpecArgs{ + Table: "t_tte", Category: "time_to_event_prediction", Intent: "train", + LabelColumn: "DEATH_EVENT", TimeColumn: "time", + Schema: map[string]string{"age": "INT", "time": "INT", "DEATH_EVENT": "INT"}, + }, true) +} + +// TestBuild_Tabular_RegressionDefaultsPolicyBucket: regression-class +// categories default to policy=bucket so the raw numeric target never +// ships to the central backend unless the customer opts out. +func TestBuild_Tabular_RegressionDefaultsPolicyBucket(t *testing.T) { + spec := SpecArgs{ + Table: "t", Category: "tabular_regression", Intent: "train", + LabelColumn: "y", Schema: map[string]string{"x": "FLOAT", "y": "FLOAT"}, + }.Build() + lbl, ok := spec["label"].(map[string]any) + if !ok { + t.Fatalf("label = %#v, want object", spec["label"]) + } + if lbl["policy"] != "bucket" { + t.Errorf("default policy = %v, want bucket", lbl["policy"]) + } +} + +// TestBuild_ImageExtras_PassSchema pins object_detection and +// keypoint_detection: OD emits an annotations field; keypoint emits +// number_of_keypoints under spec.file_options (NOT a top-level field, +// so it validates against the current vendored schema) and no +// annotations. Both must validate. +func TestBuild_ImageExtras_PassSchema(t *testing.T) { + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + validate := func(t *testing.T, spec map[string]any) { + b, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + _, errs, parseErr := v.ValidateYAML(b) + if parseErr != nil { + t.Fatalf("parse: %v\n%s", parseErr, b) + } + if len(errs) != 0 { + t.Fatalf("schema validation failed: %s\n%s", schema.FormatErrors(errs), b) + } + } + + t.Run("object_detection", func(t *testing.T) { + spec := SpecArgs{ + Table: "od", Category: "object_detection", Intent: "train", + LabelColumn: "image_label", TargetSize: []int{448, 448}, + }.Build() + if _, ok := spec["annotations"]; !ok { + t.Errorf("OD spec missing annotations: %v", spec) + } + validate(t, spec) + }) + + t.Run("keypoint_detection", func(t *testing.T) { + spec := SpecArgs{ + Table: "kp", Category: "keypoint_detection", Intent: "train", + LabelColumn: "image_label", TargetSize: []int{448, 448}, NumberOfKeypoints: 9, + }.Build() + if _, ok := spec["annotations"]; ok { + t.Errorf("keypoint spec should not emit annotations: %v", spec) + } + // keypoint requires target_size + number_of_keypoints TOP-LEVEL + // (the schema's keypoint conditional), not under file_options. + if spec["number_of_keypoints"] != 9 { + t.Errorf("top-level number_of_keypoints = %v, want 9", spec["number_of_keypoints"]) + } + if ts, ok := spec["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 448 || ts[1] != 448 { + t.Errorf("top-level target_size = %#v, want [448 448]", spec["target_size"]) + } + validate(t, spec) + }) +} + +// TestValidateTableName_Accepts pins the names that MUST pass — +// the real-world example tables plus a few edge shapes (single +// char, leading underscore, mixed case, digits). A regression +// that rejects a valid name would break legitimate pushes. +func TestValidateTableName_Accepts(t *testing.T) { + for _, name := range []string{ + "cats_dogs", + "chest_xrays_train", + "t1", + "ABC", + "table_123", + "_leading_underscore", + "9starts_with_digit", // valid MySQL identifier + safe path segment + } { + if err := ValidateTableName(name); err != nil { + t.Errorf("ValidateTableName(%q) = %v, want nil", name, err) + } + } +} + +// TestValidateTableName_RejectsTooLong: K8s label values are +// capped at 63 chars, and the stage Pod carries the raw table +// name as the tracebloc.io/table label. Without this rejection, +// a 100-char table name passes pre-flight then fails Pod create +// with an opaque label-validation error. Bugbot flagged on PR-b +// round 5. +func TestValidateTableName_RejectsTooLong(t *testing.T) { + tooLong := strings.Repeat("a", MaxTableNameLength+1) + if err := ValidateTableName(tooLong); err == nil { + t.Fatalf("ValidateTableName(%d-char name) = nil, want length-cap error", len(tooLong)) + } + // Right at the boundary: 63 chars must be accepted. + atCap := strings.Repeat("a", MaxTableNameLength) + if err := ValidateTableName(atCap); err != nil { + t.Errorf("ValidateTableName(%d-char name) = %v, want nil (at exact cap)", len(atCap), err) + } +} + +// TestValidateTableName_RejectsUnsafe is the security-regression +// pin. The path-traversal cases (../../etc, ../foo) are the ones +// Bugbot flagged on PR #8 — if this test ever goes green with +// those removed, the traversal hole is back open. +func TestValidateTableName_RejectsUnsafe(t *testing.T) { + cases := map[string]string{ + "empty": "", + "parent traversal": "../../etc", + "single parent": "../foo", + "embedded slash": "foo/bar", + "embedded dot": "foo.bar", + "bare dot": ".", + "absolute": "/etc/passwd", + "space": "my table", + "dash": "cats-dogs", // not a valid unquoted MySQL identifier + "trailing newline": "foo\n", + "shell metachar": "foo;rm", + "null-ish unicode": "foo\x00bar", + } + for desc, table := range cases { + if err := ValidateTableName(table); err == nil { + t.Errorf("%s: ValidateTableName(%q) = nil, want a rejection error", desc, table) + } + } +} + +// TestStagedPrefix_PanicsOnUnsafeName pins the defense-in-depth +// backstop: if a caller skips ValidateTableName and hands a +// traversal name straight to StagedPrefix, it must panic rather +// than silently return an escape path. PR-b adds new call sites +// for StagedPrefix — this test guards against one of them +// forgetting the validation step. +func TestStagedPrefix_PanicsOnUnsafeName(t *testing.T) { + for _, unsafe := range []string{"../../etc", "foo/bar", ""} { + t.Run(unsafe, func(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Errorf("StagedPrefix(%q) did not panic; an unsafe "+ + "name must panic, not return an escape path", unsafe) + } + }() + _ = StagedPrefix(unsafe) + }) + } +} diff --git a/internal/push/stage.go b/internal/push/stage.go new file mode 100644 index 0000000..eed5885 --- /dev/null +++ b/internal/push/stage.go @@ -0,0 +1,153 @@ +package push + +import ( + "context" + "fmt" + "io" + "time" + + "k8s.io/client-go/kubernetes" +) + +// StageOptions packages every dependency Stage needs into a single +// struct so callers (cli.runDatasetPush) can build it incrementally +// without juggling 10 positional args. +// +// Required fields are pinned in the doc; the nil-able Progress + Out +// fields get default behaviors (NoOpProgress / io.Discard) so the +// happy path is short. +type StageOptions struct { + // Client is the discovered kubernetes.Interface. + Client kubernetes.Interface + + // Executor is how the tar stream actually reaches the Pod. + // Production: a *SPDYExecutor backed by the rest.Config. + // Tests: a fakeExecutor (see stream_test.go). + Executor Executor + + // Namespace + IngestorSAName come from Phase 2's parent-release + // discovery (with optional --ingestor-sa override). + Namespace string + IngestorSAName string + + // PVCClaimName + PVCMountPath come from Phase 3 PR-a's PVC + // discovery (always "client-pvc" + "/data/shared" today). + PVCClaimName string + PVCMountPath string + + // Layout is the validated local files-to-stage. + Layout *LocalLayout + + // Table is the destination table name. MUST have already + // passed ValidateTableName. + Table string + + // StagePodImage overrides the alpine default. Empty = default. + StagePodImage string + + // Progress is the customer-facing transfer bar. Nil = no-op + // (use this in tests / non-TTY output). + Progress Progress + + // Out is where Stage prints non-progress diagnostic output + // (orphan warnings, "Created stage Pod X" etc.). Nil = io.Discard. + // Progress bar output is separate — schollz writes to its own + // configured writer (typically the same one). + Out io.Writer +} + +// Stage is Phase 3 PR-b's top-level entrypoint. Does the full +// dance: orphan scan → create stage Pod → wait Ready → tar-over- +// exec stream → defer-delete (SIGINT-safe). +// +// On success, the layout's files are on the cluster's PVC at +// /data/shared/
/{labels.csv, images/...} and the stage Pod +// has been cleaned up. Caller proceeds to Phase 4's submit step. +// +// On error, the deferred cleanup still fires — no orphan Pod left +// behind even if the stream fails partway. Phase 4's idempotency +// key handles the "retry the whole push" scenario; for v0.1 we +// don't try to resume a partial transfer. +// +// SIGINT handling: the caller is expected to wire a signal handler +// that cancels the passed-in ctx (cli/main.go pattern). When ctx +// is cancelled, the in-flight Exec stream returns ctx.Err(), the +// deferred DeleteStagePod runs with a FRESH context (so it can +// reach the API even after our parent ctx is cancelled), and the +// error propagates with the SIGINT signal preserved. +func Stage(ctx context.Context, opts StageOptions) error { + if opts.Out == nil { + opts.Out = io.Discard + } + if opts.Progress == nil { + opts.Progress = NoOpProgress{} + } + + // 1. Orphan scan — best-effort. We log either the warning or + // the scan-failure and proceed regardless. Customers who + // care about orphans get to see them; customers whose RBAC + // forbids List Pods just don't get the warning (no impact + // on the push itself). + if orphans, err := FindOrphanStagePods(ctx, opts.Client, opts.Namespace); err != nil { + _, _ = fmt.Fprintf(opts.Out, "(orphan-scan skipped: %v)\n", err) + } else if warning := FormatOrphansWarning(orphans); warning != "" { + _, _ = fmt.Fprint(opts.Out, warning) + } + + // 2. Create the stage Pod. From here on, cleanup matters. + podName, err := CreateStagePod(ctx, opts.Client, PodSpecOptions{ + Namespace: opts.Namespace, + PVCClaimName: opts.PVCClaimName, + PVCMountPath: opts.PVCMountPath, + Table: opts.Table, + Image: opts.StagePodImage, + ServiceAccountName: opts.IngestorSAName, + }) + if err != nil { + return err + } + _, _ = fmt.Fprintf(opts.Out, "Created stage Pod %s/%s\n", opts.Namespace, podName) + + // 3. Defer cleanup. The deferred call uses a FRESH context with + // its own deadline — if the parent ctx is cancelled (SIGINT, + // timeout), we still want the Pod deleted. Without this, a + // Ctrl-C right after pod-create would leak an orphan. + // + // 30s deadline on the cleanup is a safety net for the case + // where the API server itself is unreachable — better to + // print "failed to clean up stage Pod" than hang forever. + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if delErr := DeleteStagePod(cleanupCtx, opts.Client, opts.Namespace, podName); delErr != nil { + _, _ = fmt.Fprintf(opts.Out, + "WARNING: failed to delete stage Pod %s/%s (it will be activeDeadline-killed in <=%ds): %v\n", + opts.Namespace, podName, StagePodActiveDeadline, delErr) + } + }() + + // 4. Wait for the Pod to be Ready (containers up, ready to + // accept exec). Times out at StagePodReadyTimeout with + // diagnostic hints from container statuses (image pull, + // scheduling) when we have them. + _, _ = fmt.Fprintf(opts.Out, "Waiting for stage Pod to be Ready (timeout %s)...\n", StagePodReadyTimeout) + if _, err := WaitForStagePodReady(ctx, opts.Client, opts.Namespace, podName); err != nil { + return err + } + + // 5. Stream the tar. This is where actual bytes flow. The + // progress bar (if TTY) renders during this call. + _, _ = fmt.Fprintf(opts.Out, "Streaming %d files (%s) for table %q...\n", + opts.Layout.FileCount(), HumanBytes(opts.Layout.TotalBytes), opts.Table) + + if err := StreamLayout(ctx, opts.Executor, + opts.Namespace, podName, "stage", + opts.Layout, opts.Table, opts.Progress); err != nil { + return err + } + + // 6. Print "done" message. The deferred cleanup runs after this. + _, _ = fmt.Fprintf(opts.Out, "Staged %d files for table %q\n", + opts.Layout.FileCount(), opts.Table) + return nil +} diff --git a/internal/push/stage_test.go b/internal/push/stage_test.go new file mode 100644 index 0000000..b46e0cb --- /dev/null +++ b/internal/push/stage_test.go @@ -0,0 +1,288 @@ +package push + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// readyOnNextGet adds a reactor that marks any new stage Pod as +// Ready=True the moment after it's created. Without this, the fake +// clientset creates the Pod in default (empty) status and +// WaitForStagePodReady spins until its timeout. We can't easily +// patch the Pod from inside StageOptions construction, so the +// reactor is the standard pattern. +func readyOnNextGet(cs *fake.Clientset) { + cs.PrependReactor("get", "pods", + func(action k8stesting.Action) (bool, runtime.Object, error) { + ga, ok := action.(k8stesting.GetAction) + if !ok { + return false, nil, nil + } + // Let the existing tracker handle the get to find the + // real object, then patch it before returning. The + // safest way: do the get ourselves, then synthesize a + // Ready condition onto the result. + pod, err := cs.Tracker().Get(action.GetResource(), action.GetNamespace(), ga.GetName()) + if err != nil { + return false, nil, nil + } + p := pod.(*corev1.Pod) + p.Status.Phase = corev1.PodRunning + p.Status.Conditions = []corev1.PodCondition{{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + }} + return true, p, nil + }) +} + +// TestStage_HappyPath: orphan scan (empty) → create → wait ready +// → stream → delete. The fake clientset + fake executor let us +// verify the full orchestration without a real cluster. +func TestStage_HappyPath(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{} + var out bytes.Buffer + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err = Stage(ctx, StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "cats_dogs", + Out: &out, + }) + if err != nil { + t.Fatalf("Stage: %v\nout:\n%s", err, out.String()) + } + + // Diagnostic output: customer should see the lifecycle + // breadcrumbs. Pin a few key phrases so a regression that + // silences them is caught. + for _, want := range []string{"Created stage Pod", "Waiting for stage Pod", "Streaming", "Staged"} { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q:\n%s", want, out.String()) + } + } + + // Executor must have been called with the right ns + a tar + // stream on stdin (assert by inspecting captured tar). + if fe.gotNS != "tracebloc" { + t.Errorf("executor ns = %q, want tracebloc", fe.gotNS) + } + if len(fe.gotStdin) == 0 { + t.Error("executor stdin was empty; expected tar archive bytes") + } +} + +// TestStage_CleanupRunsOnError: deferred DeleteStagePod must fire +// even when StreamLayout fails. Otherwise every failed push leaks +// an orphan Pod that can't be reused for retry (the random suffix +// avoids name collisions, but it's still cluster pollution). +func TestStage_CleanupRunsOnError(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{ + errToReturn: errors.New("simulated stream failure"), + drainBeforeReturn: true, + } + var out bytes.Buffer + + err = Stage(context.Background(), StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "t1", + Out: &out, + }) + if err == nil { + t.Fatal("Stage returned nil on stream failure") + } + + // Cleanup contract: NO stage Pods should remain after Stage + // returns. The list-by-label query is the same one orphan.go + // uses, so if any leaks it's user-visible in the next push. + pods, listErr := cs.CoreV1().Pods("tracebloc").List(context.Background(), + metav1.ListOptions{LabelSelector: StagePodManagedByLabel + "=" + StagePodManagedByValue}) + if listErr != nil { + t.Fatalf("post-Stage list: %v", listErr) + } + if len(pods.Items) != 0 { + var names []string + for _, p := range pods.Items { + names = append(names, p.Name) + } + t.Errorf("Stage leaked %d Pod(s) after stream failure: %v", len(pods.Items), names) + } +} + +// TestStage_OrphanWarningSurfaces: stale stage Pods from a +// previous crash get surfaced as a warning at the start of the +// next push. Validates the integration: orphan scan → format → +// print to Out. +func TestStage_OrphanWarningSurfaces(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset(stagePodWithAge("stale-from-crash", "cats_dogs", 30*time.Minute)) + readyOnNextGet(cs) + fe := &fakeExecutor{} + var out bytes.Buffer + + err = Stage(context.Background(), StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "new_push", + Out: &out, + }) + if err != nil { + t.Fatalf("Stage: %v\nout:\n%s", err, out.String()) + } + + if !strings.Contains(out.String(), "stale-from-crash") { + t.Errorf("orphan warning didn't surface; output:\n%s", out.String()) + } + if !strings.Contains(out.String(), "kubectl delete pod") { + t.Errorf("orphan warning missing actionable hint; output:\n%s", out.String()) + } +} + +// TestStage_IngestorSANameFlowsToPod: the --ingestor-sa override +// MUST land on the stage Pod's ServiceAccountName — otherwise the +// flag is silently ignored and customers who renamed the chart's +// ingestor SA get pods running as the default SA (no PVC write +// access). Pin the integration here at the Stage layer since the +// flag wiring's effect is only observable end-to-end. Bugbot +// flagged the missing test coverage on PR-a. +func TestStage_IngestorSANameFlowsToPod(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{} + var out bytes.Buffer + + const customSA = "my-renamed-ingestor" + _ = Stage(context.Background(), StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: customSA, + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "t1", + Out: &out, + }) + + // List the stage Pods created (we deleted them in Stage's + // defer, so we need to inspect the tracker BEFORE the test + // process finishes — but the cleanup happens within Stage). + // Workaround: assert via the fake clientset's action log — + // look at the Create action's object directly. + var seenSA string + for _, action := range cs.Actions() { + if action.GetVerb() == "create" && action.GetResource().Resource == "pods" { + pod := action.(k8stesting.CreateAction).GetObject().(*corev1.Pod) + seenSA = pod.Spec.ServiceAccountName + break + } + } + if seenSA != customSA { + t.Errorf("created Pod's ServiceAccountName = %q, want %q (the --ingestor-sa override)", + seenSA, customSA) + } +} + +// TestStage_CancelledContext_StillCleansUp: when the parent ctx +// is cancelled (SIGINT scenario), the deferred cleanup MUST still +// run — using its own context with a fresh deadline. Without +// this, every Ctrl-C leaves an orphan Pod. +func TestStage_CancelledContext_StillCleansUp(t *testing.T) { + root := imgcDir(t) + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + cs := fake.NewClientset() + readyOnNextGet(cs) + fe := &fakeExecutor{ + // Simulate the SPDY stream returning ctx.Err() because the + // caller cancelled mid-stream. + errToReturn: context.Canceled, + } + var out bytes.Buffer + + // Use a context we cancel immediately. Stage's create still + // succeeds (fake clientset doesn't honor ctx on Create), but + // the executor sees Canceled. + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _ = Stage(ctx, StageOptions{ + Client: cs, + Executor: fe, + Namespace: "tracebloc", + IngestorSAName: "ingestor", + PVCClaimName: "client-pvc", + PVCMountPath: "/data/shared", + Layout: layout, + Table: "interrupted", + Out: &out, + }) + + // Whether Stage returns an error or not, the cluster MUST be + // clean. This is the SIGINT-safety contract. + pods, _ := cs.CoreV1().Pods("tracebloc").List(context.Background(), + metav1.ListOptions{LabelSelector: StagePodManagedByLabel + "=" + StagePodManagedByValue}) + if len(pods.Items) != 0 { + t.Errorf("Stage leaked %d Pod(s) on context-cancel; SIGINT cleanup contract broken", + len(pods.Items)) + } +} diff --git a/internal/push/stream.go b/internal/push/stream.go new file mode 100644 index 0000000..bdeff33 --- /dev/null +++ b/internal/push/stream.go @@ -0,0 +1,509 @@ +package push + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path" + "path/filepath" + "sort" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/remotecommand" +) + +// Executor abstracts "exec a command in a Pod with stdin/stdout/ +// stderr." The interface exists for one reason: client-go's +// fake.Clientset (the cornerstone of every test in this package) +// doesn't support the exec subresource. Without an interface +// boundary here, Stream's lifecycle couldn't be unit-tested — +// every assertion would need a real cluster. +// +// Production uses SPDYExecutor (this file). Tests construct a +// FakeExecutor (stream_test.go) that captures stdin into a buffer +// + records the command + can return a synthetic error. +type Executor interface { + // Exec runs cmd in the named container of ns/pod, wiring + // stdin/stdout/stderr to the provided streams. Returns when + // the remote command terminates or the context is cancelled. + // + // stdin, stdout, stderr may be nil — Exec must tolerate that + // (a stage Pod with no stderr to capture is a legitimate + // case for the "just delete me" command after the tar is + // done). + Exec(ctx context.Context, ns, pod, container string, + cmd []string, stdin io.Reader, stdout, stderr io.Writer) error +} + +// SPDYExecutor is the production Executor. Wraps client-go's +// remotecommand.NewSPDYExecutor — the same machinery `kubectl exec` +// uses internally. +// +// Holds both Config (needed by NewSPDYExecutor for auth/transport) +// and Client (for building the exec URL via the REST client). Both +// come from cluster.Load + cluster.NewClientset. +type SPDYExecutor struct { + Config *rest.Config + Client kubernetes.Interface +} + +func (e *SPDYExecutor) Exec( + ctx context.Context, + ns, pod, container string, + cmd []string, + stdin io.Reader, stdout, stderr io.Writer, +) error { + // Build the exec URL the way kubectl does — POST against the + // /pods//exec subresource, parametrized via PodExecOptions + // encoded through the scheme's ParameterCodec. + req := e.Client.CoreV1().RESTClient().Post(). + Resource("pods"). + Name(pod). + Namespace(ns). + SubResource("exec"). + VersionedParams(&corev1.PodExecOptions{ + Container: container, + Command: cmd, + // Toggle the streams that the caller actually wants. + // PodExecOptions ignores the unused ones, but being + // explicit here matches kubectl's behavior and means + // "stdin requested but caller passed nil" surfaces as + // a clearer error from the executor. + Stdin: stdin != nil, + Stdout: stdout != nil, + Stderr: stderr != nil, + }, scheme.ParameterCodec) + + exec, err := remotecommand.NewSPDYExecutor(e.Config, "POST", req.URL()) + if err != nil { + return fmt.Errorf("creating SPDY executor for %s/%s: %w", ns, pod, err) + } + + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + // Tty=false: we're not running an interactive shell, we're + // piping bytes. With Tty=true the tar stream would get + // line-buffered/CR-translated/etc, which corrupts the + // archive. + Tty: false, + }) + if err != nil { + return fmt.Errorf("exec stream against %s/%s: %w", ns, pod, err) + } + return nil +} + +// StreamLayout pipes a tar of the layout's files into the stage +// Pod's `tar -xf - -C /data/shared/
` so the files land at +// /data/shared/
/{labels.csv, images/...}. +// +// On success, the stage Pod's PVC mount now contains the files +// jobs-manager's ingestor will read in Phase 4. +// +// On failure, the partial-write state on the PVC is whatever tar +// managed before the error — possibly a few image files but +// likely not all. Phase 4's idempotency_key handles "retry the +// whole push"; for v0.1, customers re-run the command and the +// shell's `mkdir -p` is idempotent so re-creating the destination +// is fine. Truly-recovering a failed mid-stream push (resume from +// last-written file) is a v0.2 optimization. +// +// Why `tar` (rather than a more modern protocol like a custom REST +// upload endpoint on jobs-manager): tar-over-exec works against +// any cluster running the existing chart, with zero server-side +// changes. A custom upload endpoint would require a coordinated +// jobs-manager release before the CLI could ship. +func StreamLayout( + ctx context.Context, + exec Executor, + namespace, podName, containerName string, + layout *LocalLayout, + table string, + progress Progress, +) error { + if progress == nil { + progress = NoOpProgress{} + } + defer progress.Finish() + + // Generate the staging-dir suffix + compose the remote command + // BEFORE spawning the tar goroutine. Earlier versions did the + // suffix after, which meant a crypto/rand failure would return + // from this function without closing the pipe or waiting on + // the goroutine — the goroutine would block once the ~64 KB + // pipe buffer filled, leaking forever. Bugbot flagged on r9. + // + // Staging dir gets a fresh 8-hex-char suffix per invocation so + // two concurrent `dataset push` runs for the same table don't + // race on the same .staging path (r8). Without this, push A's + // `rm -rf .staging` could wipe push B's in-progress tar + // extraction (or vice versa), producing an interleaved + // corrupt dataset on the PVC. + stagingSuffix, err := randomSuffix(4) // 4 bytes → 8 hex chars + if err != nil { + return fmt.Errorf("generating staging-dir suffix: %w", err) + } + dest := StagedPrefix(table) + staging := dest + ".staging-" + stagingSuffix + + // Backup path for the swap. Same unique suffix as staging so + // two parallel pushes don't collide on each other's .old-, + // AND so the find pattern below catches both .staging-* and + // .old-* siblings as orphan candidates. + backup := dest + ".old-" + stagingSuffix + + parentDir := path.Dir(dest) + tableBase := path.Base(dest) + + // Remote command pipeline. Crosses three semantic guarantees: + // + // - HERMETIC (r5): old files don't survive into a re-push + // - TRANSACTIONAL (r7+r10): the previously-staged dataset + // stays intact if THIS push fails mid-transfer; backup- + // and-rollback restores on mv failure + // - PARALLEL-SAFE (r8): unique .staging-/.old- + // suffix per invocation so concurrent pushes don't + // interleave each other's tar extraction + // - EXIT-FAITHFUL (r11): set -e propagates any non-zero + // status; the prior `&&-chain ; find || true` shape + // could silently return success on actual failures + // + // Pipeline steps: + // 1. set -e — any unhandled non-zero status aborts the script + // 2. rm -rf $STAGING (clean prior failed attempt) + // 3. mkdir + tar (extract to staging) + // 4. If $DEST exists, mv $DEST → $DEST.old- (backup) + // 5. mv $STAGING → $DEST (commit) + // On failure: restore backup, exit 1 + // 6. rm -rf $BACKUP (success) + // 7. find ... orphan cleanup (best-effort, || true) + // + // The earlier `&&-chained ... ; find ... || true` shape had a + // hidden correctness bug (Bugbot r11): POSIX `;` runs the + // next command regardless of prior status, and `|| true` then + // forces exit 0. A failed tar mid-script would silently + // return success to the exec subprocess, and the CLI would + // report "Staged N files" on what was actually a failed push. + // + // set -e fixes that: any unguarded non-zero exits the script + // with that status. The find's `|| true` is still fine + // because under set -e a `cmd || fallback` is treated as + // "cmd may fail" and doesn't trigger the immediate exit. + // + // Failure modes (recap): + // - tar fails: set -e aborts; $DEST untouched (transactional from r7) + // - backup mv fails: same — aborts before any change to $DEST + // - main mv fails: explicit if/then rolls back the backup + // then `exit 1` (set -e doesn't preempt the explicit + // handler because of the `||`) + // - final rm fails: set -e exits, but the customer's new + // data is already in place — the leftover .old- + // gets picked up by the find sweep on next invocation + // - find fails: `|| true` suppresses (cleanup is best-effort) + remoteCmd := []string{ + "/bin/sh", "-c", + fmt.Sprintf( + "set -e\n"+ + "rm -rf %q\n"+ + "mkdir -p %q\n"+ + "/bin/tar -xf - -C %q\n"+ + "if [ -e %q ]; then mv %q %q; fi\n"+ + "if ! mv %q %q; then\n"+ + " if [ -e %q ]; then mv %q %q; fi\n"+ + " exit 1\n"+ + "fi\n"+ + "rm -rf %q\n"+ + "find %q -maxdepth 1 \\( -name %q -o -name %q \\) -mmin +60 -exec rm -rf {} + 2>/dev/null || true\n", + staging, // rm + staging, // mkdir + staging, // tar -C + dest, dest, backup, // if exists, backup + staging, dest, // main mv + backup, backup, dest, // rollback + backup, // success cleanup + parentDir, tableBase+".staging-*", tableBase+".old-*"), // find + } + + // Use a synchronous pipe so the tar Writer's writes block on + // the exec stream's reads — no in-memory buffering of the + // whole archive. For a 1 GiB dataset this is the difference + // between using 1 GiB of laptop RAM and using a few KB. + pr, pw := io.Pipe() + + // Wire the progress sink on the WRITE side so every byte the + // tar Writer emits (headers + bodies) counts toward the bar. + countedPW := &progressWriter{w: pw, p: progress} + + // Capture stderr from the in-cluster tar so failures surface + // with the actual remote message. + var stderrBuf bytes.Buffer + + // Kick off the tar build in a goroutine. The Pipe.Writer MUST + // be closed when we're done — without that, the Pipe.Reader + // side blocks forever waiting for more bytes. + // + // CloseWithError vs Close: CloseWithError preserves the tar + // error across the pipe so the exec side sees the cause. + tarErrCh := make(chan error, 1) + go func() { + err := writeLayoutTar(countedPW, layout) + if err != nil { + _ = pw.CloseWithError(err) + } else { + _ = pw.Close() + } + tarErrCh <- err + }() + + streamErr := exec.Exec(ctx, namespace, podName, containerName, + remoteCmd, pr, nil, &stderrBuf) + + // Close the reader side. If the executor returned without + // fully draining stdin (early exit, ctx cancellation, remote + // tar dying), the tar-write goroutine is blocked on its next + // pipe.Write — closing the reader unblocks it with + // io.ErrClosedPipe so it can finish and report on tarErrCh. + // Without this, every error path deadlocks. + _ = pr.Close() + + // Drain the tar goroutine — even on stream error, it must + // finish or we leak. Buffered channel size 1 means this + // receive won't block (the goroutine already sent or will + // send imminently after CloseWithError or the pr.Close above). + tarErr := <-tarErrCh + + // Order matters here. Bugbot r11 caught the previous logic: + // when the LOCAL tar build fails (e.g. r8's stream-time size + // cap recheck rejecting a file that grew on disk), the + // CloseWithError on the pipe causes exec to see "broken pipe" + // and return a generic stream error. If we report streamErr + // first, the customer sees "streaming failed" instead of the + // actually-actionable "dataset exceeded v0.1 cap" diagnostic + // from the tar side. + // + // Check tarErr first — when both are non-nil, the tar side + // is usually the upstream cause. streamErr-only (no tarErr) + // is the genuine network/RBAC/remote-tar-failed case where + // the exec wording is the right surface. + if tarErr != nil { + return fmt.Errorf("building tar archive: %w", tarErr) + } + if streamErr != nil { + hint := "" + if remote := stderrBuf.String(); remote != "" { + hint = fmt.Sprintf(" (remote tar stderr: %s)", remote) + } + return fmt.Errorf("streaming files to %s/%s: %w%s", namespace, podName, streamErr, hint) + } + return nil +} + +// writeLayoutTar packages the layout's files into the writer as a +// flat tar archive with this structure: +// +// labels.csv (file) +// images/ (file, per layout.Images) +// +// The destination tar is unpacked with `tar -xf - -C /data/shared/
`, +// so file paths are RELATIVE to that root — that's why we write +// "labels.csv" not "/data/shared/
/labels.csv". +// +// Mode is 0644 on all files (read-only for the ingestor SA). Tar's +// type flag is TypeReg (regular file). No symlinks, no hard links, +// no extended attrs — the layout we accept doesn't have any. +// +// Uses a named return + deferred Close so a tar trailer-write error +// (truncated archive — GNU tar refuses to extract these) propagates +// even on the happy path. errcheck would otherwise flag the bare +// `defer tw.Close()`, and rightly: silently dropping that error +// means a truncated stream looks identical to a successful one +// from this function's caller. +func writeLayoutTar(w io.Writer, layout *LocalLayout) (err error) { + tw := tar.NewWriter(w) + defer func() { + // If we already have an error, preserve it; the close error + // is less useful than whatever caused the early return. + // Otherwise surface the close error so a failed trailer + // write doesn't silently corrupt the archive. + if cerr := tw.Close(); cerr != nil && err == nil { + err = fmt.Errorf("closing tar writer: %w", cerr) + } + }() + + // Running total enforces the v0.1 MaxTotalBytes cap at STREAM + // time, not just at Discover time. A file that grew between + // pre-flight and now (TOCTOU) — or just a sum miscount — would + // otherwise sneak past the cap silently. Bugbot flagged on + // PR-b r8 as Medium. + var totalBytes int64 + + // labels.csv first (small, sanity-checks the stream quickly). + n, err := writeTarFile(tw, layout.LabelsCSV, "labels.csv") + if err != nil { + return fmt.Errorf("packaging labels.csv: %w", err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s during stream "+ + "(labels.csv alone is %s; pre-flight likely raced with "+ + "a file growing on disk)", + HumanBytes(MaxTotalBytes), HumanBytes(totalBytes)) + } + + // Then each image. We write them in the order Discover returned + // (filesystem-walk order, not sorted) — sorting would make the + // stream deterministic across runs but adds zero customer value. + for _, abs := range layout.Images { + // The destination filename inside images/ is the file's + // basename — strips the customer's local path so a push + // from /home/alice/datasets/cats_dogs/images/001.jpg + // becomes images/001.jpg in the tar (and on the PVC). + // + // Use path.Join (always forward-slash) NOT filepath.Join + // (OS-native separator) for the tar HEADER name. USTAR/POSIX + // tar requires forward slashes; a Windows-built CLI using + // filepath.Join would emit `images\001.jpg` headers that the + // Linux stage Pod's `tar -xf` either rejects or extracts as + // a flat-named file. Bugbot flagged on PR-b round 6. + dst := path.Join("images", filepath.Base(abs)) + n, err := writeTarFile(tw, abs, dst) + if err != nil { + return fmt.Errorf("packaging %s: %w", abs, err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s after streaming %s "+ + "(reached %s; file growth between pre-flight and stream)", + HumanBytes(MaxTotalBytes), dst, HumanBytes(totalBytes)) + } + } + + // Extra root-level files (e.g. masked_language_modeling's + // tokenizer.json), staged at the table root under their dest name. + // Sorted for deterministic stream order. + for _, dest := range sortedKeys(layout.ExtraFiles) { + n, err := writeTarFile(tw, layout.ExtraFiles[dest], dest) + if err != nil { + return fmt.Errorf("packaging %s: %w", dest, err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s after streaming %s (reached %s)", + HumanBytes(MaxTotalBytes), dest, HumanBytes(totalBytes)) + } + } + + // Generic sidecar directories (texts/, sequences/, and — later — + // annotations/, masks/), each staged under "/". + // Sorted by dir name for deterministic stream order. + for _, name := range sortedKeys(layout.Sidecars) { + for _, abs := range layout.Sidecars[name] { + dst := path.Join(name, filepath.Base(abs)) + n, err := writeTarFile(tw, abs, dst) + if err != nil { + return fmt.Errorf("packaging %s: %w", abs, err) + } + totalBytes += n + if totalBytes > MaxTotalBytes { + return fmt.Errorf( + "dataset exceeded v0.1 total cap of %s after streaming %s (reached %s)", + HumanBytes(MaxTotalBytes), dst, HumanBytes(totalBytes)) + } + } + } + + // tw.Close() in the defer above writes the tar footer + // (two zero blocks). Without that, GNU tar treats the archive + // as truncated and refuses to extract. + return nil +} + +// sortedKeys returns a map's string keys in sorted order, for +// deterministic iteration when packaging ExtraFiles / Sidecars. +func sortedKeys[V any](m map[string]V) []string { + ks := make([]string, 0, len(m)) + for k := range m { + ks = append(ks, k) + } + sort.Strings(ks) + return ks +} + +// writeTarFile writes one file from `src` into tw under the +// archive-relative name `dst`. Streams the file body — no full- +// read into memory — so a single 500 MiB image doesn't balloon +// the CLI's memory. +// +// Returns the file's declared size (from os.Lstat) so the caller +// can maintain a running total against MaxTotalBytes — closes the +// stream-time half of the TOCTOU window Bugbot flagged on r8. +// +// Uses os.Lstat (not Stat) + an explicit symlink reject as a +// defense-in-depth second layer behind walk.go's rejectSymlink +// (Bugbot r4). Also re-checks the single-file size cap at stream +// time: a file that grew between Discover and now would otherwise +// upload past the advertised 500 MiB cap (Bugbot r8). +func writeTarFile(tw *tar.Writer, src, dst string) (int64, error) { + st, err := os.Lstat(src) + if err != nil { + return 0, err + } + if st.Mode()&os.ModeSymlink != 0 { + // Reaching here means Discover let a symlink through — + // shouldn't happen in production, but the explicit error + // keeps the security property locally enforceable. + return 0, fmt.Errorf("refusing to stream symbolic link %q (defense-in-depth; should have been rejected by Discover)", src) + } + if st.Size() > MaxSingleFileBytes { + // Stream-time recheck of the single-file cap. Discover + // caught this in pre-flight; if we see it again here, the + // file grew between then and now. + return 0, sizeError(dst, st.Size(), MaxSingleFileBytes) + } + hdr := &tar.Header{ + Name: dst, + Size: st.Size(), + Mode: 0o644, + Typeflag: tar.TypeReg, + // We don't carry ModTime forward — the customer's local + // mtime has no useful semantic on the PVC, and zeroing it + // keeps the tar bit-for-bit reproducible across runs + // (helps tests, makes future content-hash checks easier). + } + if err := tw.WriteHeader(hdr); err != nil { + return 0, err + } + f, err := os.Open(src) + if err != nil { + return 0, err + } + // Read-only file Close errors are not meaningful — there's + // nothing the caller can do, and the underlying file descriptor + // gets returned to the OS regardless. Explicit-discard pattern, + // same as the Fprintf writers elsewhere. + defer func() { _ = f.Close() }() + // io.CopyN caps the actual stream at the declared header size. + // Without the cap, a file that grew between Lstat above and + // the Copy here would overflow the header — the tar would be + // corrupted (header says N bytes, body has > N). Cap-and-trust + // is the safe choice: if the file shrank, the tar trailer + // surfaces the mismatch; if the file grew, we just stop at + // the declared size and let the new bytes get re-pushed next + // invocation. Closes the body-side half of the TOCTOU window. + if _, err := io.CopyN(tw, f, st.Size()); err != nil && !errors.Is(err, io.EOF) { + return 0, err + } + return st.Size(), nil +} diff --git a/internal/push/stream_test.go b/internal/push/stream_test.go new file mode 100644 index 0000000..ea0e055 --- /dev/null +++ b/internal/push/stream_test.go @@ -0,0 +1,441 @@ +package push + +import ( + "archive/tar" + "bytes" + "context" + "errors" + "io" + "regexp" + "sort" + "strings" + "testing" +) + +// fakeExecutor is the stand-in for SPDYExecutor in tests. Captures +// every Exec call's parameters into fields the test can assert on, +// and reads the stdin stream into a buffer (so we can parse the tar +// archive back out and verify its contents). +type fakeExecutor struct { + // Captured call parameters. + gotNS, gotPod, gotContainer string + gotCmd []string + + // Captured stdin (the tar archive). + gotStdin []byte + + // What to write back to the caller as stderr (simulates a + // "tar: no space left on device" style remote-side failure). + stderrToReturn []byte + + // What to return as the exec error (simulates a 4xx/5xx from + // the apiserver, a network drop, etc.). + errToReturn error + + // drainBeforeReturn: if false, errToReturn fires BEFORE the + // stdin pipe is drained — simulates a fast failure where the + // remote tar dies on the first byte. Test infrastructure for + // broken-pipe coverage. + drainBeforeReturn bool +} + +func (f *fakeExecutor) Exec( + _ context.Context, + ns, pod, container string, + cmd []string, + stdin io.Reader, _ io.Writer, stderr io.Writer, +) error { + f.gotNS, f.gotPod, f.gotContainer = ns, pod, container + f.gotCmd = cmd + + if stdin != nil && (f.drainBeforeReturn || f.errToReturn == nil) { + b, _ := io.ReadAll(stdin) + f.gotStdin = b + } + if len(f.stderrToReturn) > 0 && stderr != nil { + _, _ = stderr.Write(f.stderrToReturn) + } + return f.errToReturn +} + +// TestStreamLayout_TarPathsAreForwardSlash pins the cross-platform +// tar header convention. On Windows, filepath.Join uses '\' as +// separator — using it for the tar HEADER name would produce +// archive entries the Linux stage Pod's `tar -xf` either rejects +// or extracts as flat-named files (collapsing the images/ subdir). +// The fix is path.Join (always forward-slash); this test asserts +// the portable property regardless of which OS the test runs on. +// Bugbot flagged the Windows bug on PR-b round 6. +func TestStreamLayout_TarPathsAreForwardSlash(t *testing.T) { + root := imgcDir(t, "a.jpg", "b.png") + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{} + if err := StreamLayout(context.Background(), fe, "tracebloc", "p", "stage", + layout, "t", NoOpProgress{}); err != nil { + t.Fatalf("StreamLayout: %v", err) + } + + tr := tar.NewReader(bytes.NewReader(fe.gotStdin)) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("reading tar: %v", err) + } + if strings.Contains(hdr.Name, `\`) { + t.Errorf("tar entry %q contains backslash; "+ + "Linux stage Pod's `tar -xf` requires forward-slash paths", hdr.Name) + } + } +} + +// TestStreamLayout_TarContents end-to-end: the bytes the executor +// sees on stdin MUST be a valid tar with the layout's files at the +// expected paths (labels.csv at the root, images/ under +// images/). If this drifts, the in-cluster `tar -xf - -C +// /data/shared/
/` lands files in the wrong place and +// jobs-manager's ingestor in Phase 4 silently sees zero rows. +func TestStreamLayout_TarContents(t *testing.T) { + root := imgcDir(t, "a.jpg", "b.png") + layout, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + fe := &fakeExecutor{} + err = StreamLayout(context.Background(), fe, "tracebloc", "pod-x", "stage", + layout, "cats_dogs", NoOpProgress{}) + if err != nil { + t.Fatalf("StreamLayout: %v", err) + } + + // Parse the captured tar back out and collect (name, size). + type entry struct { + name string + size int64 + } + var got []entry + tr := tar.NewReader(bytes.NewReader(fe.gotStdin)) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("reading tar: %v", err) + } + got = append(got, entry{name: hdr.Name, size: hdr.Size}) + } + + // Sort for deterministic comparison — the producer walks + // images/ in OS order which isn't stable across filesystems. + sort.Slice(got, func(i, j int) bool { return got[i].name < got[j].name }) + + // Three entries: labels.csv + 2 images. + if len(got) != 3 { + t.Fatalf("tar entry count = %d, want 3 (labels.csv + 2 images); got=%v", len(got), got) + } + want := []entry{ + {name: "images/a.jpg", size: 100}, // imgcDir writes 100-byte images + {name: "images/b.png", size: 100}, + {name: "labels.csv", size: 39}, // "image_id,label\n001.jpg,cat\n002.jpg,dog\n" + } + for i := range want { + if got[i] != want[i] { + t.Errorf("entry[%d] = %v, want %v", i, got[i], want[i]) + } + } +} + +// TestStreamLayout_RemoteCommand pins the in-cluster command. If +// this ever drifts (e.g. someone changes the destination path +// helper), the tar gets extracted to the wrong PVC subdirectory +// and Phase 4 silently fails. The mkdir -p is what makes re-push +// overwrite semantics work. +func TestStreamLayout_RemoteCommand(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{} + _ = StreamLayout(context.Background(), fe, "tracebloc", "pod-x", "stage", + layout, "my_table", NoOpProgress{}) + + if got, want := len(fe.gotCmd), 3; got != want { + t.Fatalf("len(cmd) = %d, want %d", got, want) + } + if fe.gotCmd[0] != "/bin/sh" || fe.gotCmd[1] != "-c" { + t.Errorf("cmd[:2] = %v, want [/bin/sh -c]", fe.gotCmd[:2]) + } + script := fe.gotCmd[2] + // The remote command must be: + // 1. HERMETIC (Bugbot r5): old files don't survive a re-push + // 2. TRANSACTIONAL (Bugbot r7): tar failure preserves prev + // dataset + // 3. RACE-SAFE under parallel-push (Bugbot r8): unique + // .staging- suffix per invocation so concurrent pushes + // don't interleave each other's tar extraction + // + // Pattern: extract to .staging-, then on tar SUCCESS + // rm the old dest + mv staging → dest. The order matters: + // + // - tar BEFORE any rm of $DEST (preserves on tar failure) + // - mv AFTER tar succeeds (atomic-ish swap) + // + // `dest` is StagedPrefix(table) — the CLI's SOURCE staging dir, + // which (since #26) lives under SharedRoot/.tracebloc-staging/ so + // it never collides with the ingestor's DEST_PATH. Derive the + // expected paths from it so this test tracks StagedPrefix rather + // than hardcoding the prefix. + dest := StagedPrefix("my_table") + + // Extract the staging path with a regex so the random hex + // suffix doesn't pin us to a specific invocation's bytes. + stagingRE := regexp.MustCompile(regexp.QuoteMeta(dest) + `\.staging-[0-9a-f]{8}`) + stagingPaths := stagingRE.FindAllString(script, -1) + if len(stagingPaths) == 0 { + t.Fatalf("remote script has no %s.staging-<8hex> path (race-safety regression):\n%s", dest, script) + } + // All staging mentions must refer to the SAME suffix in a single + // invocation. If we see two distinct suffixes that's a bug: + // rm/mkdir/tar/mv would target different dirs. + for i := 1; i < len(stagingPaths); i++ { + if stagingPaths[i] != stagingPaths[0] { + t.Errorf("remote script mixes staging suffixes %q and %q:\n%s", + stagingPaths[0], stagingPaths[i], script) + } + } + staging := stagingPaths[0] + + // Pin the four key operations. + for _, want := range []string{ + `mkdir -p "` + staging + `"`, + `tar -xf - -C "` + staging + `"`, + `mv "` + staging + `" "` + dest + `"`, + } { + if !strings.Contains(script, want) { + t.Errorf("remote script missing %q: %s", want, script) + } + } + // Transactional property: tar must come BEFORE any mutation + // of the real destination. r5+r7 used `rm $DEST` for the + // pre-mv step; r10 swapped that to `mv $DEST $DEST.old-` + // (backup-and-swap, so a mv failure can be rolled back). + // The contract is the same: tar runs while $DEST is intact. + tarIdx := strings.Index(script, `tar -xf - -C "`+staging+`"`) + destBackupMvIdx := strings.Index(script, `mv "`+dest+`" "`) + if tarIdx < 0 || destBackupMvIdx < 0 { + t.Fatalf("remote script missing tar or destination-backup mv: %s", script) + } + if tarIdx >= destBackupMvIdx { + t.Errorf("remote script touches $DEST BEFORE tar succeeds — partial-transfer could destroy previous data:\n%s", script) + } + // Single-segment guarantee: rm targets must never be the shared + // root or the staging parent themselves (that would nuke sibling + // tables / every in-flight push). + for _, forbidden := range []string{ + `rm -rf "/data/shared"`, + `rm -rf "/data/shared/"`, + `rm -rf "` + SharedRoot + "/" + stagingDirName + `"`, + } { + if strings.Contains(script, forbidden) { + t.Errorf("remote script contains dangerous %q (would nuke sibling tables/pushes):\n%s", forbidden, script) + } + } + + // Bugbot r9 + r10: orphan cleanup for previously-failed pushes + // must catch BOTH .staging-* and .old-* siblings. The .old-* + // dirs are created by the round-10 backup-and-rollback swap; + // without including them in the find pattern, failed-mid-swap + // pushes would leak .old-* dirs. + for _, wantName := range []string{`-name "my_table.staging-*"`, `-name "my_table.old-*"`} { + if !strings.Contains(script, wantName) { + t.Errorf("remote script missing %s in orphan cleanup:\n%s", wantName, script) + } + } + if !strings.Contains(script, `-mmin +60 -exec rm -rf {} +`) { + t.Errorf("remote script missing -mmin+60 orphan window:\n%s", script) + } + + // Bugbot r10: backup-and-swap pattern. The previous dataset + // must be backed up to .old- BEFORE the new dataset + // arrives, and restored if the main mv fails. Pin the key + // shape pieces — backup mv, primary mv, rollback mv, cleanup. + backupRE := regexp.MustCompile(regexp.QuoteMeta(dest) + `\.old-[0-9a-f]{8}`) + backupPaths := backupRE.FindAllString(script, -1) + if len(backupPaths) == 0 { + t.Fatalf("remote script has no .old- backup path (r10 rollback regression):\n%s", script) + } + // All .old- mentions in a single invocation must agree — + // otherwise the rollback would target a different path than + // the backup created. + for i := 1; i < len(backupPaths); i++ { + if backupPaths[i] != backupPaths[0] { + t.Errorf("remote script mixes backup suffixes %q vs %q:\n%s", + backupPaths[0], backupPaths[i], script) + } + } + backup := backupPaths[0] + // Backup must use the SAME suffix as staging — different + // suffixes would defeat the "find -name ...staging-* ...old-*" + // orphan-cleanup symmetry, AND would risk collision with a + // concurrent push's .old-. + if strings.TrimPrefix(backup, dest+".old-") != + strings.TrimPrefix(staging, dest+".staging-") { + t.Errorf("backup and staging suffixes diverge: %q vs %q", backup, staging) + } + // Backup mv (DEST → .old) must appear BEFORE primary mv + // (.staging → DEST), or rollback wouldn't have anything to + // restore. + backupMvIdx := strings.Index(script, `mv "`+dest+`" "`+backup+`"`) + primaryMvIdx := strings.Index(script, `mv "`+staging+`" "`+dest+`"`) + if backupMvIdx < 0 || primaryMvIdx < 0 { + t.Fatalf("remote script missing backup or primary mv:\n%s", script) + } + if backupMvIdx >= primaryMvIdx { + t.Errorf("backup mv (DEST→.old) must come BEFORE primary mv (.staging→DEST); rollback contract broken:\n%s", script) + } +} + +// TestStreamLayout_StagingSuffixIsUniquePerInvocation: two back-to- +// back StreamLayout calls for the same table MUST produce different +// staging-dir suffixes. Without this, concurrent `dataset push` runs +// for the same table would race on the same .staging path and +// corrupt each other's tar extraction. Bugbot r8 fix. +func TestStreamLayout_StagingSuffixIsUniquePerInvocation(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + stagingRE := regexp.MustCompile(regexp.QuoteMeta(StagedPrefix("t")) + `\.staging-[0-9a-f]{8}`) + + collect := func() string { + fe := &fakeExecutor{} + if err := StreamLayout(context.Background(), fe, "tracebloc", "p", "stage", + layout, "t", NoOpProgress{}); err != nil { + t.Fatalf("StreamLayout: %v", err) + } + m := stagingRE.FindString(fe.gotCmd[2]) + if m == "" { + t.Fatalf("no staging path in: %s", fe.gotCmd[2]) + } + return m + } + a, b := collect(), collect() + if a == b { + t.Errorf("back-to-back StreamLayout produced identical staging path %q; race-safety regression", a) + } +} + +// TestStreamLayout_TargetingFromArgs pins that namespace + pod name +// flow through to the executor unchanged. Catches a refactor that +// hardcodes the namespace or builds the pod name in the wrong layer. +func TestStreamLayout_TargetingFromArgs(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{} + _ = StreamLayout(context.Background(), fe, "custom-ns", "weird-pod-name", "container-x", + layout, "t", NoOpProgress{}) + if fe.gotNS != "custom-ns" { + t.Errorf("namespace = %q, want custom-ns", fe.gotNS) + } + if fe.gotPod != "weird-pod-name" { + t.Errorf("pod = %q, want weird-pod-name", fe.gotPod) + } + if fe.gotContainer != "container-x" { + t.Errorf("container = %q, want container-x", fe.gotContainer) + } +} + +// TestStreamLayout_ExecErrorWithRemoteStderr: when the in-cluster +// tar fails (e.g. read-only FS, no-space), the remote stderr must +// surface in the customer-facing error. This is the actionable +// signal — without it, customers see "exec stream failed" with no +// way to diagnose what went wrong server-side. +func TestStreamLayout_ExecErrorWithRemoteStderr(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fe := &fakeExecutor{ + stderrToReturn: []byte("tar: write error: No space left on device"), + errToReturn: errors.New("command terminated with exit code 1"), + drainBeforeReturn: true, + } + err = StreamLayout(context.Background(), fe, "tracebloc", "p", "stage", + layout, "t", NoOpProgress{}) + if err == nil { + t.Fatal("StreamLayout returned nil on exec error") + } + for _, want := range []string{"streaming files", "No space left on device", "exit code 1"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %v", want, err) + } + } +} + +// TestStreamLayout_ProgressBytes: the Progress sink receives byte +// counts during the stream. This is the contract the schollz bar +// keys off — if the wrapper ever stops calling Add(), the bar would +// freeze at 0%. +func TestStreamLayout_ProgressBytes(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + fp := &fakeProgress{} + err = StreamLayout(context.Background(), &fakeExecutor{}, "tracebloc", "p", "stage", + layout, "t", fp) + if err != nil { + t.Fatalf("StreamLayout: %v", err) + } + // At minimum we expect the file bodies (200 + 39 = 239) plus + // tar headers (~512 per file = ~1536 bytes overhead minimum). + // Assert "some bytes flowed" rather than an exact count + // because the tar overhead is implementation-defined. + if fp.added < 239 { + t.Errorf("progress.Add total = %d, want at least 239 (file body bytes)", fp.added) + } + if !fp.finished { + t.Errorf("progress.Finish() never called") + } +} + +// TestWriteLayoutTar_LabelsCSVFirst: ordering matters for fail-fast +// diagnostic — labels.csv at the start means a corrupt tar is more +// likely to surface before we've shipped a hundred image bytes. +func TestWriteLayoutTar_LabelsCSVFirst(t *testing.T) { + layout, err := Discover(imgcDir(t)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + var buf bytes.Buffer + if err := writeLayoutTar(&buf, layout); err != nil { + t.Fatalf("writeLayoutTar: %v", err) + } + tr := tar.NewReader(&buf) + hdr, err := tr.Next() + if err != nil { + t.Fatalf("reading first entry: %v", err) + } + if hdr.Name != "labels.csv" { + t.Errorf("first entry name = %q, want labels.csv (ordering pins this)", hdr.Name) + } +} + +// fakeProgress: a Progress that records total bytes added and +// whether Finish was called. +type fakeProgress struct { + added int + finished bool +} + +func (p *fakeProgress) Add(n int) { p.added += n } +func (p *fakeProgress) Finish() { p.finished = true } diff --git a/internal/push/tabular.go b/internal/push/tabular.go new file mode 100644 index 0000000..8b10e74 --- /dev/null +++ b/internal/push/tabular.go @@ -0,0 +1,229 @@ +package push + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +// reservedColumns are framework-managed columns the ingestor adds +// itself; a user schema must not redeclare them — data-ingestors' +// database.create_table rejects collisions with a clear error. Schema +// auto-inference skips them so a CSV that happens to carry an `id` +// (or data_id, filename, …) column doesn't produce a schema the +// ingestor refuses. `label` is intentionally NOT reserved — it's the +// mapped label column. Mirrors database.py's _RESERVED set. +var reservedColumns = map[string]bool{ + "id": true, + "created_at": true, + "updated_at": true, + "status": true, + "data_intent": true, + "data_id": true, + "filename": true, + "extension": true, + "annotation": true, + "ingestor_id": true, +} + +// schemaInferenceSampleRows caps how many data rows InferSchema reads +// to decide each column's type. The whole CSV would be more accurate +// but a few thousand rows is plenty to distinguish INT/FLOAT/text in +// practice, and bounds the work for large files. A column whose true +// type only reveals itself past the sample (e.g. an int column that +// turns float on row 10k) is the case --schema exists to override. +const schemaInferenceSampleRows = 5000 + +// DiscoverTabular validates a local directory for a tabular / +// time-series ingestion. Unlike the image layout, tabular categories +// have NO sidecar files — the dataset IS a single CSV. The directory +// must contain exactly one .csv file; that becomes the labels/data +// CSV staged for the ingestor. +// +// The returned LocalLayout reuses the image layout's LabelsCSV field +// (staged as labels.csv) with an empty Images slice, so the existing +// tar/stream machinery handles it unchanged. +func DiscoverTabular(rootDir string) (*LocalLayout, error) { + abs, err := filepath.Abs(rootDir) + if err != nil { + return nil, fmt.Errorf("resolving %q: %w", rootDir, err) + } + st, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("reading dataset directory %q: %w", abs, err) + } + if !st.IsDir() { + return nil, fmt.Errorf( + "%q is not a directory; pass the directory containing the dataset CSV", abs) + } + + entries, err := os.ReadDir(abs) + if err != nil { + return nil, fmt.Errorf("reading %q: %w", abs, err) + } + var csvs []string + for _, e := range entries { + if e.IsDir() { + continue + } + if strings.EqualFold(filepath.Ext(e.Name()), ".csv") { + csvs = append(csvs, e.Name()) + } + } + sort.Strings(csvs) + switch len(csvs) { + case 0: + return nil, fmt.Errorf( + "no .csv file found in %q. Tabular / time-series categories expect a "+ + "single CSV holding the dataset (one column per feature, plus the "+ + "label column).", abs) + case 1: + // happy path + default: + return nil, fmt.Errorf( + "found %d .csv files in %q (%s); the tabular layout expects exactly one. "+ + "Put the dataset CSV in its own directory and re-run.", + len(csvs), abs, strings.Join(csvs, ", ")) + } + + csvPath := filepath.Join(abs, csvs[0]) + // Lstat (not Stat) so a symlinked CSV is rejected rather than + // silently followed — mirrors the image layout's symlink guard. + info, err := os.Lstat(csvPath) + if err != nil { + return nil, fmt.Errorf("stat %s: %w", csvs[0], err) + } + if err := rejectSymlink(info, csvs[0]); err != nil { + return nil, err + } + if info.Size() > MaxSingleFileBytes { + return nil, sizeError(csvs[0], info.Size(), MaxSingleFileBytes) + } + + layout := &LocalLayout{Root: abs, LabelsCSV: csvPath, TotalBytes: info.Size()} + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. For larger datasets, the "+ + "cloud-source path is on the v0.2 roadmap (tracebloc/client#147).", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + return layout, nil +} + +// ParseSchema parses a --schema flag value of the form +// "col:TYPE,col:TYPE,..." into a column→type map. Types are passed +// through verbatim (the ingestor validates them against the SQL types +// it supports: INT, BIGINT, FLOAT, BOOLEAN, DATE, DATETIME, +// TIMESTAMP, TIME, TEXT, VARCHAR(n), ...). Whitespace around tokens +// is trimmed. +func ParseSchema(s string) (map[string]string, error) { + out := map[string]string{} + for _, pair := range strings.Split(s, ",") { + pair = strings.TrimSpace(pair) + if pair == "" { + continue + } + col, typ, ok := strings.Cut(pair, ":") + col, typ = strings.TrimSpace(col), strings.TrimSpace(typ) + if !ok || col == "" || typ == "" { + return nil, fmt.Errorf( + "schema entry %q must be col:TYPE (e.g. age:INT,price:FLOAT)", pair) + } + out[col] = typ + } + if len(out) == 0 { + return nil, fmt.Errorf("--schema is empty; expected col:TYPE,col:TYPE,...") + } + return out, nil +} + +// InferSchema reads the CSV header and a sample of rows and infers a +// column→SQL-type map: all-integer columns → INT, otherwise +// all-numeric → FLOAT, otherwise VARCHAR(255). Empty cells are +// ignored when judging a column; a column with no non-empty sampled +// value falls back to VARCHAR(255). +// +// It's a convenience so customers don't hand-write a --schema for the +// common case. Non-numeric specials (timestamps, dates, booleans) +// infer as VARCHAR(255); pass --schema to declare them precisely. +// +// Framework-managed columns (see reservedColumns — id, data_id, …) +// are skipped and returned as the second value so the caller can tell +// the customer they weren't included. +func InferSchema(csvPath string) (schema map[string]string, skipped []string, err error) { + f, err := os.Open(csvPath) + if err != nil { + return nil, nil, err + } + defer func() { _ = f.Close() }() + + r := csv.NewReader(f) + header, err := r.Read() + if err != nil { + return nil, nil, fmt.Errorf("reading CSV header from %s: %w", csvPath, err) + } + if len(header) == 0 { + return nil, nil, fmt.Errorf("CSV %s has no columns", csvPath) + } + + // Per-column running judgement. + couldBeInt := make([]bool, len(header)) + couldBeFloat := make([]bool, len(header)) + sawValue := make([]bool, len(header)) + for i := range header { + couldBeInt[i] = true + couldBeFloat[i] = true + } + + for n := 0; n < schemaInferenceSampleRows; n++ { + row, err := r.Read() + if err == io.EOF { + break + } + if err != nil { + return nil, nil, fmt.Errorf("reading CSV row from %s: %w", csvPath, err) + } + for i := 0; i < len(header) && i < len(row); i++ { + v := strings.TrimSpace(row[i]) + if v == "" { + continue + } + sawValue[i] = true + if couldBeInt[i] { + if _, e := strconv.ParseInt(v, 10, 64); e != nil { + couldBeInt[i] = false + } + } + if couldBeFloat[i] { + if _, e := strconv.ParseFloat(v, 64); e != nil { + couldBeFloat[i] = false + } + } + } + } + + schema = make(map[string]string, len(header)) + for i, col := range header { + col = strings.TrimSpace(col) + if reservedColumns[col] { + // Framework-managed (id, data_id, …): the ingestor adds it + // and rejects a schema that redeclares it. Skip + report. + skipped = append(skipped, col) + continue + } + switch { + case sawValue[i] && couldBeInt[i]: + schema[col] = "INT" + case sawValue[i] && couldBeFloat[i]: + schema[col] = "FLOAT" + default: + schema[col] = "VARCHAR(255)" + } + } + return schema, skipped, nil +} diff --git a/internal/push/tabular_test.go b/internal/push/tabular_test.go new file mode 100644 index 0000000..322101c --- /dev/null +++ b/internal/push/tabular_test.go @@ -0,0 +1,153 @@ +package push + +import ( + "os" + "path/filepath" + "testing" +) + +func writeFile(t *testing.T, dir, name, body string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +// TestDiscoverTabular_SingleCSV: a directory with exactly one CSV +// resolves to a layout whose LabelsCSV is that file and whose Images +// slice is empty (so the existing tar/stream machinery stages just +// the CSV). +func TestDiscoverTabular_SingleCSV(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", "a,b\n1,2\n") + + layout, err := DiscoverTabular(dir) + if err != nil { + t.Fatalf("DiscoverTabular: %v", err) + } + if layout.LabelsCSV != csv { + t.Errorf("LabelsCSV = %q, want %q", layout.LabelsCSV, csv) + } + if len(layout.Images) != 0 { + t.Errorf("Images = %v, want empty (tabular has no sidecar files)", layout.Images) + } + if layout.TotalBytes == 0 { + t.Errorf("TotalBytes = 0, want the CSV's size") + } +} + +// TestDiscoverTabular_NoCSV and _MultipleCSV: the layout requires +// exactly one CSV; zero or many is a clear, actionable error rather +// than a guess. +func TestDiscoverTabular_NoCSV(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "notes.txt", "hello") + if _, err := DiscoverTabular(dir); err == nil { + t.Error("DiscoverTabular with no .csv returned nil error") + } +} + +func TestDiscoverTabular_MultipleCSV(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "a.csv", "x\n1\n") + writeFile(t, dir, "b.csv", "y\n2\n") + if _, err := DiscoverTabular(dir); err == nil { + t.Error("DiscoverTabular with two .csv files returned nil error") + } +} + +// TestInferSchema covers the INT / FLOAT / VARCHAR inference from a +// CSV header + sample rows. Integer-only columns → INT, numeric (with +// a decimal) → FLOAT, anything else → VARCHAR(255). +func TestInferSchema(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", + "count,age,price,name\n1,30,9.99,alice\n2,40,19.5,bob\n") + + schema, _, err := InferSchema(csv) + if err != nil { + t.Fatalf("InferSchema: %v", err) + } + want := map[string]string{ + "count": "INT", + "age": "INT", + "price": "FLOAT", + "name": "VARCHAR(255)", + } + for col, typ := range want { + if schema[col] != typ { + t.Errorf("schema[%q] = %q, want %q (full: %v)", col, schema[col], typ, schema) + } + } +} + +// TestInferSchema_EmptyColumnIsVarchar: a column with no non-empty +// sampled value can't be typed, so it falls back to VARCHAR(255) +// rather than being mislabeled INT/FLOAT. +func TestInferSchema_EmptyColumnIsVarchar(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", "filled,empty\n1,\n2,\n") + schema, _, err := InferSchema(csv) + if err != nil { + t.Fatalf("InferSchema: %v", err) + } + if schema["empty"] != "VARCHAR(255)" { + t.Errorf("schema[empty] = %q, want VARCHAR(255)", schema["empty"]) + } + if schema["filled"] != "INT" { + t.Errorf("schema[filled] = %q, want INT", schema["filled"]) + } +} + +// TestInferSchema_SkipsReservedColumns: a CSV with an `id` (or other +// framework-managed) column must NOT produce a schema that includes +// it — data-ingestors' create_table rejects such collisions (the +// #135b guard). The reserved columns come back in the skipped list. +func TestInferSchema_SkipsReservedColumns(t *testing.T) { + dir := t.TempDir() + csv := writeFile(t, dir, "data.csv", "id,feature_00,label\n1,1.5,0\n2,2.5,1\n") + + schema, skipped, err := InferSchema(csv) + if err != nil { + t.Fatalf("InferSchema: %v", err) + } + if _, present := schema["id"]; present { + t.Errorf("schema includes reserved column id: %v", schema) + } + if schema["feature_00"] != "FLOAT" || schema["label"] != "INT" { + t.Errorf("schema = %v, want feature_00:FLOAT, label:INT", schema) + } + foundID := false + for _, s := range skipped { + if s == "id" { + foundID = true + } + } + if !foundID { + t.Errorf("skipped = %v, want it to contain id", skipped) + } +} + +func TestParseSchema(t *testing.T) { + got, err := ParseSchema("age:INT, price:FLOAT ,name:VARCHAR(255)") + if err != nil { + t.Fatalf("ParseSchema: %v", err) + } + want := map[string]string{"age": "INT", "price": "FLOAT", "name": "VARCHAR(255)"} + if len(got) != len(want) { + t.Fatalf("ParseSchema len = %d, want %d (%v)", len(got), len(want), got) + } + for k, v := range want { + if got[k] != v { + t.Errorf("ParseSchema[%q] = %q, want %q", k, got[k], v) + } + } + + for _, bad := range []string{"", "age", "age:", ":INT", "age=INT"} { + if _, err := ParseSchema(bad); err == nil { + t.Errorf("ParseSchema(%q) = nil error, want rejection", bad) + } + } +} diff --git a/internal/push/text.go b/internal/push/text.go new file mode 100644 index 0000000..163292e --- /dev/null +++ b/internal/push/text.go @@ -0,0 +1,172 @@ +package push + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// textExtensions are the file types the text / MLM ingestor reads by +// default (the schema's file_options.extension allows .txt / .text). +var textExtensions = map[string]struct{}{ + ".txt": {}, + ".text": {}, +} + +// DiscoverText validates a local directory for a text-family ingestion +// (text_classification or masked_language_modeling): +// +// - /labels.csv (required) +// - //*.txt (required; sidecar = texts | sequences) +// - /tokenizer.json (required for masked_language_modeling) +// +// The returned layout stages the CSV (as labels.csv), the text files +// under "/", and — for MLM — tokenizer.json at the table root +// (the ingestor reads it from SRC_PATH/tokenizer.json for [MASK]/[PAD]). +func DiscoverText(category, rootDir string) (*LocalLayout, error) { + abs, err := filepath.Abs(rootDir) + if err != nil { + return nil, fmt.Errorf("resolving %q: %w", rootDir, err) + } + st, err := os.Stat(abs) + if err != nil { + return nil, fmt.Errorf("reading dataset directory %q: %w", abs, err) + } + if !st.IsDir() { + return nil, fmt.Errorf( + "%q is not a directory; pass the directory containing labels.csv + the text files", abs) + } + + layout := &LocalLayout{ + Root: abs, + Sidecars: map[string][]string{}, + ExtraFiles: map[string]string{}, + } + dirName := TextSidecarDir(category) + + // labels.csv (required) — same Lstat-based symlink guard as the + // image layout. + labelsPath := filepath.Join(abs, "labels.csv") + labelsStat, err := os.Lstat(labelsPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing labels.csv in %q. Text categories expect "+ + "/labels.csv + /%s/.", abs, dirName) + } + return nil, fmt.Errorf("stat labels.csv: %w", err) + } + if err := rejectSymlink(labelsStat, "labels.csv"); err != nil { + return nil, err + } + if labelsStat.IsDir() { + return nil, fmt.Errorf("%q is a directory, not a file", labelsPath) + } + if labelsStat.Size() > MaxSingleFileBytes { + return nil, sizeError("labels.csv", labelsStat.Size(), MaxSingleFileBytes) + } + layout.LabelsCSV = labelsPath + layout.TotalBytes += labelsStat.Size() + + // Sidecar text dir (required). + files, sidecarBytes, err := discoverSidecarFiles(abs, dirName, textExtensions) + if err != nil { + return nil, err + } + if len(files) == 0 { + return nil, fmt.Errorf( + "no .txt files found in %q. Text categories expect "+ + "/%s/*.txt.", filepath.Join(abs, dirName), dirName) + } + layout.Sidecars[dirName] = files + layout.TotalBytes += sidecarBytes + + // masked_language_modeling needs a tokenizer.json at the root. + if category == "masked_language_modeling" { + tokPath := filepath.Join(abs, "tokenizer.json") + tokStat, terr := os.Lstat(tokPath) + if terr != nil { + if errors.Is(terr, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing tokenizer.json in %q. masked_language_modeling requires a "+ + "tokenizer.json (HuggingFace tokenizers format) at the dataset root; "+ + "the ingestor reads it for the [MASK]/[PAD] tokens.", abs) + } + return nil, fmt.Errorf("stat tokenizer.json: %w", terr) + } + if err := rejectSymlink(tokStat, "tokenizer.json"); err != nil { + return nil, err + } + if tokStat.IsDir() { + return nil, fmt.Errorf("%q is a directory, not a file", tokPath) + } + if tokStat.Size() > MaxSingleFileBytes { + return nil, sizeError("tokenizer.json", tokStat.Size(), MaxSingleFileBytes) + } + layout.ExtraFiles["tokenizer.json"] = tokPath + layout.TotalBytes += tokStat.Size() + } + + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. For larger datasets, the "+ + "cloud-source path is on the v0.2 roadmap (tracebloc/client#147).", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + return layout, nil +} + +// discoverSidecarFiles walks / (non-recursive) for files +// whose extension is in exts, rejecting symlinks and enforcing the +// single-file cap. Returns the absolute paths + their total size. A +// missing directory is an error (the caller's category requires it). +// +// Shared by the text family today; object detection / segmentation +// (annotations/, masks/) will reuse it in a later increment. +func discoverSidecarFiles(root, dirName string, exts map[string]struct{}) ([]string, int64, error) { + dir := filepath.Join(root, dirName) + dirStat, err := os.Lstat(dir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, 0, fmt.Errorf("missing %s/ subdirectory in %q", dirName, root) + } + return nil, 0, fmt.Errorf("stat %s/: %w", dirName, err) + } + if err := rejectSymlink(dirStat, dirName); err != nil { + return nil, 0, err + } + if !dirStat.IsDir() { + return nil, 0, fmt.Errorf("%q exists but is not a directory", dir) + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil, 0, fmt.Errorf("reading %s/: %w", dirName, err) + } + var ( + files []string + total int64 + ) + for _, entry := range entries { + if entry.IsDir() { + continue + } + if _, ok := exts[strings.ToLower(filepath.Ext(entry.Name()))]; !ok { + continue + } + info, err := entry.Info() + if err != nil { + return nil, 0, fmt.Errorf("stat %q: %w", entry.Name(), err) + } + if err := rejectSymlink(info, filepath.Join(dirName, entry.Name())); err != nil { + return nil, 0, err + } + if info.Size() > MaxSingleFileBytes { + return nil, 0, sizeError(filepath.Join(dirName, entry.Name()), info.Size(), MaxSingleFileBytes) + } + files = append(files, filepath.Join(dir, entry.Name())) + total += info.Size() + } + return files, total, nil +} diff --git a/internal/push/text_test.go b/internal/push/text_test.go new file mode 100644 index 0000000..66ae629 --- /dev/null +++ b/internal/push/text_test.go @@ -0,0 +1,133 @@ +package push + +import ( + "os" + "path/filepath" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/tracebloc/cli/internal/schema" +) + +// mkTextDir builds a text-family dataset dir: labels.csv + a sidecar +// directory (texts/ or sequences/) with two .txt files, optionally +// plus a tokenizer.json at the root (for MLM). +func mkTextDir(t *testing.T, sidecar string, withTokenizer bool) string { + t.Helper() + dir := t.TempDir() + writeFile(t, dir, "labels.csv", "filename,label\na.txt,pos\nb.txt,neg\n") + sub := filepath.Join(dir, sidecar) + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sub, "a.txt"), []byte("hello"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(sub, "b.txt"), []byte("world"), 0o644); err != nil { + t.Fatal(err) + } + if withTokenizer { + writeFile(t, dir, "tokenizer.json", `{"version":"1.0"}`) + } + return dir +} + +// TestDiscoverText_Classification: text_classification stages +// labels.csv + the texts/ directory, no images, no extra files. +func TestDiscoverText_Classification(t *testing.T) { + dir := mkTextDir(t, "texts", false) + layout, err := DiscoverText("text_classification", dir) + if err != nil { + t.Fatalf("DiscoverText: %v", err) + } + if len(layout.Sidecars["texts"]) != 2 { + t.Errorf("texts files = %d, want 2", len(layout.Sidecars["texts"])) + } + if len(layout.Images) != 0 { + t.Errorf("Images should be empty for text, got %v", layout.Images) + } + if len(layout.ExtraFiles) != 0 { + t.Errorf("ExtraFiles should be empty for text_classification, got %v", layout.ExtraFiles) + } + if got := layout.FileCount(); got != 3 { // labels.csv + 2 texts + t.Errorf("FileCount = %d, want 3", got) + } +} + +// TestDiscoverText_MLM_RequiresTokenizer: masked_language_modeling +// errors without tokenizer.json, and stages it as an ExtraFile when +// present (the ingestor reads SRC_PATH/tokenizer.json). +func TestDiscoverText_MLM_RequiresTokenizer(t *testing.T) { + if _, err := DiscoverText("masked_language_modeling", mkTextDir(t, "sequences", false)); err == nil { + t.Error("DiscoverText(MLM) without tokenizer.json returned nil error") + } + + layout, err := DiscoverText("masked_language_modeling", mkTextDir(t, "sequences", true)) + if err != nil { + t.Fatalf("DiscoverText(MLM): %v", err) + } + if len(layout.Sidecars["sequences"]) != 2 { + t.Errorf("sequences files = %d, want 2", len(layout.Sidecars["sequences"])) + } + if layout.ExtraFiles["tokenizer.json"] == "" { + t.Errorf("tokenizer.json not staged as an ExtraFile: %v", layout.ExtraFiles) + } + if got := layout.FileCount(); got != 4 { // labels.csv + 2 sequences + tokenizer + t.Errorf("FileCount = %d, want 4", got) + } +} + +// TestDiscoverText_MissingSidecarDir: a text dataset without its +// text-file directory is a clear error, not a silent empty stage. +func TestDiscoverText_MissingSidecarDir(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "labels.csv", "filename,label\na.txt,pos\n") + if _, err := DiscoverText("text_classification", dir); err == nil { + t.Error("DiscoverText without texts/ returned nil error") + } +} + +// TestBuild_Text_PassesSchema: the text Build branch emits the right +// sidecar field (texts vs sequences), a label for text_classification +// but NOT for masked_language_modeling, never an images field, and a +// schema-valid spec. +func TestBuild_Text_PassesSchema(t *testing.T) { + v, err := schema.NewV1Validator() + if err != nil { + t.Fatalf("NewV1Validator: %v", err) + } + check := func(name string, a SpecArgs, wantSidecar string, wantLabel bool) { + t.Run(name, func(t *testing.T) { + spec := a.Build() + if _, ok := spec[wantSidecar]; !ok { + t.Errorf("spec missing %q field: %v", wantSidecar, spec) + } + if _, hasImages := spec["images"]; hasImages { + t.Errorf("text spec emitted an images field: %v", spec) + } + if _, hasLabel := spec["label"]; hasLabel != wantLabel { + t.Errorf("label present = %v, want %v (%v)", hasLabel, wantLabel, spec) + } + b, err := yaml.Marshal(spec) + if err != nil { + t.Fatalf("marshal: %v", err) + } + _, errs, parseErr := v.ValidateYAML(b) + if parseErr != nil { + t.Fatalf("parse: %v\n%s", parseErr, b) + } + if len(errs) != 0 { + t.Fatalf("schema validation failed: %s\n%s", schema.FormatErrors(errs), b) + } + }) + } + + check("text_classification", SpecArgs{ + Table: "t_txt", Category: "text_classification", Intent: "train", LabelColumn: "label", + }, "texts", true) + + check("masked_language_modeling", SpecArgs{ + Table: "t_mlm", Category: "masked_language_modeling", Intent: "train", + }, "sequences", false) +} diff --git a/internal/push/walk.go b/internal/push/walk.go new file mode 100644 index 0000000..a6bb163 --- /dev/null +++ b/internal/push/walk.go @@ -0,0 +1,350 @@ +package push + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" +) + +// Size limits enforced before we touch the cluster. Both caps are +// soft engineering choices, not protocol limits — they exist +// because tar-over-exec via client-go's remotecommand executor has +// a memory profile that degrades steeply past ~1 GB total transfer. +// Customers hitting these get pointed at the v0.2 cloud-source +// story (S3/GCS/HTTPS sources, currently in epic non-goals). +// +// The single-file cap is stricter than the total cap because the +// streaming buffer for one file lives in memory longer than the +// inter-file overhead — a 1 GB single file is worse than ten 100 MB +// files for the executor's working-set. +const ( + // MaxTotalBytes is the v0.1 ceiling on the sum of all files in + // a single `dataset push`. Picked from the epic's stated + // "anything above ~1GB needs the cloud-source story (v0.2)." + MaxTotalBytes int64 = 1 * 1024 * 1024 * 1024 + + // MaxSingleFileBytes caps any single file. Tuned via the + // data-ingestors templates' largest sample image (~30 KB) and + // a 10000x safety margin for typical user uploads. Files in the + // hundreds-of-MB range work in testing but degrade noticeably. + MaxSingleFileBytes int64 = 500 * 1024 * 1024 +) + +// LocalLayout describes a validated local directory ready to stage. +// All paths are absolute, resolved against the customer's working +// directory before this struct is returned. +type LocalLayout struct { + // Root is the absolute path the customer passed (after cleanup). + Root string + + // LabelsCSV is the absolute path to labels.csv inside Root. + // Required for image_classification. + LabelsCSV string + + // Images is the list of absolute paths to image files under + // Root/images/. Order is filesystem-walk order — Discover + // doesn't sort, so callers that need determinism (e.g. + // reproducible-build tests) sort before use. Empty for non-image + // categories (which use Sidecars instead). + Images []string + + // Sidecars maps a sidecar directory name (e.g. "texts", + // "sequences", "annotations", "masks") to the absolute paths of + // the files in it. Each is staged under "/" — the + // generic counterpart to Images, used by the text family (and, + // later, object detection / segmentation). nil for image_ + // classification (which uses Images) and tabular (no sidecars). + Sidecars map[string][]string + + // ExtraFiles maps a staged destination filename to its absolute + // source path, for single root-level files beyond labels.csv — + // e.g. masked_language_modeling's tokenizer.json, which the + // ingestor reads from SRC_PATH/tokenizer.json. Staged verbatim at + // the table root. + ExtraFiles map[string]string + + // TotalBytes is the sum of all files Discover will stage — + // labels.csv plus every entry in Images / Sidecars / ExtraFiles. + // Pre-computed during the walk so the size-cap check + the + // progress bar can read it without re-stat'ing. + TotalBytes int64 +} + +// FileCount returns the total number of files this layout stages: +// labels.csv, every ExtraFile, and every Images / Sidecars entry. Used +// for the "staging N files" messaging so it's accurate across all +// category families. +func (l *LocalLayout) FileCount() int { + n := 1 // labels.csv + n += len(l.Images) + n += len(l.ExtraFiles) + for _, files := range l.Sidecars { + n += len(files) + } + return n +} + +// imageExtensions accepts the file types the chart's +// image_classification ingestor processes by default. From +// data-ingestors' FileTypeValidator(images) defaults: .jpg, .jpeg, +// .png. The chart's defaults file (see chartversion 1.3.5+) also +// accepts .webp; we mirror that here so customers on a recent +// chart can stage webp files without hitting "no images found." +// +// Comparison is case-insensitive — filesystems vary (case-sensitive +// on Linux, case-preserving-but-insensitive on macOS default APFS). +var imageExtensions = map[string]struct{}{ + ".jpg": {}, + ".jpeg": {}, + ".png": {}, + ".webp": {}, +} + +// Discover walks rootDir and validates it matches the layout Phase 3 +// expects for image_classification: +// +// - /labels.csv (required) +// - /images/*.{jpg,jpeg,png,webp} (at least one file) +// +// Returns specific errors keyed to the layout mistakes a customer +// is most likely to hit — these surface as the CLI's diagnostic +// output before any cluster work, so they're a primary UX surface. +// +// Enforces both v0.1 size caps (MaxTotalBytes, MaxSingleFileBytes); +// over-cap returns ErrTooBig with a pointer to the cloud-source +// story. +func Discover(rootDir string) (*LocalLayout, error) { + abs, err := filepath.Abs(rootDir) + if err != nil { + return nil, fmt.Errorf("resolving %q: %w", rootDir, err) + } + + st, err := os.Stat(abs) + if err != nil { + // Stat covers both "path doesn't exist" and "permission + // denied" via the wrapped fs.PathError; the customer sees + // the underlying message which is already clear. + return nil, fmt.Errorf("reading dataset directory %q: %w", abs, err) + } + if !st.IsDir() { + return nil, fmt.Errorf( + "%q is not a directory; pass the directory containing labels.csv + images/", + abs) + } + + layout := &LocalLayout{Root: abs} + + // labels.csv (required). Use Lstat — NOT Stat — so a symlink + // shows up as a symlink (mode includes ModeSymlink) rather + // than being silently followed. v0.1 rejects symlinks entirely + // (see rejectSymlink); without Lstat the size cap below would + // see the symlink's own ~100-byte size while writeTarFile + // (which uses os.Stat → follows symlinks) would happily stream + // the target's full contents — a size-cap bypass and an + // arbitrary-local-file disclosure to the cluster PVC. Bugbot + // flagged this as Medium-severity security on PR-b round 4. + labelsPath := filepath.Join(abs, "labels.csv") + labelsStat, err := os.Lstat(labelsPath) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing labels.csv in %q. The CLI expects "+ + "/labels.csv + /images/ for image_classification; "+ + "see https://github.com/tracebloc/client/issues/147 for the "+ + "v0.1 layout contract.", + abs) + } + return nil, fmt.Errorf("stat labels.csv: %w", err) + } + if err := rejectSymlink(labelsStat, "labels.csv"); err != nil { + return nil, err + } + if labelsStat.IsDir() { + // A directory literally named "labels.csv" passes the + // os.Stat above — without this check the pre-flight would + // accept it, and PR-b's tar stream would fail confusingly + // trying to read a directory as a CSV. Symmetric with the + // imagesStat.IsDir() check below. + return nil, fmt.Errorf( + "%q is a directory, not a file. labels.csv must be the "+ + "CSV file holding the image_id,label rows.", + labelsPath) + } + if labelsStat.Size() > MaxSingleFileBytes { + return nil, sizeError("labels.csv", labelsStat.Size(), MaxSingleFileBytes) + } + layout.LabelsCSV = labelsPath + layout.TotalBytes += labelsStat.Size() + + // images/ subdir (required, must contain at least one + // image-extension file). Use os.Lstat — NOT Stat — so a + // symlinked-directory case (e.g. `ln -s /etc images`) gets + // caught here, not silently followed into the linked path. + // Without Lstat, the symlink-image fixes from the previous + // commit don't matter: the whole directory could be a link. + // Bugbot flagged the missing dir-level Lstat on PR-b round 5. + imagesDir := filepath.Join(abs, "images") + imagesStat, err := os.Lstat(imagesDir) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf( + "missing images/ subdirectory in %q. The CLI expects "+ + "/labels.csv + /images/*.{jpg,jpeg,png,webp}.", + abs) + } + return nil, fmt.Errorf("stat images/: %w", err) + } + if err := rejectSymlink(imagesStat, "images"); err != nil { + return nil, err + } + if !imagesStat.IsDir() { + return nil, fmt.Errorf("%q exists but is not a directory", imagesDir) + } + + // Walk just the images/ directory — we don't recurse, image + // classification's layout is flat. If a customer has nested + // subdirs (e.g. images/cats/ + images/dogs/), that's a + // different category convention and out of scope for v0.1. + entries, err := os.ReadDir(imagesDir) + if err != nil { + return nil, fmt.Errorf("reading images/: %w", err) + } + for _, entry := range entries { + if entry.IsDir() { + // Silently skip subdirectories so a stray .DS_Store or + // thumbnails dir doesn't error out the whole walk. + // We DO surface the count of accepted images at the + // end, so a customer with all-nested-subdirs gets + // "0 images found" which is the right diagnostic. + continue + } + ext := strings.ToLower(filepath.Ext(entry.Name())) + if _, ok := imageExtensions[ext]; !ok { + continue + } + // entry.Info() returns Lstat-like metadata for the + // directory entry (the symlink's own mode if it's a + // symlink, not the target's). That's exactly what we + // want here — combined with rejectSymlink it closes the + // symlink-bypass-size-caps hole Bugbot flagged on PR-b + // round 4. + info, err := entry.Info() + if err != nil { + return nil, fmt.Errorf("stat %q: %w", entry.Name(), err) + } + if err := rejectSymlink(info, filepath.Join("images", entry.Name())); err != nil { + return nil, err + } + if info.Size() > MaxSingleFileBytes { + return nil, sizeError(filepath.Join("images", entry.Name()), + info.Size(), MaxSingleFileBytes) + } + layout.Images = append(layout.Images, filepath.Join(imagesDir, entry.Name())) + layout.TotalBytes += info.Size() + } + + if len(layout.Images) == 0 { + return nil, fmt.Errorf( + "no image files found in %q. Expected .jpg, .jpeg, .png, or .webp; "+ + "got %d non-image entries.", + imagesDir, len(entries)) + } + + if layout.TotalBytes > MaxTotalBytes { + return nil, fmt.Errorf( + "dataset is %s, exceeds v0.1 cap of %s. "+ + "For larger datasets, the cloud-source path (S3/GCS/HTTPS) "+ + "is on the v0.2 roadmap — see tracebloc/client#147 non-goals. "+ + "Workaround for v0.1: split the push into multiple smaller "+ + "tables, or stage directly via the existing helm flow.", + HumanBytes(layout.TotalBytes), HumanBytes(MaxTotalBytes)) + } + + return layout, nil +} + +// rejectSymlink returns a non-nil error if info describes a symlink. +// v0.1 refuses symlinks under /{labels.csv, images, images/*} +// entirely because: +// +// - SECURITY: writeTarFile uses os.Open (which follows symlinks). +// Discover sized entries via DirEntry.Info() (which does not). +// A symlink whose target is a multi-GB local file would pass +// Discover's size cap (the symlink itself is ~100 bytes) yet +// stream the target's full contents to the cluster PVC — a +// size-cap bypass and arbitrary-local-file disclosure to the +// cluster admin. Bugbot caught this on PR-b round 4. +// - UX: legitimate image_classification datasets don't use +// symlinks. A clear "symlinks not supported" error is better +// than the alternative fixes (resolve + re-stat the target, +// blanket Stat() everywhere) — both of which would let the +// security hole creep back in on a future refactor. +// +// Customers with symlink-heavy layouts (rare; usually means their +// data lives on another filesystem) can `cp -L` to materialize +// the files before pushing. +// +// Known limitation: HARD LINKS are NOT rejected. The filesystem +// doesn't expose a Mode bit for hard links the way ModeSymlink +// distinguishes symlinks, and a high-Nlink check has too many +// false positives (legitimate hard-linked datasets where the +// dataset dir is the only entry point). The implicit security +// boundary is the CLI's process-level read permissions: a +// customer can only hard-link files they already have read +// access to, so the cluster admin reading a hard-linked +// /etc/shadow via the PVC isn't a privilege escalation — they'd +// need the CLI to be running as root for that to be exploitable, +// and the docs already recommend running as a low-privileged +// user. v0.2 may add openat(O_NOFOLLOW)-based sandboxing if a +// real customer needs harder isolation; tracked alongside the +// cloud-source story (#147 non-goals). +func rejectSymlink(info os.FileInfo, relPath string) error { + if info.Mode()&os.ModeSymlink == 0 { + return nil + } + return fmt.Errorf( + "%q is a symbolic link, which v0.1 does not allow in the dataset "+ + "layout (security: a symlink could escape the dataset tree or "+ + "bypass size caps). Materialize the link target (e.g. `cp -L`) "+ + "and re-run, or wait for v0.2's cloud-source story if the data "+ + "lives elsewhere.", + relPath) +} + +// sizeError builds the over-the-single-file-cap error with the same +// human-readable framing as the total-cap branch above. Centralized +// so the message stays consistent if we tune the wording later. +func sizeError(relPath string, got, cap int64) error { + return fmt.Errorf( + "file %q is %s, exceeds v0.1 single-file cap of %s. "+ + "For larger files, see tracebloc/client#147's v0.2 cloud-source story.", + relPath, HumanBytes(got), HumanBytes(cap)) +} + +// HumanBytes renders a byte count in the shortest readable unit. +// Not internationalized — the CLI is English-only for v0.1. +// +// Exported because the CLI's pre-flight summary (internal/cli) needs +// the identical formatting — keeping one implementation here means +// the size shown in an over-cap error and the size shown in the +// dry-run summary can never drift (Bugbot flagged the earlier +// copy-pasted variant on PR #8). +func HumanBytes(n int64) string { + const ( + KiB = 1024 + MiB = 1024 * KiB + GiB = 1024 * MiB + ) + switch { + case n >= GiB: + return fmt.Sprintf("%.2f GiB", float64(n)/float64(GiB)) + case n >= MiB: + return fmt.Sprintf("%.2f MiB", float64(n)/float64(MiB)) + case n >= KiB: + return fmt.Sprintf("%.2f KiB", float64(n)/float64(KiB)) + default: + return fmt.Sprintf("%d B", n) + } +} diff --git a/internal/push/walk_test.go b/internal/push/walk_test.go new file mode 100644 index 0000000..441400c --- /dev/null +++ b/internal/push/walk_test.go @@ -0,0 +1,366 @@ +package push + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" +) + +// imgcDir builds a valid image_classification layout under t.TempDir() +// and returns its absolute path. Used as the happy-path baseline that +// individual negative-case tests mutate. +func imgcDir(t *testing.T, withImages ...string) string { + t.Helper() + root := t.TempDir() + + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n002.jpg,dog\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images/: %v", err) + } + if len(withImages) == 0 { + withImages = []string{"001.jpg", "002.jpg"} + } + for _, name := range withImages { + // 100 bytes of stub data per image — keeps the total-bytes + // math predictable in TestDiscover_TotalBytesSum without + // generating real image headers. + if err := os.WriteFile(filepath.Join(imagesDir, name), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image %s: %v", name, err) + } + } + return root +} + +func TestDiscover_HappyPath(t *testing.T) { + root := imgcDir(t) + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if got.Root == "" || !filepath.IsAbs(got.Root) { + t.Errorf("Root = %q, want non-empty absolute path", got.Root) + } + if filepath.Base(got.LabelsCSV) != "labels.csv" { + t.Errorf("LabelsCSV basename = %q, want labels.csv", filepath.Base(got.LabelsCSV)) + } + if len(got.Images) != 2 { + t.Errorf("len(Images) = %d, want 2", len(got.Images)) + } +} + +func TestDiscover_TotalBytesSum(t *testing.T) { + // Two 100-byte images + the inline labels.csv string (39 bytes: + // "image_id,label\n001.jpg,cat\n002.jpg,dog\n"). 100+100+39 = 239. + // This pins the pre-cluster size summary the dry-run output + // prints — if we ever undercount, customers see "0 bytes" + // pre-push and panic. + root := imgcDir(t) + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + const want = int64(100 + 100 + 39) + if got.TotalBytes != want { + t.Errorf("TotalBytes = %d, want %d", got.TotalBytes, want) + } +} + +func TestDiscover_AcceptsAllImageExtensions(t *testing.T) { + // Mirror the chart's FileTypeValidator(images) defaults — if a + // customer's image-set has .png + .webp, both should stage. + root := imgcDir(t, "a.jpg", "b.jpeg", "c.png", "d.webp", "e.JPG") + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if len(got.Images) != 5 { + t.Errorf("len(Images) = %d, want 5 (case-insensitive); names=%v", + len(got.Images), got.Images) + } +} + +func TestDiscover_SkipsNonImageFiles(t *testing.T) { + // .DS_Store, thumbnails.db, sibling .yaml etc. should be + // silently skipped — not error-out, not stage. The "no image + // files" diagnostic only fires when *zero* accepted files + // remain after filtering. + root := imgcDir(t, "real.jpg") + if err := os.WriteFile(filepath.Join(root, "images", ".DS_Store"), + make([]byte, 50), 0o644); err != nil { + t.Fatalf("write .DS_Store: %v", err) + } + got, err := Discover(root) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if len(got.Images) != 1 { + t.Errorf("len(Images) = %d, want 1; .DS_Store should be filtered", len(got.Images)) + } +} + +func TestDiscover_MissingLabelsCSV(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "images"), 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "images", "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image: %v", err) + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected missing-labels error") + } + if !strings.Contains(err.Error(), "missing labels.csv") { + t.Errorf("error = %q, want it to mention missing labels.csv", err) + } +} + +func TestDiscover_MissingImagesDir(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("x"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected missing-images-dir error") + } + if !strings.Contains(err.Error(), "missing images/") { + t.Errorf("error = %q, want it to mention missing images/", err) + } +} + +func TestDiscover_NoAcceptedImageExtensions(t *testing.T) { + // images/ exists but only contains .gif and .bmp — neither + // in our accept-set. Customer should see "no image files" + // pointing at the accepted extensions, not a successful walk + // with len(Images)==0 that then succeeds the dry-run. + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("x"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + for _, n := range []string{"old.gif", "old.bmp"} { + if err := os.WriteFile(filepath.Join(imagesDir, n), []byte("x"), 0o644); err != nil { + t.Fatalf("write %s: %v", n, err) + } + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected no-images error") + } + if !strings.Contains(err.Error(), "no image files") { + t.Errorf("error = %q, want it to mention no image files", err) + } +} + +func TestDiscover_LabelsCSVIsDirectory(t *testing.T) { + // A directory literally named "labels.csv" — os.Stat succeeds, + // so without the IsDir guard the pre-flight would accept it and + // PR-b's tar stream would fail confusingly. Symmetric with the + // images/ check. Bugbot flagged the missing guard on PR #8. + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "labels.csv"), 0o755); err != nil { + t.Fatalf("mkdir labels.csv/: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images/: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write image: %v", err) + } + _, err := Discover(root) + if err == nil { + t.Fatal("Discover returned nil error; expected labels.csv-is-a-directory error") + } + if !strings.Contains(err.Error(), "is a directory") { + t.Errorf("error = %q, want it to mention 'is a directory'", err) + } +} + +// TestDiscover_RejectsSymlinkedImage is the security regression pin +// for the Bugbot-Medium finding on PR-b round 4. A symlink in +// images/ whose target points outside the dataset tree (or at a +// big local file) used to pass Discover's size cap (because +// DirEntry.Info() returns the link's own ~100-byte size) yet +// stream the target's full contents into the cluster PVC via +// writeTarFile's os.Stat+os.Open path. That's both a size-cap +// bypass AND an arbitrary-local-file disclosure to the cluster +// admin. v0.1 rejects symlinks outright. +func TestDiscover_RejectsSymlinkedImage(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require admin privileges on Windows") + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + // Create a real image file outside the dataset tree to be the + // symlink target. The symlink inside images/ points at it. + target := filepath.Join(t.TempDir(), "outside.jpg") + if err := os.WriteFile(target, make([]byte, 100), 0o644); err != nil { + t.Fatalf("write outside target: %v", err) + } + link := filepath.Join(imagesDir, "evil.jpg") + if err := os.Symlink(target, link); err != nil { + t.Fatalf("symlink: %v", err) + } + + _, err := Discover(root) + if err == nil { + t.Fatal("Discover accepted a symlinked image — security regression; v0.1 must reject") + } + for _, want := range []string{"symbolic link", "security", "cp -L"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q in: %v", want, err) + } + } +} + +// TestDiscover_RejectsSymlinkedImagesDir is the round-5 follow-up +// to the symlinked-FILE fix from round 4. The DIRECTORY itself +// could also be a symlink (e.g. `ln -s /etc images`), which the +// previous fix didn't catch because it only Lstat'd the entries +// INSIDE images/. Bugbot flagged it; this test pins the +// dir-level Lstat. +func TestDiscover_RejectsSymlinkedImagesDir(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require admin privileges on Windows") + } + root := t.TempDir() + if err := os.WriteFile(filepath.Join(root, "labels.csv"), + []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write labels.csv: %v", err) + } + // Create a real images dir somewhere ELSE that the symlink + // will point at. The symlink in `root` is what should be + // rejected. + realDir := filepath.Join(t.TempDir(), "real-images") + if err := os.MkdirAll(realDir, 0o755); err != nil { + t.Fatalf("mkdir real-images: %v", err) + } + if err := os.WriteFile(filepath.Join(realDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write img: %v", err) + } + if err := os.Symlink(realDir, filepath.Join(root, "images")); err != nil { + t.Fatalf("symlink images: %v", err) + } + + _, err := Discover(root) + if err == nil { + t.Fatal("Discover accepted a symlinked images/ directory — security regression") + } + if !strings.Contains(err.Error(), "symbolic link") { + t.Errorf("error missing symbolic-link rejection: %v", err) + } +} + +// TestDiscover_RejectsSymlinkedLabelsCSV: same hole, different +// file. The labels.csv path can also be a symlink (e.g. pointing +// at a huge local file). Lstat + reject covers both cases +// symmetrically. +func TestDiscover_RejectsSymlinkedLabelsCSV(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("symlinks require admin privileges on Windows") + } + root := t.TempDir() + target := filepath.Join(t.TempDir(), "outside-labels.csv") + if err := os.WriteFile(target, []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatalf("write outside target: %v", err) + } + link := filepath.Join(root, "labels.csv") + if err := os.Symlink(target, link); err != nil { + t.Fatalf("symlink: %v", err) + } + imagesDir := filepath.Join(root, "images") + if err := os.MkdirAll(imagesDir, 0o755); err != nil { + t.Fatalf("mkdir images: %v", err) + } + if err := os.WriteFile(filepath.Join(imagesDir, "a.jpg"), + make([]byte, 100), 0o644); err != nil { + t.Fatalf("write img: %v", err) + } + + _, err := Discover(root) + if err == nil { + t.Fatal("Discover accepted a symlinked labels.csv — security regression") + } + if !strings.Contains(err.Error(), "symbolic link") { + t.Errorf("error missing symbolic-link rejection: %v", err) + } +} + +func TestDiscover_NotADirectory(t *testing.T) { + // Customer passes a path to a single file instead of a dir. + // This is a common autocomplete-mistake (tab-completing + // into the file). Should be caught early with a clear error. + root := t.TempDir() + filePath := filepath.Join(root, "looks-like-data.tar") + if err := os.WriteFile(filePath, []byte("x"), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + _, err := Discover(filePath) + if err == nil { + t.Fatal("Discover returned nil error; expected not-a-directory error") + } + if !strings.Contains(err.Error(), "not a directory") { + t.Errorf("error = %q, want it to mention not a directory", err) + } +} + +func TestDiscover_OverSingleFileCap(t *testing.T) { + // Use a fake-size pattern: create a real small file but assert + // the cap logic by exercising the human-readable error format + // at the boundary. We can't easily generate a 500MB+ file in + // CI without slowing the suite — instead pin the human-bytes + // formatter (which is what the customer sees) via its own + // boundary test below, and exercise sizeError() directly. + got := sizeError("images/big.jpg", 600*1024*1024, MaxSingleFileBytes).Error() + for _, want := range []string{"images/big.jpg", "600.00 MiB", "500.00 MiB", "v0.2", "cloud-source"} { + if !strings.Contains(got, want) { + t.Errorf("sizeError missing %q in: %s", want, got) + } + } +} + +func TestHumanBytes(t *testing.T) { + // Boundary check: the formatter is what surfaces in every + // diagnostic, so a regression here makes the error messages + // unreadable for the customer. Pin a few representative values. + cases := []struct { + in int64 + want string + }{ + {0, "0 B"}, + {1023, "1023 B"}, + {1024, "1.00 KiB"}, + {1024 * 1024, "1.00 MiB"}, + {1024 * 1024 * 1024, "1.00 GiB"}, + {500 * 1024 * 1024, "500.00 MiB"}, + } + for _, c := range cases { + if got := HumanBytes(c.in); got != c.want { + t.Errorf("HumanBytes(%d) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/schema/embed.go b/internal/schema/embed.go new file mode 100644 index 0000000..9fa8f87 --- /dev/null +++ b/internal/schema/embed.go @@ -0,0 +1,30 @@ +// Package schema holds the embedded ingest schema(s) and the +// validator built on top of them. +// +// Today only v1 is supported. When a v2 lands, this package grows +// a SchemaVersion dispatch (per the ingest.yaml's apiVersion field) +// without changing the validator's public API. +// +// Why embed? Two reasons. +// +// 1. Local validation. `tracebloc ingest validate ` runs +// entirely offline — no cluster, no network — which is the +// whole point of doing it on the CLI rather than waiting for +// jobs-manager's POST-time check. +// 2. Drift detection. Bundling the schema makes drift between +// the CLI's view and data-ingestors' canonical source +// observable at build time, not at runtime. scripts/sync-schema.sh +// enforces parity in CI. +package schema + +import _ "embed" + +// V1Bytes is the raw JSON of ingest.v1.json, vendored from +// tracebloc/data-ingestors at build time via scripts/sync-schema.sh. +// +// Exposed as []byte (not parsed) so consumers can either pipe it +// into their preferred JSON-Schema implementation or inspect the +// raw $id / $schema fields to assert which version they got. +// +//go:embed ingest.v1.json +var V1Bytes []byte diff --git a/internal/schema/ingest.v1.json b/internal/schema/ingest.v1.json new file mode 100644 index 0000000..d8eb83c --- /dev/null +++ b/internal/schema/ingest.v1.json @@ -0,0 +1,402 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://tracebloc.io/schemas/ingest.v1.json", + "title": "tracebloc IngestConfig (v1)", + "description": "Declarative configuration for the tracebloc data ingestor. Replaces the per-customer Python script + Dockerfile pattern. Customers describe their dataset; the official image runs it.", + "type": "object", + "additionalProperties": false, + + "required": [ + "apiVersion", + "kind", + "category", + "table", + "intent" + ], + + "properties": { + "apiVersion": { + "const": "tracebloc.io/v1", + "description": "Schema version. Breaking changes require v2." + }, + "kind": { + "const": "IngestConfig" + }, + + "category": { + "type": "string", + "enum": [ + "image_classification", + "object_detection", + "keypoint_detection", + "semantic_segmentation", + "instance_segmentation", + "text_classification", + "tabular_classification", + "tabular_regression", + "time_series_forecasting", + "time_to_event_prediction", + "masked_language_modeling" + ], + "description": "Task category. Drives convention defaults: validators, data_format, default columns, default file extensions, default validator set." + }, + + "table": { + "type": "string", + "minLength": 1, + "description": "Destination table name in the cluster-internal MySQL. Used to identify the dataset in subsequent runs." + }, + + "intent": { + "type": "string", + "enum": ["train", "test"], + "description": "Whether this dataset is for training or held-out testing." + }, + + "csv": { + "type": "string", + "minLength": 1, + "description": "Path to the labels CSV. Mutually exclusive with `json`. The dominant source type." + }, + + "json": { + "type": "string", + "minLength": 1, + "description": "Path to a JSON dataset. Mutually exclusive with `csv`." + }, + + "images": { + "type": "string", + "minLength": 1, + "description": "Directory holding the image files referenced by the labels CSV. Required for image-based categories." + }, + + "annotations": { + "type": "string", + "minLength": 1, + "description": "Directory holding annotation files (e.g. Pascal VOC XML). Required for object_detection." + }, + + "masks": { + "type": "string", + "minLength": 1, + "description": "Directory holding segmentation mask images (PNG). Required for semantic_segmentation." + }, + + "texts": { + "type": "string", + "minLength": 1, + "description": "Directory holding text files referenced by the labels CSV. Required for text_classification." + }, + + "sequences": { + "type": "string", + "minLength": 1, + "description": "Directory holding sequence text files (.txt). Required for masked_language_modeling." + }, + + "label": { + "oneOf": [ + { + "type": "string", + "minLength": 1, + "description": "Shorthand: column name in the CSV that holds the label. Implies policy=passthrough." + }, + { + "type": "object", + "additionalProperties": false, + "required": ["column"], + "properties": { + "column": { + "type": "string", + "minLength": 1, + "description": "Column name in the CSV that holds the label." + }, + "policy": { + "type": "string", + "enum": ["passthrough", "bucket"], + "description": "How label values are sent to the central backend. `passthrough` (default for classification) sends the raw value. `bucket` (required for regression-class tasks) bins the value before send so the central backend never sees the raw target." + } + } + } + ], + "description": "Label specification. Can be a string (column name) for the common case, or an object for explicit policy control." + }, + + "schema": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "string" + }, + "description": "Column → SQL type map (e.g. {'age': 'INT', 'name': 'VARCHAR(255)'}). Required for tabular and time-series categories." + }, + + "time_column": { + "type": "string", + "minLength": 1, + "description": "Name of the time column for time_to_event_prediction. Falls back to a column named `time` if unset." + }, + + "target_size": { + "type": "array", + "items": { "type": "integer", "minimum": 1 }, + "minItems": 2, + "maxItems": 2, + "description": "[height, width] to resize images to. Required for keypoint_detection (no default — depends on the customer's pose model). Other image categories use category defaults if unset." + }, + + "number_of_keypoints": { + "type": "integer", + "minimum": 1, + "description": "Number of keypoints per sample. Required for keypoint_detection — dataset-specific (e.g. 17 for COCO pose, 9 for the upper-body sample in templates/keypoint_detection)." + }, + + "data_id": { + "type": "object", + "additionalProperties": false, + "required": ["strategy"], + "properties": { + "strategy": { + "type": "string", + "enum": ["uuid", "column"], + "default": "uuid", + "description": "How `data_id` is generated. `uuid` (default) generates a fresh UUID per record — no source column leaves the cluster. `column` maps a source column's value, which is louder (logged warning) and only safe when the column doesn't carry PII." + }, + "column": { + "type": "string", + "minLength": 1, + "description": "Required when strategy=column. The source column whose values become `data_id`." + } + }, + "if": { "properties": { "strategy": { "const": "column" } }, "required": ["strategy"] }, + "then": { "required": ["column"] } + }, + + "spec": { + "type": "object", + "additionalProperties": false, + "description": "Advanced overrides. Most customers don't touch this; convention defaults from `category` cover the dominant case.", + "properties": { + "csv_options": { + "type": "object", + "additionalProperties": false, + "properties": { + "chunk_size": { "type": "integer", "minimum": 1 }, + "delimiter": { "type": "string", "minLength": 1 }, + "quotechar": { "type": "string", "minLength": 1 }, + "escapechar": { "type": "string", "minLength": 1 } + }, + "description": "Pandas read_csv passthrough. Defaults: chunk_size=1000, delimiter=',', quotechar='\"', escapechar='\\\\'." + }, + "file_options": { + "type": "object", + "additionalProperties": true, + "properties": { + "target_size": { + "type": "array", + "items": { "type": "integer", "minimum": 1 }, + "minItems": 2, + "maxItems": 2, + "description": "[height, width]. Image categories only. Default [512, 512]." + }, + "extension": { + "type": "string", + "enum": [".jpg", ".jpeg", ".png", ".txt", ".text", ".xml"], + "description": "Allowed extension for sidecar files. Defaults: .jpg for image categories, .txt for text_classification." + } + } + }, + "validators": { + "type": "array", + "items": { "type": "string" }, + "description": "Override the default validator set resolved by `map_validators(category)`. Rarely needed." + }, + "sidecars": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["column", "source"], + "properties": { + "column": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "dest": { "type": "string", "minLength": 1 }, + "transform": { + "type": "object", + "additionalProperties": false, + "properties": { + "resize": { + "type": "array", + "items": { "type": "integer", "minimum": 1 }, + "minItems": 2, + "maxItems": 2 + } + } + }, + "validators": { + "type": "array", + "items": { "type": "string" } + } + } + }, + "description": "Advanced: explicit sidecar-file declaration when the convention defaults don't fit. Most users rely on `images`/`annotations`/`masks`/`texts` shorthand instead." + }, + "processors": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["script", "class"], + "properties": { + "script": { + "type": "string", + "minLength": 1, + "description": "Path inside the pod where the script body has been mounted (typically a ConfigMap mount under /custom/...)." + }, + "class": { + "type": "string", + "minLength": 1, + "description": "Class name within the script. Must subclass tracebloc_ingestor.processors.BaseProcessor." + }, + "args": { + "type": "object", + "description": "Keyword arguments passed to the processor's constructor." + } + } + }, + "description": "Custom-processor escape hatch. The Helm subchart (client#86) takes the script body as a chart value, writes a ConfigMap, mounts it. The official image dynamically imports the class and applies it to every record. No customer-built Docker image required." + } + } + } + }, + + "allOf": [ + { + "description": "Exactly one data source: csv or json.", + "oneOf": [ + { "required": ["csv"], "not": { "required": ["json"] } }, + { "required": ["json"], "not": { "required": ["csv"] } } + ] + }, + { + "description": "Image-based categories require `images`.", + "if": { + "properties": { + "category": { + "enum": [ + "image_classification", + "object_detection", + "keypoint_detection", + "semantic_segmentation", + "instance_segmentation" + ] + } + }, + "required": ["category"] + }, + "then": { "required": ["images"] } + }, + { + "description": "object_detection requires `annotations`.", + "if": { + "properties": { "category": { "const": "object_detection" } }, + "required": ["category"] + }, + "then": { "required": ["annotations"] } + }, + { + "description": "semantic_segmentation requires `masks`.", + "if": { + "properties": { "category": { "const": "semantic_segmentation" } }, + "required": ["category"] + }, + "then": { "required": ["masks"] } + }, + { + "description": "text_classification requires `texts`.", + "if": { + "properties": { "category": { "const": "text_classification" } }, + "required": ["category"] + }, + "then": { "required": ["texts"] } + }, + { + "description": "masked_language_modeling requires `sequences`.", + "if": { + "properties": { "category": { "const": "masked_language_modeling" } }, + "required": ["category"] + }, + "then": { "required": ["sequences"] } + }, + { + "description": "tabular and time-series categories require `schema`.", + "if": { + "properties": { + "category": { + "enum": [ + "tabular_classification", + "tabular_regression", + "time_series_forecasting", + "time_to_event_prediction" + ] + } + }, + "required": ["category"] + }, + "then": { "required": ["schema"] } + }, + { + "description": "Regression-class tasks require an explicit label.policy decision (must be the object form).", + "if": { + "properties": { + "category": { + "enum": [ + "tabular_regression", + "time_series_forecasting", + "time_to_event_prediction" + ] + } + }, + "required": ["category"] + }, + "then": { + "properties": { + "label": { + "type": "object", + "required": ["column", "policy"] + } + }, + "required": ["label"] + } + }, + { + "description": "keypoint_detection requires customer-supplied `target_size` and `number_of_keypoints` (no convention defaults — both are dataset-specific).", + "if": { + "properties": { "category": { "const": "keypoint_detection" } }, + "required": ["category"] + }, + "then": { "required": ["target_size", "number_of_keypoints"] } + }, + { + "description": "Most categories require `label`.", + "if": { + "properties": { + "category": { + "enum": [ + "image_classification", + "object_detection", + "keypoint_detection", + "semantic_segmentation", + "instance_segmentation", + "text_classification", + "tabular_classification" + ] + } + }, + "required": ["category"] + }, + "then": { "required": ["label"] } + } + ] +} diff --git a/internal/schema/validate.go b/internal/schema/validate.go new file mode 100644 index 0000000..b71266e --- /dev/null +++ b/internal/schema/validate.go @@ -0,0 +1,238 @@ +package schema + +import ( + "bytes" + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/santhosh-tekuri/jsonschema/v6" + "golang.org/x/text/language" + "golang.org/x/text/message" + "gopkg.in/yaml.v3" +) + +// englishPrinter is the per-package message.Printer we pass to +// jsonschema's ErrorKind.LocalizedString. The library panics on a +// nil printer (it tries to construct one with a nil language tag); +// having a process-wide English printer avoids that and keeps the +// output language stable across customer environments. Internationalizing +// the validator output is a v2 concern. +var englishPrinter = message.NewPrinter(language.English) + +// V1SchemaID is the $id of the embedded v1 schema. Used as the +// "source URL" when compiling — jsonschema/v6 needs a name to +// associate the loaded bytes with, and the canonical $id keeps +// error messages anchored to the same identifier the Python +// implementation uses. +const V1SchemaID = "https://tracebloc.io/schemas/ingest.v1.json" + +// ValidationError is a single schema violation, normalized to the +// ">: " shape that +// tracebloc_ingestor.cli.run._format_errors emits in Python. Pinning +// the format lets us tell customers "the CLI and the server-side +// validator say the same thing about your YAML" with high +// confidence. +type ValidationError struct { + // Path is the json-pointer-style location, e.g. "spec.processors.0.script". + // "" for top-level violations (additionalProperties, + // missing required fields at the document level, etc.). + Path string + + // Message is the human-readable description of the violation, + // taken directly from the underlying jsonschema/v6 library's + // per-error message. The wording is stable across the library's + // minor versions; we don't post-process it. + Message string +} + +// Format returns the canonical " : " line, matching +// the indentation the Python implementation produces. The leading +// two spaces are deliberate — they match _format_errors so customer +// docs / runbooks can reference one wording across both +// implementations. +func (e ValidationError) Format() string { + return fmt.Sprintf(" %s: %s", e.Path, e.Message) +} + +// FormatErrors renders a slice of violations as one error per line, +// deterministically ordered by Path then Message. Mirrors +// _format_errors. +// +// Pure: the input slice's order is preserved. An earlier version +// called sort.Slice directly on errs, which silently reordered the +// caller's underlying array — bugbot caught this as a hidden side +// effect that contradicted the function's name + doc. Copying the +// slice header before sorting keeps FormatErrors a true formatter. +func FormatErrors(errs []ValidationError) string { + // slices.Clone would be cleaner but it's Go 1.21+; the manual + // copy is portable and the allocation is tiny relative to the + // schema validation itself. + sorted := make([]ValidationError, len(errs)) + copy(sorted, errs) + + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Path != sorted[j].Path { + return sorted[i].Path < sorted[j].Path + } + return sorted[i].Message < sorted[j].Message + }) + + var b strings.Builder + for i, e := range sorted { + if i > 0 { + b.WriteByte('\n') + } + b.WriteString(e.Format()) + } + return b.String() +} + +// Validator wraps a compiled jsonschema.Schema for repeated use. +// Compile once at process startup; validate many. Schema compilation +// is non-trivial (the v1 schema has nested if/then/oneOf chains); +// reusing the compiled form is meaningfully faster for batch +// validation. +type Validator struct { + schema *jsonschema.Schema +} + +// NewV1Validator compiles the embedded v1 schema. Returns an error +// only if the embedded JSON is malformed — which can only happen if +// scripts/sync-schema.sh wrote garbage, in which case the build's +// embed step or the CI drift check would have caught it. The error +// path stays for defense-in-depth. +func NewV1Validator() (*Validator, error) { + var raw any + if err := json.Unmarshal(V1Bytes, &raw); err != nil { + return nil, fmt.Errorf("embedded schema is not valid JSON: %w", err) + } + + c := jsonschema.NewCompiler() + if err := c.AddResource(V1SchemaID, raw); err != nil { + return nil, fmt.Errorf("registering embedded schema: %w", err) + } + s, err := c.Compile(V1SchemaID) + if err != nil { + return nil, fmt.Errorf("compiling embedded schema: %w", err) + } + return &Validator{schema: s}, nil +} + +// ValidateYAML parses the input as YAML and validates the resulting +// document against the schema. Returns the parsed document (useful +// for callers that want to inspect category/table/etc. after a +// successful validation) plus the list of violations. +// +// Two failure modes are distinct and important to separate: +// +// - The input isn't valid YAML at all (parseErr != nil): callers +// should surface this as a parse-level error, separately from +// schema violations. +// - The input parses but doesn't match the schema (errs non-empty): +// these are the customer-facing "your config has problems" cases. +func (v *Validator) ValidateYAML(input []byte) (parsed map[string]any, errs []ValidationError, parseErr error) { + if len(bytes.TrimSpace(input)) == 0 { + return nil, nil, fmt.Errorf("input is empty") + } + + var doc any + if err := yaml.Unmarshal(input, &doc); err != nil { + return nil, nil, fmt.Errorf("not valid YAML: %w", err) + } + + // The schema expects a mapping at the top. Anything else (a + // sequence, a scalar) gets surfaced as a parse-level error so + // the customer doesn't see a wall of unhelpful schema messages + // blaming the document's type. + asMap, ok := doc.(map[string]any) + if !ok { + return nil, nil, fmt.Errorf( + "document must be a YAML mapping at the top level (apiVersion / kind / category / ...); got %T", + doc, + ) + } + + // Schema validators expect canonical Go-native types + // (map[string]any, []any, string, float64, bool, nil). yaml.v3 + // produces those directly from Unmarshal-into-any, so no + // conversion needed. + if err := v.schema.Validate(doc); err != nil { + // Recurse the error tree into our flat ValidationError list. + // jsonschema/v6 returns a *ValidationError tree where each + // node may have child Causes — we flatten to leaves so the + // customer sees one line per actual problem, not an outline + // of the validator's traversal path. + var ve *jsonschema.ValidationError + if errors_as(err, &ve) { + errs = flattenValidationError(ve) + } else { + // Defensive: shouldn't happen with current jsonschema/v6, + // but fall back to the raw error string rather than + // crashing. + errs = []ValidationError{{Path: "", Message: err.Error()}} + } + } + + return asMap, errs, nil +} + +// flattenValidationError walks the jsonschema error tree to produce +// one leaf error per real violation. The library returns a tree +// because oneOf / anyOf / allOf can fail in multiple ways at once; +// we want each leaf surfaced individually so the customer can see +// every problem with a single validate run. +func flattenValidationError(ve *jsonschema.ValidationError) []ValidationError { + if ve == nil { + return nil + } + + // Internal nodes (with causes) carry the structural context; + // the actual violations live at the leaves. + if len(ve.Causes) > 0 { + var out []ValidationError + for _, c := range ve.Causes { + out = append(out, flattenValidationError(c)...) + } + return out + } + + return []ValidationError{{ + Path: instanceLocationToPath(ve.InstanceLocation), + Message: ve.ErrorKind.LocalizedString(englishPrinter), + }} +} + +// instanceLocationToPath converts jsonschema's slice-of-strings +// instance pointer into the dotted form data-ingestors uses +// (matching _format_errors's `".".join(...)`). An empty location +// becomes "" so the customer sees something concrete instead +// of a blank-looking error line. +func instanceLocationToPath(loc []string) string { + if len(loc) == 0 { + return "" + } + return strings.Join(loc, ".") +} + +// errors_as is a tiny inlining of errors.As to avoid the import +// just for this one site. Keeps the dependency graph of this file +// minimal; the only third-party imports stay jsonschema + yaml. +func errors_as(err error, target **jsonschema.ValidationError) bool { + for cur := err; cur != nil; cur = unwrap(cur) { + if ve, ok := cur.(*jsonschema.ValidationError); ok { + *target = ve + return true + } + } + return false +} + +func unwrap(err error) error { + type unwrapper interface{ Unwrap() error } + if u, ok := err.(unwrapper); ok { + return u.Unwrap() + } + return nil +} diff --git a/internal/schema/validate_test.go b/internal/schema/validate_test.go new file mode 100644 index 0000000..c09ae59 --- /dev/null +++ b/internal/schema/validate_test.go @@ -0,0 +1,405 @@ +package schema + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// Build the validator once; reuse across cases. Compilation cost +// shouldn't be in the critical path of every test. +func mustValidator(t *testing.T) *Validator { + t.Helper() + v, err := NewV1Validator() + if err != nil { + t.Fatalf("compile embedded schema: %v", err) + } + return v +} + +func TestEmbeddedSchemaCompiles(t *testing.T) { + // Compiling is the most basic invariant — if the bundled JSON + // is malformed, every other test fails too, but this gives the + // clearest first signal. + _ = mustValidator(t) +} + +func TestEmbeddedSchemaIsV1ByID(t *testing.T) { + // Spot-check that the bundled bytes are actually the v1 schema + // (not, say, a stale draft or a v2 that got pulled by accident). + // We grep on the canonical $id, not on schema structure, so a + // future field rename or example update doesn't break this test + // for spurious reasons. + if !strings.Contains(string(V1Bytes), V1SchemaID) { + t.Fatalf("embedded schema doesn't mention its expected $id %q", V1SchemaID) + } +} + +// Each of the canonical examples in tracebloc/data-ingestors must +// validate cleanly — if any fail, either the example is broken or +// the schema's drifted from what the examples assume. Pin them by +// inlining (the alternative is loading from an out-of-tree path +// which makes the test brittle to data-ingestors layout changes). +// +// The fixtures intentionally mirror examples/yaml/.yaml +// from data-ingestors. When that set grows, this slice grows too. +func TestValidate_HappyPath_AllCategories(t *testing.T) { + cases := []struct { + name string + yaml string + }{ + { + name: "image_classification", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: chest_xrays_train +intent: train +category: image_classification +csv: /data/labels.csv +images: /data/images/ +label: image_label +`, + }, + { + name: "object_detection", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: visdrone_train +intent: train +category: object_detection +csv: /data/labels.csv +images: /data/images/ +annotations: /data/annotations/ +label: image_label +`, + }, + { + name: "semantic_segmentation", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: tumors_train +intent: train +category: semantic_segmentation +csv: /data/labels.csv +images: /data/images/ +masks: /data/masks/ +label: image_label +`, + }, + { + name: "text_classification", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: support_tickets_train +intent: train +category: text_classification +csv: /data/labels.csv +texts: /data/texts/ +schema: + text_id: VARCHAR(255) + label: VARCHAR(64) +label: label +`, + }, + { + name: "tabular_classification", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: churn_train +intent: train +category: tabular_classification +csv: /data/customers.csv +schema: + age: INT + tenure_months: INT + churned: VARCHAR(8) +label: churned +`, + }, + { + name: "tabular_regression_requires_bucket_policy", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: house_prices_train +intent: train +category: tabular_regression +csv: /data/houses.csv +schema: + square_feet: FLOAT + price: FLOAT +label: + column: price + policy: bucket +`, + }, + } + + v := mustValidator(t) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, errs, parseErr := v.ValidateYAML([]byte(c.yaml)) + if parseErr != nil { + t.Fatalf("YAML parse failed: %v", parseErr) + } + if len(errs) > 0 { + t.Errorf("expected clean validation, got %d issue(s):\n%s", + len(errs), FormatErrors(errs)) + } + }) + } +} + +// Negative cases — each exercises a different schema rule. The +// assertion is on the path the error reports (not the exact message +// wording, which is jsonschema/v6's responsibility to define) so +// library version bumps don't break the test for cosmetic reasons. +func TestValidate_NegativeCases(t *testing.T) { + cases := []struct { + name string + yaml string + wantPaths []string // substring match on Path field of any violation + }{ + { + name: "missing apiVersion + intent", + yaml: ` +kind: IngestConfig +category: image_classification +table: t +csv: /data/labels.csv +images: /data/images/ +label: my_label +`, + wantPaths: []string{""}, // both go to root level + }, + { + name: "category not in enum", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: not_a_category +csv: /data/labels.csv +images: /data/images/ +label: my_label +`, + wantPaths: []string{"category"}, + }, + { + name: "image category missing images key", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: image_classification +csv: /data/labels.csv +label: my_label +`, + wantPaths: []string{""}, // missing required at root + }, + { + name: "regression without explicit label.policy", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: tabular_regression +csv: /data/houses.csv +schema: + price: FLOAT +label: price +`, + wantPaths: []string{"label"}, // schema enforces object form for regression + }, + { + name: "tabular missing schema block", + yaml: ` +apiVersion: tracebloc.io/v1 +kind: IngestConfig +table: t +intent: train +category: tabular_classification +csv: /data/customers.csv +label: churned +`, + wantPaths: []string{""}, + }, + } + + v := mustValidator(t) + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + _, errs, parseErr := v.ValidateYAML([]byte(c.yaml)) + if parseErr != nil { + t.Fatalf("YAML parse failed: %v", parseErr) + } + if len(errs) == 0 { + t.Fatalf("expected at least one schema violation, got none") + } + + gotPaths := make([]string, 0, len(errs)) + for _, e := range errs { + gotPaths = append(gotPaths, e.Path) + } + + for _, want := range c.wantPaths { + found := false + for _, got := range gotPaths { + if strings.Contains(got, want) { + found = true + break + } + } + if !found { + t.Errorf("expected a violation with path containing %q, got paths: %v\nfull errors:\n%s", + want, gotPaths, FormatErrors(errs)) + } + } + }) + } +} + +// Parse-level failures (vs schema violations) need a separate +// failure mode so callers can render them differently in the UI — +// "your file isn't YAML" vs "your file is YAML but doesn't match +// the schema" are different problems with different remediations. +func TestValidate_ParseFailures(t *testing.T) { + v := mustValidator(t) + + t.Run("empty input", func(t *testing.T) { + _, _, parseErr := v.ValidateYAML([]byte("")) + if parseErr == nil { + t.Fatal("expected parseErr on empty input, got nil") + } + if !strings.Contains(parseErr.Error(), "empty") { + t.Errorf("expected message to mention empty, got: %v", parseErr) + } + }) + + t.Run("whitespace only", func(t *testing.T) { + _, _, parseErr := v.ValidateYAML([]byte(" \n\t \n")) + if parseErr == nil { + t.Fatal("expected parseErr on whitespace-only input") + } + }) + + t.Run("non-mapping top level (sequence)", func(t *testing.T) { + _, _, parseErr := v.ValidateYAML([]byte("- one\n- two\n")) + if parseErr == nil { + t.Fatal("expected parseErr on top-level sequence") + } + if !strings.Contains(parseErr.Error(), "mapping") { + t.Errorf("expected message to mention 'mapping', got: %v", parseErr) + } + }) + + t.Run("malformed YAML", func(t *testing.T) { + // Trailing colon + missing value, mixed indent — yaml.v3 + // flags this. + _, _, parseErr := v.ValidateYAML([]byte("foo:\n bar:\n\tbaz: 1\n")) + if parseErr == nil { + t.Fatal("expected parseErr on malformed YAML") + } + }) +} + +// FormatErrors is the contract with the UI layer; pin its exact +// output shape so any change is intentional. The leading two +// spaces mirror tracebloc_ingestor.cli.run._format_errors so +// customers see one wording across both implementations. +func TestFormatErrors_ContractShape(t *testing.T) { + errs := []ValidationError{ + {Path: "category", Message: "value must be one of …"}, + {Path: "", Message: "missing properties 'foo'"}, + } + got := FormatErrors(errs) + + // Sorted output: < category alphabetically. + want := " : missing properties 'foo'\n category: value must be one of …" + if got != want { + t.Errorf("FormatErrors mismatch\ngot:\n%s\nwant:\n%s", got, want) + } +} + +// Pins that FormatErrors does NOT mutate its input. An earlier +// version sorted in place, which made callers that wanted to +// further process `violations` after formatting see a different +// order from the one they passed in. Bugbot flagged this as a +// hidden side effect; this test ensures the regression can't +// silently come back. +func TestFormatErrors_DoesNotMutateInput(t *testing.T) { + in := []ValidationError{ + {Path: "z_field", Message: "bad"}, + {Path: "a_field", Message: "bad"}, + } + original := append([]ValidationError(nil), in...) // capture pre-call order + + _ = FormatErrors(in) + + if in[0] != original[0] || in[1] != original[1] { + t.Errorf("FormatErrors mutated input order.\nbefore: %v\nafter: %v", + original, in) + } +} + +// Round-trip test against the canonical example YAMLs in the +// data-ingestors repo. If that checkout is missing, skip — we +// don't want CI to fail for a local layout mismatch. CI runs the +// test in the runner's filesystem where data-ingestors isn't +// checked out alongside us, so this only fires for local devs. +// +// Opt in by setting TRACEBLOC_INGESTORS_EXAMPLES to the absolute +// path of data-ingestors/examples/yaml (or any directory of +// `.yaml` files that follow the v1 schema). Without the env var +// the test skips silently — a hardcoded absolute path like +// `/Volumes/VPPD/...` was bugbot-flagged as leaking one +// developer's filesystem layout and being unusable by anyone else. +func TestValidate_AgainstRealExamplesIfPresent(t *testing.T) { + examplesDir := os.Getenv("TRACEBLOC_INGESTORS_EXAMPLES") + if examplesDir == "" { + t.Skip("TRACEBLOC_INGESTORS_EXAMPLES not set; skipping the round-trip-against-real-examples test " + + "(point it at data-ingestors/examples/yaml to enable)") + } + entries, err := os.ReadDir(examplesDir) + if err != nil { + t.Skipf("TRACEBLOC_INGESTORS_EXAMPLES=%q is not readable, skipping: %v", examplesDir, err) + } + + v := mustValidator(t) + for _, e := range entries { + if e.IsDir() || filepath.Ext(e.Name()) != ".yaml" { + continue + } + t.Run(e.Name(), func(t *testing.T) { + body, err := os.ReadFile(filepath.Join(examplesDir, e.Name())) + if err != nil { + t.Fatalf("read: %v", err) + } + _, errs, parseErr := v.ValidateYAML(body) + if parseErr != nil { + t.Fatalf("parse: %v", parseErr) + } + if len(errs) > 0 { + // custom_processor.yaml may legitimately not match + // (the example shows a feature deferred to v1.1); + // skip rather than fail so the test stays useful as + // the canonical examples evolve. + if strings.Contains(e.Name(), "custom_processor") { + t.Skipf("custom_processor example exercises deferred features; %d issue(s)", + len(errs)) + } + t.Errorf("expected clean, got %d issue(s):\n%s", + len(errs), FormatErrors(errs)) + } + }) + } +} diff --git a/internal/submit/body.go b/internal/submit/body.go new file mode 100644 index 0000000..80b641e --- /dev/null +++ b/internal/submit/body.go @@ -0,0 +1,107 @@ +// Package submit owns the `tracebloc dataset push` Phase 4 step: +// POST the synthesized ingest spec to jobs-manager's +// /internal/submit-ingestion-run endpoint, then watch the ingestor +// Job the cluster spawns in response. +// +// Phase 4 sits between Phase 3's stage Pod (which lays files on the +// PVC) and Phase 5's release distribution. The protocol is the same +// one tracebloc/client's ingestor subchart's post-install hook uses +// today — see ingestor/templates/configmap-ingest-config.yaml. +// Keeping the protocol identical means the CLI and the helm flow +// are interchangeable at the cluster's API surface, and the chart +// stays a fully-supported alternative for ops folks who prefer it. +// +// The package is split into: +// - body.go (this file): synthesize the POST body +// - client.go: HTTP client + bearer token + 4xx framing +// - watch.go: poll Job + stream Pod logs +// - summary.go: parse the 📊 INGESTION SUMMARY banner +// - submit.go: top-level orchestrator +package submit + +import ( + "crypto/rand" + "encoding/hex" + "fmt" +) + +// SubmitRequest is the wire shape POSTed to jobs-manager's +// /internal/submit-ingestion-run. Field names mirror the chart's +// ingestor/templates/configmap-ingest-config.yaml body.json key +// for-key so the chart and the CLI are interchangeable on the +// server side. +// +// json struct tags are explicit-snake so a future re-import via +// encoding/json doesn't silently produce mixedCase keys. +type SubmitRequest struct { + // IngestConfig is the customer's ingest spec as a verbatim + // YAML string. jobs-manager re-parses + revalidates this + // server-side. We don't re-marshal — the CLI's Phase 3 spec + // synthesis already produced canonical YAML. + IngestConfig string `json:"ingest_config"` + + // IdempotencyKey is the per-invocation replay token. + // jobs-manager records this in its idempotency-key table; a + // second POST with the same key returns the SAME job_name as + // the first (with replay=true) instead of spawning a new Job. + // + // Default is a fresh UUID-ish 16-byte hex string per + // invocation; `--idempotency-key ` overrides for the + // at-most-once-across-attempts case where the customer + // genuinely wants retry-safety across multiple CLI runs. + IdempotencyKey string `json:"idempotency_key"` + + // ImageDigest optionally pins the ingestor container image. + // Empty = let jobs-manager use the cluster's configured + // default (set by the parent client chart's + // `images.ingestor.digest`, kept current by the auto-upgrade + // cronjob). Setting it locks the run to a specific image, + // matching the chart's --set image.digest=... override path. + // + // `omitempty` on the JSON tag means jobs-manager sees no + // image_digest key at all when this is empty, which is the + // well-tested default-image code path on the server side. + ImageDigest string `json:"image_digest,omitempty"` +} + +// BuildRequest is the constructor used by the orchestrator. Both +// IngestConfig (the YAML the CLI already synthesized in Phase 3) +// and the optional ImageDigest flow through unchanged; the +// idempotency key is the only non-trivial bit. +// +// If override is empty, a fresh 16-byte hex string is generated +// from crypto/rand. UUID-shaped without the dashes — the chart's +// own helper does the same (ingestor.idempotencyKey in +// _helpers.tpl), so server-side hash-table lookups are uniform +// across both flows. +func BuildRequest(ingestYAML string, idempotencyKeyOverride, imageDigest string) (*SubmitRequest, error) { + key := idempotencyKeyOverride + if key == "" { + raw := make([]byte, 16) + if _, err := rand.Read(raw); err != nil { + return nil, fmt.Errorf("generating idempotency key: %w", err) + } + key = hex.EncodeToString(raw) + } + return &SubmitRequest{ + IngestConfig: ingestYAML, + IdempotencyKey: key, + ImageDigest: imageDigest, + }, nil +} + +// SubmitResponse is jobs-manager's 201 reply. job_name is the +// ingestor Job watch.go will poll; namespace is the resolved API +// namespace (usually the same one the CLI POSTed to, but +// jobs-manager can in principle redirect cross-namespace). +// +// Replay distinguishes "we just spawned this Job" (replay=false) +// from "we already have a Job for this idempotency key, here it +// is" (replay=true). The CLI prints a different lifecycle banner +// for each — replay means "another invocation already kicked +// this off; we're attaching to it." +type SubmitResponse struct { + JobName string `json:"job_name"` + Namespace string `json:"namespace"` + Replay bool `json:"replay"` +} diff --git a/internal/submit/body_test.go b/internal/submit/body_test.go new file mode 100644 index 0000000..dd6e459 --- /dev/null +++ b/internal/submit/body_test.go @@ -0,0 +1,100 @@ +package submit + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestBuildRequest_GeneratesIdempotencyKey: when no override is +// provided, BuildRequest produces a fresh hex-encoded 16-byte key. +// Pins both "non-empty" + "looks-like-hex" so a future change to +// random source can't silently produce a malformed key. +func TestBuildRequest_GeneratesIdempotencyKey(t *testing.T) { + req, err := BuildRequest("yaml content", "", "") + if err != nil { + t.Fatalf("BuildRequest: %v", err) + } + if req.IdempotencyKey == "" { + t.Fatal("generated idempotency key is empty") + } + if len(req.IdempotencyKey) != 32 { + t.Errorf("generated key length = %d, want 32 (16 bytes hex)", len(req.IdempotencyKey)) + } + for i, c := range req.IdempotencyKey { + if !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'f')) { + t.Errorf("key[%d] = %c, want lowercase hex", i, c) + break + } + } +} + +// TestBuildRequest_IdempotencyKeyOverride: --idempotency-key flag +// path. The override is plumbed verbatim — no hashing, no munging — +// so a customer using the same key across retries gets the chart's +// replay semantics. +func TestBuildRequest_IdempotencyKeyOverride(t *testing.T) { + req, err := BuildRequest("yaml", "my-fixed-key-abc123", "") + if err != nil { + t.Fatalf("BuildRequest: %v", err) + } + if req.IdempotencyKey != "my-fixed-key-abc123" { + t.Errorf("IdempotencyKey = %q, want override value", req.IdempotencyKey) + } +} + +// TestBuildRequest_KeysAreUniquePerCall: two back-to-back +// BuildRequest calls with no override produce distinct keys. +// Critical for jobs-manager's idempotency table — same key means +// "this is a retry, replay the previous run." +func TestBuildRequest_KeysAreUniquePerCall(t *testing.T) { + a, err := BuildRequest("yaml", "", "") + if err != nil { + t.Fatalf("BuildRequest a: %v", err) + } + b, err := BuildRequest("yaml", "", "") + if err != nil { + t.Fatalf("BuildRequest b: %v", err) + } + if a.IdempotencyKey == b.IdempotencyKey { + t.Errorf("back-to-back BuildRequest produced identical keys %q", a.IdempotencyKey) + } +} + +// TestBuildRequest_JSONShape: the wire format jobs-manager expects. +// Field names + omitempty behavior are the contract; if any of +// these drift, the server-side handler fails to parse. +func TestBuildRequest_JSONShape(t *testing.T) { + t.Run("with image digest", func(t *testing.T) { + req, _ := BuildRequest("yaml-content", "key123", "sha256:abc") + b, err := json.Marshal(req) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + s := string(b) + for _, want := range []string{ + `"ingest_config":"yaml-content"`, + `"idempotency_key":"key123"`, + `"image_digest":"sha256:abc"`, + } { + if !strings.Contains(s, want) { + t.Errorf("JSON missing %q in: %s", want, s) + } + } + }) + t.Run("omits empty image digest", func(t *testing.T) { + req, _ := BuildRequest("yaml", "key", "") + b, err := json.Marshal(req) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + // jobs-manager's no-image-digest code path is the + // well-tested default; passing an empty string would + // route through the override path on the server. The + // omitempty tag is the contract that keeps the default + // path engaged. + if strings.Contains(string(b), "image_digest") { + t.Errorf("JSON includes image_digest when empty: %s", b) + } + }) +} diff --git a/internal/submit/client.go b/internal/submit/client.go new file mode 100644 index 0000000..972d91b --- /dev/null +++ b/internal/submit/client.go @@ -0,0 +1,194 @@ +package submit + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// SubmitTimeout caps how long we wait for jobs-manager to respond to +// the POST. jobs-manager validates synchronously (schema re-check, +// idempotency lookup, Job creation) — the chart's hook bounds this +// at 30s and we mirror that. Beyond 30s, something genuinely wrong +// is happening server-side, and the customer wants the diagnostic +// rather than a longer wait. +const SubmitTimeout = 30 * time.Second + +// Submitter is the narrow surface the orchestrator (submit.go) uses +// for the POST. Real impl is *HTTPSubmitter; tests use a fake that +// records the request and returns a synthetic response. +type Submitter interface { + Submit(ctx context.Context, req *SubmitRequest) (*SubmitResponse, error) +} + +// HTTPSubmitter is the production implementation. Wraps net/http +// with a fixed jobs-manager URL + a SA token from Phase 2's mint. +// +// Endpoint comes from Phase 2's cluster.DiscoverParentRelease +// (`http://-jobs-manager..svc.cluster.local:8080`), +// SubmitPath is hardcoded per the chart's published contract. +type HTTPSubmitter struct { + // Endpoint is the full jobs-manager URL, e.g. + // "http://release-jobs-manager.tracebloc.svc.cluster.local:8080". + // No trailing slash — the submitter appends SubmitPath. + Endpoint string + + // Token is the bearer token to send in the Authorization + // header. Comes from Phase 2's cluster.MintIngestorToken. + Token string + + // Client is the underlying *http.Client. Set in NewHTTPSubmitter + // with a sensible timeout. Exposed for tests that need to point + // at an httptest.Server with a custom RoundTripper. + Client *http.Client +} + +// SubmitPath is the well-known URL path on jobs-manager. Pinned +// here as a constant rather than a knob because the chart's +// post-install hook also pins it; if jobs-manager ever moves the +// endpoint, both have to bump together (which is a coordinated +// release across tracebloc/client + tracebloc/cli, exactly what +// you want for a protocol-level change). +const SubmitPath = "/internal/submit-ingestion-run" + +// NewHTTPSubmitter returns a Submitter wired with a sensible +// timeout + a transport that DOESN'T do TLS verification against +// the in-cluster CA. The jobs-manager endpoint is HTTP-only inside +// the cluster (kube-proxy handles all the rest), so TLS isn't in +// the picture at all today. If a future jobs-manager exposes +// HTTPS, the InsecureSkipVerify path is the right v0.1 default +// because the customer's kubeconfig has already authenticated them +// to the cluster — the cluster-internal jobs-manager doesn't have +// a CA the laptop would recognize anyway. +func NewHTTPSubmitter(endpoint, token string) *HTTPSubmitter { + return &HTTPSubmitter{ + Endpoint: strings.TrimRight(endpoint, "/"), + Token: token, + Client: &http.Client{ + Timeout: SubmitTimeout, + Transport: &http.Transport{ + // See doc on NewHTTPSubmitter for the + // InsecureSkipVerify rationale. + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec + }, + }, + } +} + +// Submit POSTs the request body to jobs-manager and decodes the +// 201 response into a SubmitResponse. On non-201 status codes, the +// remote body is surfaced verbatim so the customer sees whatever +// jobs-manager said (typically a JSON {error, detail} from the +// fastapi handler) rather than just "HTTP 422". +// +// Replays (idempotency-key already seen → 200 with replay=true) +// are reported in the response struct; this method treats them as +// success because the upstream behavior IS "your run is already +// in progress / already done." The orchestrator handles the +// replay branch in its own diagnostic output. +func (s *HTTPSubmitter) Submit(ctx context.Context, req *SubmitRequest) (*SubmitResponse, error) { + body, err := json.Marshal(req) + if err != nil { + // Marshaling a struct of strings shouldn't fail at all; + // surfacing the error means a future struct change broke + // the wire format. + return nil, fmt.Errorf("marshaling submit request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, + s.Endpoint+SubmitPath, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("building submit request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+s.Token) + + httpResp, err := s.Client.Do(httpReq) + if err != nil { + // Network errors (DNS, connection refused, TLS handshake + // failure, ctx cancellation). The error wraps net.OpError + // already; just frame it with our endpoint so the + // customer knows what was being attempted. + return nil, fmt.Errorf("POST %s%s: %w", s.Endpoint, SubmitPath, err) + } + defer func() { _ = httpResp.Body.Close() }() + + respBody, err := io.ReadAll(httpResp.Body) + if err != nil { + // Short read on a 201 body — extremely rare; the + // connection dropped between header and body. + return nil, fmt.Errorf("reading submit response body: %w", err) + } + + // 2xx (typically 201 Created; also 200 for replays) is success. + // Everything else surfaces the remote body verbatim so the + // customer sees jobs-manager's actual diagnostic (HTTP 4xx + // schema-rejection, HTTP 5xx kube-apiserver-failure, etc.) + // rather than just a status code. + if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 { + return nil, &SubmitError{ + StatusCode: httpResp.StatusCode, + Body: string(respBody), + Endpoint: s.Endpoint + SubmitPath, + } + } + + var parsed SubmitResponse + if err := json.Unmarshal(respBody, &parsed); err != nil { + // 2xx but the body isn't the expected shape. Almost + // certainly a protocol mismatch (jobs-manager version + // drift). Include the raw body so the customer can pin + // the version they're on. + return nil, fmt.Errorf("decoding submit response (got body %q): %w", string(respBody), err) + } + if parsed.JobName == "" { + // Defensive: a malformed-but-parsing response with no + // job_name would silently break Phase 4's watch step. + return nil, fmt.Errorf("submit response missing job_name (got body %q)", string(respBody)) + } + if parsed.Namespace == "" { + // Same shape as the job_name check: a missing namespace + // would route subsequent k8s API calls (watch the Pod, + // stream logs) at the empty string — kubelet returns + // confusing errors, and the kubectl-logs reconnect hint + // printed in --detach output would be malformed. Bugbot + // PR #10 r2 flagged the gap. + return nil, fmt.Errorf("submit response missing namespace (got body %q)", string(respBody)) + } + return &parsed, nil +} + +// SubmitError is the typed non-2xx response. Pulled into a struct +// (rather than an opaque string) so the orchestrator can branch on +// StatusCode for the exit-code mapping: 401/403 → auth exit code, +// 4xx other → submit-validation exit code, 5xx → submit-server. +// +// Implements `error` + an Is for errors.Is detection in tests. +type SubmitError struct { + StatusCode int + Body string + Endpoint string +} + +func (e *SubmitError) Error() string { + // Compact framing: the customer sees status + body. The body + // is jobs-manager's actual diagnostic (e.g. fastapi's + // {"detail": "..."}) which is the actionable part. + return fmt.Sprintf("jobs-manager %s returned HTTP %d: %s", + e.Endpoint, e.StatusCode, strings.TrimSpace(e.Body)) +} + +// IsSubmitError reports whether err is a *SubmitError. Convenience +// for the orchestrator's exit-code mapping; errors.As would also +// work but this reads cleaner at the branch site. +func IsSubmitError(err error) bool { + var se *SubmitError + return errors.As(err, &se) +} diff --git a/internal/submit/client_test.go b/internal/submit/client_test.go new file mode 100644 index 0000000..7519c14 --- /dev/null +++ b/internal/submit/client_test.go @@ -0,0 +1,234 @@ +package submit + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestHTTPSubmitter_HappyPath: jobs-manager returns 201 with the +// canonical body shape; client decodes correctly + surfaces all +// three response fields. +func TestHTTPSubmitter_HappyPath(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Pin the wire format jobs-manager actually expects. + if r.Method != http.MethodPost { + t.Errorf("got method %s, want POST", r.Method) + } + if r.URL.Path != SubmitPath { + t.Errorf("got path %s, want %s", r.URL.Path, SubmitPath) + } + if got := r.Header.Get("Authorization"); got != "Bearer fake-token-deadbeef" { + t.Errorf("Authorization = %q, want Bearer fake-token-deadbeef", got) + } + if got := r.Header.Get("Content-Type"); got != "application/json" { + t.Errorf("Content-Type = %q, want application/json", got) + } + body, _ := io.ReadAll(r.Body) + // Light shape check — body_test.go pins the full JSON shape. + if !strings.Contains(string(body), `"ingest_config"`) { + t.Errorf("body missing ingest_config: %s", body) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]any{ + "job_name": "ingestor-abc123", + "namespace": "tracebloc", + "replay": false, + }) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "fake-token-deadbeef") + req, _ := BuildRequest("yaml-content", "key1", "") + + resp, err := s.Submit(context.Background(), req) + if err != nil { + t.Fatalf("Submit: %v", err) + } + if resp.JobName != "ingestor-abc123" { + t.Errorf("JobName = %q, want ingestor-abc123", resp.JobName) + } + if resp.Namespace != "tracebloc" { + t.Errorf("Namespace = %q, want tracebloc", resp.Namespace) + } + if resp.Replay { + t.Errorf("Replay = true, want false") + } +} + +// TestHTTPSubmitter_ReplayResponse: replay=true is also a success +// path. The orchestrator distinguishes the two via the response +// flag, not via HTTP status — both come back 2xx. +func TestHTTPSubmitter_ReplayResponse(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) // jobs-manager returns 200 for replays per source + _ = json.NewEncoder(w).Encode(map[string]any{ + "job_name": "existing-job", + "namespace": "tracebloc", + "replay": true, + }) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "same-key", "") + resp, err := s.Submit(context.Background(), req) + if err != nil { + t.Fatalf("Submit on replay: %v", err) + } + if !resp.Replay { + t.Errorf("Replay = false, want true") + } +} + +// TestHTTPSubmitter_4xxSurfacesBody: 4xx from jobs-manager +// surfaces the verbatim body so the customer sees jobs-manager's +// actual diagnostic (typically {"detail": "..."}), not just +// "HTTP 422". +func TestHTTPSubmitter_4xxSurfacesBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnprocessableEntity) + _, _ = w.Write([]byte(`{"detail":"ingest_config schema rejected: missing required field 'intent'"}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on 4xx") + } + if !IsSubmitError(err) { + t.Errorf("err is not *SubmitError: %T", err) + } + for _, want := range []string{"HTTP 422", "missing required field 'intent'"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %v", want, err) + } + } +} + +// TestHTTPSubmitter_401IsAuthError: distinguishes the auth case +// (401/403) from generic 4xx for the orchestrator's exit-code +// mapping. Used by the CLI to return "your SA token doesn't +// work" vs "your spec was rejected." +func TestHTTPSubmitter_401IsAuthError(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden} { + t.Run(http.StatusText(status), func(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(status) + _, _ = w.Write([]byte(`{"detail":"token expired"}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on auth error") + } + if !IsAuthError(err) { + t.Errorf("IsAuthError(%v) = false, want true (status=%d)", err, status) + } + }) + } +} + +// TestHTTPSubmitter_5xxNotAuthError: 5xx is server-side trouble +// (kube-apiserver flake, jobs-manager bug), NOT an auth issue. +// IsAuthError should return false so the orchestrator routes to +// the right exit-code bucket. +func TestHTTPSubmitter_5xxNotAuthError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"detail":"internal"}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on 500") + } + if IsAuthError(err) { + t.Errorf("IsAuthError(500) = true, want false") + } + if !IsSubmitError(err) { + t.Errorf("err is not *SubmitError: %T", err) + } +} + +// TestHTTPSubmitter_2xxMissingJobName: a malformed 2xx response +// (server bug or version drift) without job_name is a hard error, +// not a silent success — the orchestrator wouldn't know what to +// watch. +func TestHTTPSubmitter_2xxMissingJobName(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"namespace":"tracebloc","replay":false}`)) + })) + defer srv.Close() + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on missing job_name") + } + if !strings.Contains(err.Error(), "missing job_name") { + t.Errorf("error missing 'missing job_name' framing: %v", err) + } +} + +// TestHTTPSubmitter_NetworkError: unreachable server (closed +// httptest.Server) surfaces as a wrapped net error with the +// endpoint in the message so customers know what was attempted. +func TestHTTPSubmitter_NetworkError(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + })) + endpoint := srv.URL + srv.Close() // kill the server + + s := NewHTTPSubmitter(endpoint, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(context.Background(), req) + if err == nil { + t.Fatal("Submit returned nil on unreachable server") + } + if !strings.Contains(err.Error(), endpoint) { + t.Errorf("error missing endpoint URL %q: %v", endpoint, err) + } +} + +// TestHTTPSubmitter_RespectsContext: ctx cancellation aborts the +// in-flight POST. Critical for the SIGINT path (main.go's +// signal.NotifyContext). +func TestHTTPSubmitter_RespectsContext(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Wait longer than the test's ctx allows. + <-r.Context().Done() + w.WriteHeader(http.StatusGatewayTimeout) + })) + defer srv.Close() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() // already cancelled + + s := NewHTTPSubmitter(srv.URL, "tok") + req, _ := BuildRequest("yaml", "k", "") + _, err := s.Submit(ctx, req) + if err == nil { + t.Fatal("Submit returned nil on cancelled ctx") + } + if !errors.Is(err, context.Canceled) { + t.Errorf("error doesn't wrap context.Canceled: %v", err) + } +} diff --git a/internal/submit/portforward.go b/internal/submit/portforward.go new file mode 100644 index 0000000..20c6155 --- /dev/null +++ b/internal/submit/portforward.go @@ -0,0 +1,238 @@ +package submit + +import ( + "context" + "fmt" + "io" + "net/http" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/portforward" + "k8s.io/client-go/transport/spdy" +) + +// ForwardedConnection is a live port-forward to an in-cluster +// Service. LocalPort is the random port the kernel picked on the +// CLI host; the Submitter POSTs to http://localhost:LocalPort. +// +// Close MUST be called when the caller is done — otherwise the +// goroutine running the SPDY tunnel leaks for the lifetime of the +// process. +type ForwardedConnection struct { + LocalPort int + + stopCh chan struct{} + done chan struct{} +} + +// Close tears down the port-forward. Safe to call multiple times. +func (f *ForwardedConnection) Close() { + select { + case <-f.stopCh: + return // already closed + default: + } + close(f.stopCh) + // Wait briefly for the goroutine to drain — without this, a + // fast Close-then-process-exit could race the SPDY teardown + // and leave a half-open connection on the apiserver side. + select { + case <-f.done: + case <-time.After(2 * time.Second): + } +} + +// PortForwardJobsManager opens a port-forward to a Pod backing the +// jobs-manager Service in `namespace`. The customer's CLI runs +// off-cluster (on a laptop, in a CI runner outside the cluster +// network); the discovered jobs-manager URL is a +// *.svc.cluster.local name that's NOT resolvable from there. The +// port-forward routes traffic through the kubeconfig-authenticated +// kube-apiserver connection — same machinery `kubectl port-forward` +// uses internally. +// +// Returns a ForwardedConnection whose LocalPort the caller targets +// for HTTP. Bugbot PR #10 r3 caught the broken-by-design assumption +// in the initial Phase 4 implementation. +// +// Lifecycle: caller MUST defer Close(). The port-forward stays open +// for the entire submit + watch sequence (submit needs a single +// POST, watch only uses the kubeconfig API server connection, so +// strictly speaking we could close right after the POST — but +// keeping it open is simpler and the resource cost is one idle +// goroutine). +func PortForwardJobsManager( + ctx context.Context, + cs kubernetes.Interface, + restConfig *rest.Config, + namespace, serviceName string, + targetPort int, +) (*ForwardedConnection, error) { + // 1. Find a Running Pod backing the Service. client-go's + // port-forward API speaks Pods, not Services — even though + // kubectl port-forward accepts both, it resolves the Service + // to a Pod internally. + pod, err := pickServicePod(ctx, cs, namespace, serviceName) + if err != nil { + return nil, fmt.Errorf("resolving Service %s/%s to a Pod: %w", + namespace, serviceName, err) + } + + // 2. Build the SPDY transport. client-go bundles helpers in + // transport/spdy for exactly this; the round-tripper handles + // the SPDY upgrade negotiation the apiserver expects on the + // /portforward subresource. + transport, upgrader, err := spdy.RoundTripperFor(restConfig) + if err != nil { + return nil, fmt.Errorf("building SPDY transport: %w", err) + } + + // 3. Construct the portforward URL on the Pod. The path is + // /api/v1/namespaces//pods//portforward; we build + // it via the REST client so kubeconfig's authentication + + // TLS config flow through automatically. + req := cs.CoreV1().RESTClient().Post(). + Resource("pods"). + Namespace(namespace). + Name(pod.Name). + SubResource("portforward") + + dialer := spdy.NewDialer(upgrader, + &http.Client{Transport: transport}, + "POST", req.URL()) + + // 4. Create the port-forwarder. "0:" means "pick + // any free local port; map it to in the Pod." + // The kernel allocates the local port at goroutine start; + // we read it back after readyCh fires. + stopCh := make(chan struct{}) + readyCh := make(chan struct{}) + pf, err := portforward.New(dialer, + []string{fmt.Sprintf("0:%d", targetPort)}, + stopCh, readyCh, + io.Discard, // forwarder's stdout — verbose listener logs we don't want + io.Discard, // forwarder's stderr + ) + if err != nil { + return nil, fmt.Errorf("creating port-forwarder: %w", err) + } + + // 5. Launch the forward goroutine. ForwardPorts blocks until + // stopCh closes (or it errors). The select below waits for + // EITHER ready (success) or done (early failure). + // + // errCh is buffered (capacity 1) AND the send is wrapped in + // a non-blocking select with a default — two layers of + // safety against the "goroutine leaks waiting to send" + // pattern. ForwardPorts only ever sends once, so the buffer + // is sufficient; the non-blocking select is paranoid + // defensive against a future refactor that adds a second + // send path. Bugbot PR #10 r5 flagged the (already- + // buffered) channel as a potential leak — the false alarm + // is worth closing out structurally. + done := make(chan struct{}) + errCh := make(chan error, 1) + go func() { + defer close(done) + // ForwardPorts is the long-running call. Its return value + // is meaningful only on early failure — on normal close, + // it returns nil after stopCh fires. + err := pf.ForwardPorts() + if err == nil { + return + } + select { + case errCh <- err: + default: + // Buffer full = receiver already saw an earlier error + // (impossible today — single sender, single error — + // but the default arm makes the goroutine drainable + // regardless of any future change). + } + }() + + select { + case <-readyCh: + // happy path — port allocated, tunnel up + case err := <-errCh: + return nil, fmt.Errorf("port-forward to %s/%s failed during startup: %w", + namespace, pod.Name, err) + case <-ctx.Done(): + close(stopCh) + return nil, ctx.Err() + } + + ports, err := pf.GetPorts() + if err != nil { + close(stopCh) + return nil, fmt.Errorf("reading allocated port: %w", err) + } + if len(ports) == 0 { + close(stopCh) + return nil, fmt.Errorf("port-forward allocated zero ports") + } + + return &ForwardedConnection{ + LocalPort: int(ports[0].Local), + stopCh: stopCh, + done: done, + }, nil +} + +// pickServicePod resolves a Service to a Running Pod backing it. +// Uses the Service's own selector (read from the Service spec) to +// match Pods — same mechanism the cluster's own kube-proxy uses. +// +// Picks the first Running Pod found; in the common case there's +// only one (jobs-manager is single-replica by chart default). For +// multi-replica deployments, picking any Running Pod is fine — +// they're load-balanced equivalents. +func pickServicePod(ctx context.Context, cs kubernetes.Interface, namespace, serviceName string) (*corev1.Pod, error) { + svc, err := cs.CoreV1().Services(namespace).Get(ctx, serviceName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("reading service %s/%s: %w", namespace, serviceName, err) + } + if len(svc.Spec.Selector) == 0 { + // A Service with no selector is one whose Endpoints are + // hand-managed (e.g. ExternalName). The chart's jobs- + // manager is a normal selector-based Service so this + // shouldn't happen in production; the error makes the + // debugging path obvious if it ever does. + return nil, fmt.Errorf( + "service %s/%s has no selector — can't resolve to a Pod for port-forwarding", + namespace, serviceName) + } + + // Build a label selector from the Service's spec.selector map. + // strings.Join keeps the order deterministic for readable + // error output; the order doesn't affect the actual Pods + // returned. + parts := make([]string, 0, len(svc.Spec.Selector)) + for k, v := range svc.Spec.Selector { + parts = append(parts, k+"="+v) + } + selector := strings.Join(parts, ",") + + pods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector, + }) + if err != nil { + return nil, fmt.Errorf("listing Pods for service %s/%s: %w", + namespace, serviceName, err) + } + for i := range pods.Items { + p := &pods.Items[i] + if p.Status.Phase == corev1.PodRunning { + return p, nil + } + } + return nil, fmt.Errorf( + "no Running Pod backing service %s/%s (found %d Pod(s); "+ + "check `kubectl get pods -n %s -l %s`)", + namespace, serviceName, len(pods.Items), namespace, selector) +} diff --git a/internal/submit/portforward_test.go b/internal/submit/portforward_test.go new file mode 100644 index 0000000..933831c --- /dev/null +++ b/internal/submit/portforward_test.go @@ -0,0 +1,142 @@ +package submit + +import ( + "context" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" +) + +// The full port-forward (PortForwardJobsManager) requires a real +// apiserver + SPDY upgrade, so it's out of scope for unit tests — +// covered by the EKS smoke. What IS testable: pickServicePod's +// Service→Pod resolution, which is the only client-go-only logic +// in the file. + +// svc constructs a Service with the given selector. Used to seed +// the fake clientset. +func svc(name string, selector map[string]string) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tracebloc"}, + Spec: corev1.ServiceSpec{Selector: selector}, + } +} + +// podForSvc constructs a Pod with labels matching the selector. +func podForSvc(name string, labels map[string]string, phase corev1.PodPhase) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "tracebloc", + Labels: labels, + }, + Status: corev1.PodStatus{Phase: phase}, + } +} + +// TestPickServicePod_HappyPath: a Service with one matching Running +// Pod resolves to that Pod's name. +func TestPickServicePod_HappyPath(t *testing.T) { + sel := map[string]string{"app": "jobs-manager"} + cs := fake.NewClientset( + svc("jobs-manager", sel), + podForSvc("jobs-manager-abc", sel, corev1.PodRunning), + ) + p, err := pickServicePod(context.Background(), cs, "tracebloc", "jobs-manager") + if err != nil { + t.Fatalf("pickServicePod: %v", err) + } + if p.Name != "jobs-manager-abc" { + t.Errorf("Pod name = %q, want jobs-manager-abc", p.Name) + } +} + +// TestPickServicePod_SkipsNonRunning: Pending / Failed Pods backing +// the same Service are filtered out — the port-forward only works +// against a Running Pod. +func TestPickServicePod_SkipsNonRunning(t *testing.T) { + sel := map[string]string{"app": "jobs-manager"} + cs := fake.NewClientset( + svc("jobs-manager", sel), + podForSvc("crashed", sel, corev1.PodFailed), + podForSvc("pending", sel, corev1.PodPending), + podForSvc("running", sel, corev1.PodRunning), + ) + p, err := pickServicePod(context.Background(), cs, "tracebloc", "jobs-manager") + if err != nil { + t.Fatalf("pickServicePod: %v", err) + } + if p.Name != "running" { + t.Errorf("Pod name = %q, want running", p.Name) + } +} + +// TestPickServicePod_NoMatchingPod: a Service whose Pods are all +// non-Running (or absent) surfaces a clear error pointing at the +// kubectl command to debug. +func TestPickServicePod_NoMatchingPod(t *testing.T) { + sel := map[string]string{"app": "jobs-manager"} + cs := fake.NewClientset( + svc("jobs-manager", sel), + podForSvc("crashed", sel, corev1.PodFailed), + ) + _, err := pickServicePod(context.Background(), cs, "tracebloc", "jobs-manager") + if err == nil { + t.Fatal("pickServicePod returned nil on no-Running-Pod") + } + for _, want := range []string{ + "no Running Pod", + "jobs-manager", + "kubectl get pods", + } { + if !strings.Contains(err.Error(), want) { + t.Errorf("error missing %q: %v", want, err) + } + } +} + +// TestPickServicePod_ServiceMissing: trying to port-forward to a +// non-existent Service surfaces with the Service name in the error. +func TestPickServicePod_ServiceMissing(t *testing.T) { + cs := fake.NewClientset() // empty + _, err := pickServicePod(context.Background(), cs, "tracebloc", "missing-svc") + if err == nil { + t.Fatal("pickServicePod returned nil on missing Service") + } + if !strings.Contains(err.Error(), "missing-svc") { + t.Errorf("error missing service name: %v", err) + } +} + +// TestPickServicePod_NoSelector: ExternalName Services and other +// selector-less shapes can't be port-forwarded by Pod lookup. +// Surface a clear error rather than silently picking nothing. +func TestPickServicePod_NoSelector(t *testing.T) { + cs := fake.NewClientset(&corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Name: "externalname-svc", Namespace: "tracebloc"}, + // no Spec.Selector + }) + _, err := pickServicePod(context.Background(), cs, "tracebloc", "externalname-svc") + if err == nil { + t.Fatal("pickServicePod returned nil on selector-less Service") + } + if !strings.Contains(err.Error(), "no selector") { + t.Errorf("error missing selector-less framing: %v", err) + } +} + +// TestForwardedConnection_CloseIdempotent: Close is safe to call +// multiple times. defer-Close patterns at multiple levels of the +// orchestrator shouldn't risk a double-close panic. +func TestForwardedConnection_CloseIdempotent(t *testing.T) { + stopCh := make(chan struct{}) + done := make(chan struct{}) + close(done) // simulate goroutine already finished + f := &ForwardedConnection{LocalPort: 12345, stopCh: stopCh, done: done} + f.Close() + f.Close() // must not panic + f.Close() +} diff --git a/internal/submit/submit.go b/internal/submit/submit.go new file mode 100644 index 0000000..d4ccab3 --- /dev/null +++ b/internal/submit/submit.go @@ -0,0 +1,206 @@ +package submit + +import ( + "context" + "errors" + "fmt" + "io" + + "k8s.io/client-go/kubernetes" +) + +// Options bundles every dependency Run needs. The CLI builds one +// from the resolved Phase 2/3 state (kubeconfig clientset, SA +// token, jobs-manager endpoint) + the flags from Phase 4 +// (--detach, --idempotency-key, --image-digest). +type Options struct { + // Submitter is how the POST reaches jobs-manager. Production + // uses NewHTTPSubmitter(endpoint, token); tests inject a + // fake that captures the request + returns a canned response. + Submitter Submitter + + // Client is the kubernetes.Interface for the watch loop's + // Job + Pod polls. Same clientset Phase 3 used. + Client kubernetes.Interface + + // IngestConfigYAML is the synthesized YAML body to POST. + // Phase 3 already produced this in canonical form + // (push.SpecArgs.Build → yaml.Marshal) so we don't re-marshal + // here. + IngestConfigYAML string + + // IdempotencyKey overrides the auto-generated random key. + // Empty = let BuildRequest generate a fresh one. Used by + // the --idempotency-key flag for retry-safety across + // invocations. + IdempotencyKey string + + // ImageDigest optionally pins the ingestor image. Empty = + // jobs-manager uses the cluster's configured default. + ImageDigest string + + // Detach exits immediately after the 201 — no watch, no log + // streaming, no summary. Used by CI scenarios where the + // customer just wants the Job name in stdout and the run + // proceeds asynchronously in the cluster. + Detach bool + + // Out is the customer-facing log stream. Submit writes the + // 201 announcement here, then either streams the Pod's logs + // to it (live watch) or prints the Job name (detach). The + // rendered summary panel also goes here. + Out io.Writer +} + +// Result is what Run reports back to the CLI orchestrator. +// Outcome drives the exit-code mapping (see cli/dataset.go's +// Phase 4 wiring); JobName + PodName are echoed back so the CLI +// can build "reconnect with kubectl logs -n " hints. +type Result struct { + // Submit is the 201 response from jobs-manager. Non-nil on + // any path that got past the POST (including --detach + + // failed watches). nil only if the POST itself failed. + Submit *SubmitResponse + + // Watch is the result of the watch loop. nil on --detach + // (we never started watching). nil on early POST failure. + Watch *WatchResult +} + +// Run is the Phase 4 top-level entrypoint. Steps: +// +// 1. BuildRequest from the YAML + flags +// 2. POST via opts.Submitter; surface SubmitError verbatim +// 3. Print the 201 announcement (job_name / namespace / replay flag) +// 4. If --detach, exit +// 5. WatchJob until Pod terminates or ctx cancels +// 6. Render the parsed Summary panel +// +// Returns the Result + an error. Errors come from steps 1-2-5; on +// steps 3 + 4 + 6, success means "we got far enough" and the +// outcome of the actual ingestion is in Result.Watch.Outcome. +func Run(ctx context.Context, opts Options) (*Result, error) { + if opts.Out == nil { + opts.Out = io.Discard + } + + req, err := BuildRequest(opts.IngestConfigYAML, opts.IdempotencyKey, opts.ImageDigest) + if err != nil { + return nil, fmt.Errorf("building submit request: %w", err) + } + + resp, err := opts.Submitter.Submit(ctx, req) + if err != nil { + return nil, err + } + + // 201 announcement. Customer sees this whether --detach is + // set or not, so they have the Job name for kubectl-poke + // follow-up. + if resp.Replay { + _, _ = fmt.Fprintf(opts.Out, + "Replayed: idempotency key matches a previous run; attaching to existing Job %s/%s\n", + resp.Namespace, resp.JobName) + } else { + _, _ = fmt.Fprintf(opts.Out, + "Submitted: jobs-manager spawned ingestor Job %s/%s\n", + resp.Namespace, resp.JobName) + } + + if opts.Detach { + // --detach: print the reconnect hint and bail. The + // cluster continues without us; the customer can come + // back with `kubectl logs -f -n job/`. + _, _ = fmt.Fprintf(opts.Out, + "Detached (no log streaming). Reconnect with: kubectl logs -f -n %s job/%s\n", + resp.Namespace, resp.JobName) + return &Result{Submit: resp}, nil + } + + // Watch loop. ctx propagates SIGINT cancellation (main.go's + // signal.NotifyContext); a Ctrl-C during the watch produces + // Outcome=Detached + the reconnect hint below. + _, _ = fmt.Fprintf(opts.Out, "Streaming logs from Job %s/%s:\n", resp.Namespace, resp.JobName) + wr, err := WatchJob(ctx, opts.Client, resp.Namespace, resp.JobName, opts.Out) + if err != nil { + // Tag as WatchError so the orchestrator picks the + // ingest-flavored exit code (9), not the submit-flavored + // one (8). The cluster has already accepted the run by + // this point — the CLI just failed to follow it. + return &Result{Submit: resp}, &WatchError{Err: fmt.Errorf("watching ingestor Job: %w", err)} + } + + // Detach paths print a per-reason diagnostic + the same + // kubectl-logs reconnect hint. Bugbot PR #10 r7 caught the + // previous "on signal" framing being misleading for the + // timeout-detach cases — customers who hit the 5-min + // PodReadyTimeout or 1-hour JobWatchTimeout aren't pressing + // Ctrl-C, so attributing it to "signal" was wrong. + if wr.Outcome == JobOutcomeDetached { + _, _ = fmt.Fprintln(opts.Out) + switch wr.DetachReason { + case DetachReasonSignal: + _, _ = fmt.Fprintln(opts.Out, "Detached on signal.") + case DetachReasonPodWaitTimeout: + _, _ = fmt.Fprintln(opts.Out, + "Detached: the ingestor Pod didn't reach Ready within the observation window. "+ + "This usually means slow image pull or scheduling backlog — the run is in "+ + "jobs-manager's queue and will execute when the Pod starts.") + case DetachReasonWatchCap: + _, _ = fmt.Fprintln(opts.Out, + "Detached: the watch window (1 hour) elapsed while the ingestion was still running.") + default: + _, _ = fmt.Fprintln(opts.Out, "Detached.") + } + _, _ = fmt.Fprintf(opts.Out, + "Ingestion continues in the cluster. Reconnect with: kubectl logs -f -n %s job/%s\n", + resp.Namespace, resp.JobName) + return &Result{Submit: resp, Watch: wr}, nil + } + + // Render the summary panel if the ingestor produced one. + // Both Succeeded and Failed paths print it — on Failed, the + // banner tells the customer what got partially through. + if wr.Summary != nil { + _, _ = fmt.Fprintln(opts.Out) + _, _ = fmt.Fprint(opts.Out, RenderPanel(wr.Summary)) + } + + return &Result{Submit: resp, Watch: wr}, nil +} + +// IsAuthError reports whether the error is the auth-flavored case +// (401/403 from jobs-manager). The orchestrator's exit-code +// mapping uses this to distinguish "your SA token doesn't work" +// from "your spec was rejected." +func IsAuthError(err error) bool { + var se *SubmitError + if !errors.As(err, &se) { + return false + } + return se.StatusCode == 401 || se.StatusCode == 403 +} + +// WatchError wraps errors that originated in the watch phase +// (waitForJobPod, log streaming, finalJobStatus). The +// orchestrator distinguishes these from submit-phase errors so +// the exit-code mapping is correct: jobs-manager accepted the +// run already, the cluster is doing the work, the CLI just +// failed to follow along. Maps to exit code 9 (ingest-side +// problem), not 8 (submit-side problem). Bugbot flagged the +// previous "everything that wasn't auth → exit 8" version on +// PR #10. +type WatchError struct { + Err error +} + +func (e *WatchError) Error() string { return e.Err.Error() } +func (e *WatchError) Unwrap() error { return e.Err } + +// IsWatchError reports whether err originated in the watch phase +// rather than the submit phase. The orchestrator's exit-code +// branch uses this directly. +func IsWatchError(err error) bool { + var we *WatchError + return errors.As(err, &we) +} diff --git a/internal/submit/submit_test.go b/internal/submit/submit_test.go new file mode 100644 index 0000000..cf34126 --- /dev/null +++ b/internal/submit/submit_test.go @@ -0,0 +1,231 @@ +package submit + +import ( + "bytes" + "context" + "errors" + "fmt" + "strings" + "testing" + + "k8s.io/client-go/kubernetes/fake" +) + +// fakeSubmitter captures the request + returns a canned response. +// Used by Run() tests to exercise the orchestrator without needing +// an httptest.Server (covered by client_test.go). +type fakeSubmitter struct { + gotRequest *SubmitRequest + resp *SubmitResponse + err error +} + +func (f *fakeSubmitter) Submit(_ context.Context, req *SubmitRequest) (*SubmitResponse, error) { + f.gotRequest = req + return f.resp, f.err +} + +// TestRun_DetachPath_HappyPath: --detach exits immediately after +// the 201 with the reconnect hint. No watch loop, no log streaming. +func TestRun_DetachPath_HappyPath(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{ + JobName: "ingestor-abc", + Namespace: "tracebloc", + Replay: false, + }, + } + var out bytes.Buffer + + res, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "apiVersion: tracebloc.io/v1\n", + Detach: true, + Out: &out, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Submit == nil || res.Submit.JobName != "ingestor-abc" { + t.Errorf("Result.Submit lost: %+v", res.Submit) + } + if res.Watch != nil { + t.Errorf("Result.Watch = %+v, want nil (detach skips watch)", res.Watch) + } + for _, want := range []string{ + "Submitted: jobs-manager spawned ingestor Job tracebloc/ingestor-abc", + "Detached (no log streaming)", + "kubectl logs -f -n tracebloc job/ingestor-abc", + } { + if !strings.Contains(out.String(), want) { + t.Errorf("output missing %q in:\n%s", want, out.String()) + } + } +} + +// TestRun_ReplayPath: replay=true changes the announcement +// wording — "attaching to existing Job" instead of "spawned" — +// because the cluster is already doing the work. +func TestRun_ReplayPath(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{ + JobName: "ingestor-existing", + Namespace: "tracebloc", + Replay: true, + }, + } + var out bytes.Buffer + + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml", + Detach: true, // skip the watch for this test + Out: &out, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if !strings.Contains(out.String(), "Replayed:") { + t.Errorf("output missing Replayed framing:\n%s", out.String()) + } + if !strings.Contains(out.String(), "attaching to existing Job") { + t.Errorf("output missing replay-specific wording:\n%s", out.String()) + } +} + +// TestRun_SubmitErrorPropagates: a non-2xx from jobs-manager +// stops Run before any watching happens. The error surfaces with +// jobs-manager's body framing (the client.go path). +func TestRun_SubmitErrorPropagates(t *testing.T) { + sub := &fakeSubmitter{ + err: &SubmitError{ + StatusCode: 422, + Body: `{"detail":"bad spec"}`, + Endpoint: "http://jm/internal/submit-ingestion-run", + }, + } + var out bytes.Buffer + + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml", + Out: &out, + }) + if err == nil { + t.Fatal("Run returned nil on submit error") + } + if !IsSubmitError(err) { + t.Errorf("err is not *SubmitError: %T", err) + } +} + +// TestRun_BuildRequestErrorPropagates: a crypto/rand failure in +// BuildRequest stops Run before the submitter even gets called. +// We can't easily mock crypto/rand, but we can verify the error +// path is wired by checking that any failure here doesn't reach +// the submitter. This test is more about the contract than the +// trigger. +func TestRun_PassesRequestFieldsThrough(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{JobName: "j", Namespace: "ns"}, + } + var out bytes.Buffer + + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml-content-verbatim", + IdempotencyKey: "my-key-override", + ImageDigest: "sha256:abc", + Detach: true, + Out: &out, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + if sub.gotRequest == nil { + t.Fatal("submitter never called") + } + if sub.gotRequest.IngestConfig != "yaml-content-verbatim" { + t.Errorf("IngestConfig = %q, want yaml-content-verbatim", + sub.gotRequest.IngestConfig) + } + if sub.gotRequest.IdempotencyKey != "my-key-override" { + t.Errorf("IdempotencyKey = %q, want override value", + sub.gotRequest.IdempotencyKey) + } + if sub.gotRequest.ImageDigest != "sha256:abc" { + t.Errorf("ImageDigest = %q, want sha256:abc", + sub.gotRequest.ImageDigest) + } +} + +// TestRun_NilOutDefaultsToDiscard: callers passing nil Out +// shouldn't panic. The orchestrator silently discards output. +func TestRun_NilOutDefaultsToDiscard(t *testing.T) { + sub := &fakeSubmitter{ + resp: &SubmitResponse{JobName: "j", Namespace: "ns"}, + } + _, err := Run(context.Background(), Options{ + Submitter: sub, + Client: fake.NewClientset(), + IngestConfigYAML: "yaml", + Detach: true, + Out: nil, + }) + if err != nil { + t.Fatalf("Run with nil Out panicked or errored: %v", err) + } +} + +// TestIsWatchError: pin the contract that the orchestrator uses +// to distinguish watch-phase failures (exit 9) from submit-phase +// failures (exit 8). Bugbot r1 found the missing distinction; +// this test guards against a regression that drops the typing. +func TestIsWatchError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"plain error", errors.New("plain"), false}, + {"submit error", &SubmitError{StatusCode: 422}, false}, + {"watch error", &WatchError{Err: errors.New("inner")}, true}, + {"wrapped watch error", fmt.Errorf("outer: %w", &WatchError{Err: errors.New("inner")}), true}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsWatchError(c.err); got != c.want { + t.Errorf("IsWatchError = %v, want %v", got, c.want) + } + }) + } +} + +// TestIsAuthError: helper smoke test. Pin the contract used by +// the CLI's exit-code mapping. +func TestIsAuthError(t *testing.T) { + cases := []struct { + name string + err error + want bool + }{ + {"nil", nil, false}, + {"non-submit error", errors.New("network"), false}, + {"submit 401", &SubmitError{StatusCode: 401}, true}, + {"submit 403", &SubmitError{StatusCode: 403}, true}, + {"submit 422", &SubmitError{StatusCode: 422}, false}, + {"submit 500", &SubmitError{StatusCode: 500}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := IsAuthError(c.err); got != c.want { + t.Errorf("IsAuthError = %v, want %v", got, c.want) + } + }) + } +} diff --git a/internal/submit/summary.go b/internal/submit/summary.go new file mode 100644 index 0000000..23fd375 --- /dev/null +++ b/internal/submit/summary.go @@ -0,0 +1,388 @@ +package submit + +import ( + "bufio" + "bytes" + "fmt" + "io" + "regexp" + "strconv" + "strings" +) + +// Summary is the parsed contents of the ingestor's 📊 INGESTION +// SUMMARY 📊 banner. Fields mirror what +// tracebloc_ingestor/ingestors/base.py:624+ prints. Zero values +// are valid — an early failure may produce a summary with most +// counts at 0 (the ingestor still prints the banner so operators +// can see what got through). +// +// All counts are int64 to fit row counts well past int32's 2.1B +// ceiling — a customer ingesting a few-billion-row table would +// silently truncate with int. +type Summary struct { + // IngestorID is the run identifier the ingestor logs at the + // top of the banner. Useful in the customer-facing panel as + // "you can grep cluster logs for this ID." + IngestorID string + + // TotalRecords is the row count the ingestor saw in the + // source data. Includes every row regardless of outcome. + TotalRecords int64 + + // ProcessedRecords is the row count that made it through + // validation (passed FileTypeValidator, ImageResolutionValidator, + // etc.). Excludes invalid rows. + ProcessedRecords int64 + + // InsertedRecords is the row count that landed in the + // cluster-internal MySQL. The "I actually have this data + // staged" metric — this is what matters for downstream + // training jobs. + InsertedRecords int64 + + // APISentRecords is the row count that synced metadata to + // the central tracebloc backend. Only the row count + label + // is sent, not the raw data; this is the "central catalog + // knows about this dataset" metric. + APISentRecords int64 + + // SkippedRecords is the row count rejected by validators + // (wrong dimensions, missing image file, etc.). Non-fatal + // for the run but worth surfacing — a customer with 50% + // skipped wants to see that. + SkippedRecords int64 + + // FileTransferFailures is the count of files (NOT rows) that + // failed to transfer to the requests-proxy. Distinct from + // FailedRecords because file transfer is a separate stage + // from DB insertion. Non-zero here is the dominant "your + // network is flaky" signal. + FileTransferFailures int64 + + // FailedRecords is the row count that errored at the + // DB-insert stage (constraint violation, type mismatch, + // connection drop). The catch-all "something went wrong + // at the storage layer" bucket. + FailedRecords int64 +} + +// HasFailures returns true if any failure-class counter is non-zero. +// Used by the orchestrator to decide which exit code to return +// (success: 0, ingest-with-failures: non-zero) and how to color +// the rendered panel. +func (s *Summary) HasFailures() bool { + if s == nil { + return false + } + return s.FileTransferFailures > 0 || s.FailedRecords > 0 +} + +// SuccessRate returns a 0-100 percentage for the panel header. +// Defined as ProcessedRecords / TotalRecords; returns 0 when +// TotalRecords is 0 to avoid divide-by-zero in early-failure +// banners. +func (s *Summary) SuccessRate() float64 { + if s == nil || s.TotalRecords == 0 { + return 0 + } + return float64(s.ProcessedRecords) / float64(s.TotalRecords) * 100 +} + +// ansiCodeRE matches the ANSI SGR (Select Graphic Rendition) +// escape sequences the ingestor uses for its color output — +// `\x1b[1m` (bold), `\x1b[36m` (cyan), `\x1b[0m` (reset), etc. +// The ingestor prints these inline in the summary text; we strip +// them before parsing so a future palette change doesn't break +// the parser. +var ansiCodeRE = regexp.MustCompile(`\x1b\[[0-9;]*m`) + +// stripANSI removes SGR codes from a single line, returning the +// printable text. Implementation note: this is a hot path for the +// log stream so we avoid an allocation when no codes are present. +func stripANSI(line string) string { + if !strings.Contains(line, "\x1b[") { + return line + } + return ansiCodeRE.ReplaceAllString(line, "") +} + +// bannerStartMarker is the ingestor's literal banner-header line +// (with the BOLD + CYAN ANSI prefix stripped by stripANSI before +// matching). When we see this, we flip to "inside-banner" mode. +const bannerStartMarker = "📊 INGESTION SUMMARY 📊" + +// bannerEndMarker is the equals-rule the ingestor prints AFTER +// the last metric line (a second `═`x60 line). We use it as the +// terminator so a parser at end-of-stream emits a complete +// Summary even if the Pod was killed mid-log-flush. +const bannerEndMarker = "════════════════════════════════════════════════════════════" + +// fieldPatterns are the regex patterns for each Summary field, +// keyed by the human-readable label the ingestor prints. The +// values are populated by NewSummaryParser via Compile-once-at- +// init. Each pattern matches a `Label: ` shape with +// optional spacing and an optional `,`-separated digit format +// (e.g. "1,234,567"). +// +// Maintained as a parallel slice of {label, pointer-target} +// rather than a map so the order of parsing matches the order +// the ingestor prints them — useful if a future ingestor adds a +// new line, the parser doesn't have to re-scan the existing ones. +var fieldPatterns = []struct { + prefix string + apply func(s *Summary, n int64) +}{ + {"📈 Total Records Found:", func(s *Summary, n int64) { s.TotalRecords = n }}, + {"✅ Successfully Processed:", func(s *Summary, n int64) { s.ProcessedRecords = n }}, + {"💾 Inserted to Database:", func(s *Summary, n int64) { s.InsertedRecords = n }}, + {"🚀 Sent to API:", func(s *Summary, n int64) { s.APISentRecords = n }}, + {"⏭️ Skipped Records:", func(s *Summary, n int64) { s.SkippedRecords = n }}, + {"📁 File Transfer Failures:", func(s *Summary, n int64) { s.FileTransferFailures = n }}, + {"❌ Failed DB Insertion:", func(s *Summary, n int64) { s.FailedRecords = n }}, +} + +// numberRE captures the trailing digit-group on a metric line. +// Allows optional thousands-separator commas the ingestor's +// `f"{count:,}"` formatting emits. +var numberRE = regexp.MustCompile(`([0-9][0-9,]*)\s*$`) + +// ingestorIDRE matches the ingestor-ID line specifically; the +// value is a UUID-ish string, not a number, so it gets its own +// pattern. +var ingestorIDRE = regexp.MustCompile(`Ingestor ID:\s*(.+?)\s*$`) + +// SummaryParser is a streaming parser for the 📊 banner. Feed it +// log lines as they arrive (any chunk size, any line splitting); +// Result() returns the accumulated Summary at any point. The +// banner-end marker latches the result so post-banner log lines +// don't perturb it. +// +// The parser is stateful but not thread-safe — the watch loop +// uses it from a single goroutine (the log-streaming TeeReader), +// so no synchronization needed. +type SummaryParser struct { + // buf accumulates partial-line input across Feed calls. The + // log stream from the API server arrives in TCP-sized chunks + // that may split lines; we buffer until we see a '\n' to + // finalize each line. + buf bytes.Buffer + + // summary is the accumulator. nil until we see the banner + // header — Result returns nil if the run never produced one. + summary *Summary + + // finalized latches when we see the banner-end marker. After + // that, additional Feed calls don't modify summary (the + // ingestor may keep logging after the banner, e.g. shutdown + // messages; those shouldn't perturb the result). + finalized bool + + // insideBanner is true between bannerStartMarker and + // bannerEndMarker. Outside this window, lines are ignored + // (so e.g. a stray emoji in earlier log output doesn't + // trigger spurious parsing). + insideBanner bool + + // sawAnyField latches once we successfully parse a + // fieldPatterns line (regardless of whether the count is + // zero). Used by feedLine to distinguish the opening ═-rule + // (no fields yet → ignore) from the closing one (fields + // already parsed → finalize). The earlier "any field + // non-zero" check failed on banners where every metric was + // genuinely 0 (early failure case) — Bugbot caught this on + // PR #10 round 2. + sawAnyField bool +} + +// NewSummaryParser returns an initialized parser. Caller's +// goroutine owns it for the duration of the watch loop. +func NewSummaryParser() *SummaryParser { + return &SummaryParser{} +} + +// Feed accepts arbitrary log bytes; the parser buffers and splits +// internally. Safe to call with partial lines, multiple lines, or +// empty input. +func (p *SummaryParser) Feed(b []byte) { + if p.finalized { + return + } + _, _ = p.buf.Write(b) + for { + idx := bytes.IndexByte(p.buf.Bytes(), '\n') + if idx < 0 { + // No complete line yet — wait for more input. + return + } + line := p.buf.Next(idx + 1) // consume up to and including '\n' + p.feedLine(string(bytes.TrimRight(line, "\n"))) + } +} + +// FlushLine forces parsing of any buffered partial-line content. +// Called at end-of-stream by the watch loop in case the Pod +// terminated without a final '\n' (rare but possible if the +// container's stdout was killed mid-write). +func (p *SummaryParser) FlushLine() { + if p.buf.Len() > 0 && !p.finalized { + p.feedLine(p.buf.String()) + p.buf.Reset() + } +} + +// feedLine parses a single line, ANSI-stripped. The state machine +// has three regions: +// +// - Pre-banner: skip until we see the start marker +// - Inside banner: match each line against fieldPatterns + the +// Ingestor ID line +// - End marker: latch and stop processing +func (p *SummaryParser) feedLine(rawLine string) { + line := stripANSI(rawLine) + if strings.Contains(line, bannerStartMarker) { + p.summary = &Summary{} + p.insideBanner = true + return + } + if !p.insideBanner { + return + } + // Banner-end check: a long row of '═'. Only count when we've + // already crossed the start marker (the start banner ALSO + // has a '═' rule before the header, which we want to ignore). + if strings.Contains(line, bannerEndMarker) { + // Two ═-rules in the banner: one immediately after the + // header, one at the very end. We use a simple counter + // to distinguish — first one we see while insideBanner + // is the post-header rule (skip), second is the + // post-metrics rule (finalize). + // + // Actually the simpler approach: check whether we've + // parsed any field yet. If so, this ═ is the closing + // rule; if not, it's the opening one. The ingestor + // always prints fields between the two rules. + if p.summary != nil && p.hasParsedAnyField() { + p.finalized = true + } + return + } + + // Ingestor ID line is the first content line in the banner. + if m := ingestorIDRE.FindStringSubmatch(line); m != nil { + p.summary.IngestorID = strings.TrimSpace(m[1]) + return + } + + // Otherwise: try each field pattern. The prefix match is + // linear over a 7-element slice — microscopic overhead per + // line, and the fixed order matches the ingestor's print + // order. + for _, fp := range fieldPatterns { + if !strings.Contains(line, fp.prefix) { + continue + } + m := numberRE.FindStringSubmatch(line) + if m == nil { + return + } + // Strip thousands-separator commas. + raw := strings.ReplaceAll(m[1], ",", "") + n, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return // malformed; ignore this line + } + fp.apply(p.summary, n) + // Latch regardless of value. An all-zero banner is a + // real shape (early failure: the ingestor still prints + // the summary structure with all counts at 0). Bugbot + // PR #10 r2 flagged the "non-zero" hasParsedAnyField + // check as a finalization hole for this case. + p.sawAnyField = true + return + } +} + +// hasParsedAnyField reports whether the parser has seen and +// applied at least one fieldPatterns line. Used to disambiguate +// the two ═-rules in the banner (see feedLine). +func (p *SummaryParser) hasParsedAnyField() bool { + return p.sawAnyField +} + +// Result returns the accumulated Summary. nil if the parser never +// saw a banner header (early failure, OOM before the ingestor got +// to print its results). Safe to call at any point — the parser +// returns the in-progress accumulator if not yet finalized. +func (p *SummaryParser) Result() *Summary { + return p.summary +} + +// RenderPanel returns a multi-line, customer-facing rendering of +// the Summary for display in the orchestrator's success/failure +// message. Format: +// +// ┌─ Ingestion summary ──────────────────────────┐ +// │ Ingestor ID: │ +// │ Total records: 1,234 │ +// │ Inserted: 1,200 │ +// │ Skipped: 4 │ +// │ File transfer failures: 0 │ +// │ DB-insert failures: 30 │ +// │ Success rate: 97.2% │ +// └──────────────────────────────────────────────┘ +// +// Uses box-drawing characters for visual structure. Plain ASCII +// fallback could be added in v0.2 for terminals that don't render +// Unicode (rare on modern OS X/Linux/Windows-Terminal). +func RenderPanel(s *Summary) string { + if s == nil { + return "" + } + const labelWidth = 26 + var b strings.Builder + b.WriteString("┌─ Ingestion summary ──────────────────────────┐\n") + row := func(label, value string) { + fmt.Fprintf(&b, "│ %-*s %s\n", labelWidth, label, value) + } + if s.IngestorID != "" { + row("Ingestor ID:", s.IngestorID) + } + row("Total records:", commaSep(s.TotalRecords)) + row("Inserted:", commaSep(s.InsertedRecords)) + row("Sent to API:", commaSep(s.APISentRecords)) + row("Skipped:", commaSep(s.SkippedRecords)) + row("File transfer failures:", commaSep(s.FileTransferFailures)) + row("DB-insert failures:", commaSep(s.FailedRecords)) + row("Success rate:", fmt.Sprintf("%.1f%%", s.SuccessRate())) + b.WriteString("└──────────────────────────────────────────────┘\n") + return b.String() +} + +// commaSep formats an int64 with thousands-separator commas to +// match the ingestor's own banner format. Pure Go, no x/text. +func commaSep(n int64) string { + s := strconv.FormatInt(n, 10) + if len(s) <= 3 { + return s + } + // Insert commas every 3 digits from the right. Handle the + // optional leading '-' by carving it off first. + neg := "" + if s[0] == '-' { + neg = "-" + s = s[1:] + } + var out []byte + for i, c := range []byte(s) { + if i > 0 && (len(s)-i)%3 == 0 { + out = append(out, ',') + } + out = append(out, c) + } + return neg + string(out) +} + +// (compile-time test: bufio + io are referenced via Feed's buffer) +var _ = bufio.ScanLines +var _ io.Writer = parserWriter{} // ensures parserWriter satisfies io.Writer diff --git a/internal/submit/summary_test.go b/internal/submit/summary_test.go new file mode 100644 index 0000000..5bfe950 --- /dev/null +++ b/internal/submit/summary_test.go @@ -0,0 +1,240 @@ +package submit + +import ( + "strings" + "testing" +) + +// realIngestorBanner mirrors what +// tracebloc_ingestor/ingestors/base.py:624+ actually prints, +// ANSI codes and all. The parser strips ANSI before matching so +// the test is bit-exact with the production logs. +// +// The leading "preamble" line + trailing "post-banner" lines +// simulate what real ingestor logs look like — the parser must +// ignore non-banner content + finalize on the closing rule. +var realIngestorBanner = "starting up...\n" + + "loaded 1234 rows from labels.csv\n" + + "\n" + + "\x1b[36m" + strings.Repeat("═", 60) + "\x1b[0m\n" + + "\x1b[1m\x1b[36m📊 INGESTION SUMMARY 📊\x1b[0m\n" + + "\x1b[36m" + strings.Repeat("═", 60) + "\x1b[0m\n" + + "\x1b[1mIngestor ID:\x1b[0m \x1b[34mrun-abc-123\x1b[0m\n" + + "\x1b[1m📈 Total Records Found:\x1b[0m \x1b[34m1,234\x1b[0m\n" + + "\x1b[1m✅ Successfully Processed:\x1b[0m \x1b[32m1,200\x1b[0m\n" + + "\x1b[1m💾 Inserted to Database:\x1b[0m \x1b[32m1,200\x1b[0m\n" + + "\x1b[1m🚀 Sent to API:\x1b[0m \x1b[32m1,150\x1b[0m\n" + + "\x1b[1m⏭️ Skipped Records:\x1b[0m \x1b[33m4\x1b[0m\n" + + "\x1b[1m📁 File Transfer Failures:\x1b[0m \x1b[32m0\x1b[0m\n" + + "\x1b[1m❌ Failed DB Insertion:\x1b[0m \x1b[31m30\x1b[0m\n" + + "\x1b[36m" + strings.Repeat("═", 60) + "\x1b[0m\n" + + "ingestor exiting cleanly\n" + +// TestSummaryParser_RealBannerEndToEnd pins the parser against +// the actual ingestor's output format. If a regression breaks +// any field's extraction, this test fails with a clear "got X +// want Y" for that specific counter. +func TestSummaryParser_RealBannerEndToEnd(t *testing.T) { + p := NewSummaryParser() + p.Feed([]byte(realIngestorBanner)) + + got := p.Result() + if got == nil { + t.Fatal("parser returned nil Result; expected populated Summary") + } + + cases := []struct { + name string + got any + want any + }{ + {"IngestorID", got.IngestorID, "run-abc-123"}, + {"TotalRecords", got.TotalRecords, int64(1234)}, + {"ProcessedRecords", got.ProcessedRecords, int64(1200)}, + {"InsertedRecords", got.InsertedRecords, int64(1200)}, + {"APISentRecords", got.APISentRecords, int64(1150)}, + {"SkippedRecords", got.SkippedRecords, int64(4)}, + {"FileTransferFailures", got.FileTransferFailures, int64(0)}, + {"FailedRecords", got.FailedRecords, int64(30)}, + } + for _, c := range cases { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.name, c.got, c.want) + } + } +} + +// TestSummaryParser_HasFailures pins the failure-detection logic +// that the orchestrator uses to choose between success exit code +// (0) and ingest-failure exit code (9). +func TestSummaryParser_HasFailures(t *testing.T) { + cases := []struct { + name string + s *Summary + want bool + }{ + {"nil", nil, false}, + {"all zero", &Summary{TotalRecords: 100, ProcessedRecords: 100}, false}, + {"file transfer failures", &Summary{FileTransferFailures: 1}, true}, + {"failed records", &Summary{FailedRecords: 1}, true}, + {"both", &Summary{FileTransferFailures: 1, FailedRecords: 1}, true}, + // Skipped records are NOT failures — they're rows that + // validators rejected. The customer wants to see the + // count but it doesn't change the exit code. + {"skipped is not failure", &Summary{SkippedRecords: 100}, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.s.HasFailures(); got != c.want { + t.Errorf("HasFailures = %v, want %v", got, c.want) + } + }) + } +} + +// TestSummaryParser_SuccessRate pins the math that feeds the +// rendered panel's "Success rate: XX%" line. Divide-by-zero on +// empty banner is the critical edge case. +func TestSummaryParser_SuccessRate(t *testing.T) { + cases := []struct { + name string + s *Summary + want float64 + }{ + {"nil", nil, 0}, + {"empty banner", &Summary{}, 0}, + {"100%", &Summary{TotalRecords: 100, ProcessedRecords: 100}, 100}, + {"50%", &Summary{TotalRecords: 100, ProcessedRecords: 50}, 50}, + {"all failed", &Summary{TotalRecords: 100}, 0}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := c.s.SuccessRate(); got != c.want { + t.Errorf("SuccessRate = %v, want %v", got, c.want) + } + }) + } +} + +// TestSummaryParser_StreamedInChunks: real log streams arrive in +// arbitrary-sized chunks. Feed the banner one byte at a time to +// pin that the parser correctly buffers and finalizes despite +// partial-line input. +func TestSummaryParser_StreamedInChunks(t *testing.T) { + p := NewSummaryParser() + for i := 0; i < len(realIngestorBanner); i++ { + p.Feed([]byte{realIngestorBanner[i]}) + } + got := p.Result() + if got == nil { + t.Fatal("byte-by-byte feed produced nil Result") + } + if got.TotalRecords != 1234 || got.FailedRecords != 30 { + t.Errorf("chunked feed produced different result than monolithic: TotalRecords=%d FailedRecords=%d", + got.TotalRecords, got.FailedRecords) + } +} + +// TestSummaryParser_NoBanner: if the run died before producing a +// banner (image crashloop, OOM at startup), Result returns nil +// rather than an empty Summary. The orchestrator uses this +// nil-check to decide whether to render the panel at all. +func TestSummaryParser_NoBanner(t *testing.T) { + p := NewSummaryParser() + p.Feed([]byte("starting up...\nerror: connection refused\n")) + if got := p.Result(); got != nil { + t.Errorf("Result on no-banner log = %+v, want nil", got) + } +} + +// TestSummaryParser_PostBannerLogsIgnored: lines after the closing +// ═-rule are ignored. The ingestor may print shutdown messages +// after the summary; those shouldn't perturb the parsed counts +// (e.g. a regex-misfire interpreting "1234" in an unrelated log +// line as TotalRecords). +func TestSummaryParser_PostBannerLogsIgnored(t *testing.T) { + p := NewSummaryParser() + p.Feed([]byte(realIngestorBanner)) + pre := *p.Result() + p.Feed([]byte("📈 Total Records Found: 999999999\n")) // would alter TotalRecords if not finalized + post := *p.Result() + if pre != post { + t.Errorf("post-banner line altered Summary; pre=%+v post=%+v", pre, post) + } +} + +// TestStripANSI: the parser strips ANSI SGR codes from each line +// before matching. Validate the regex handles common shapes. +func TestStripANSI(t *testing.T) { + cases := []struct { + in, want string + }{ + {"plain text", "plain text"}, + {"\x1b[1mbold\x1b[0m", "bold"}, + {"\x1b[1;36mbold-cyan\x1b[0m", "bold-cyan"}, + {"prefix\x1b[31mred\x1b[0msuffix", "prefixredsuffix"}, + {"", ""}, + } + for _, c := range cases { + if got := stripANSI(c.in); got != c.want { + t.Errorf("stripANSI(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +// TestRenderPanel_BasicShape: the panel rendering is what the +// customer sees on success; pin a few key lines so a refactor +// breaks the test rather than silently producing weird output. +func TestRenderPanel_BasicShape(t *testing.T) { + s := &Summary{ + IngestorID: "run-abc", + TotalRecords: 1234567, + InsertedRecords: 1200000, + APISentRecords: 1150000, + SkippedRecords: 4000, + FileTransferFailures: 30, + FailedRecords: 5, + } + got := RenderPanel(s) + for _, want := range []string{ + "Ingestion summary", + "run-abc", + "1,234,567", // commaSep formatting + "1,200,000", // commaSep formatting + "30", // file transfer failures + "DB-insert failures:", + } { + if !strings.Contains(got, want) { + t.Errorf("RenderPanel missing %q in:\n%s", want, got) + } + } +} + +// TestRenderPanel_Nil: nil summary returns empty string so the +// orchestrator can blind-print without a guard. +func TestRenderPanel_Nil(t *testing.T) { + if got := RenderPanel(nil); got != "" { + t.Errorf("RenderPanel(nil) = %q, want empty", got) + } +} + +// TestCommaSep: small helper test. Pin the boundary cases that +// would catch off-by-one in the comma-insertion loop. +func TestCommaSep(t *testing.T) { + cases := []struct { + in int64 + want string + }{ + {0, "0"}, + {999, "999"}, + {1000, "1,000"}, + {12345, "12,345"}, + {1234567, "1,234,567"}, + {-1234, "-1,234"}, + } + for _, c := range cases { + if got := commaSep(c.in); got != c.want { + t.Errorf("commaSep(%d) = %q, want %q", c.in, got, c.want) + } + } +} diff --git a/internal/submit/watch.go b/internal/submit/watch.go new file mode 100644 index 0000000..3ef8d47 --- /dev/null +++ b/internal/submit/watch.go @@ -0,0 +1,517 @@ +package submit + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +// Watch-loop tunables. Both deliberately conservative — Phase 4's +// watching is the dominant time-spend of a typical push (the actual +// data move was done in Phase 3; what's left is the in-cluster +// ingestion which can take minutes-to-an-hour for large datasets). +const ( + // JobPollInterval is how often the watch loop re-Gets the Job. + // 2s is a sweet spot: human-perceptible enough that a + // 30-second ingestion has clean lifecycle output, light + // enough that an hour-long ingestion adds <2000 API calls + // (negligible at the apiserver's ~10k req/s ceiling). + JobPollInterval = 2 * time.Second + + // JobWatchTimeout is the absolute cap on a single + // dataset-push's watch phase. 1 hour is generous — typical + // image_classification ingestions finish in <10 min; cap + // exists to avoid infinite hangs if the cluster goes weird + // (kubelet stops reporting status, etc.). Customers running + // hours-long ingestions should use --detach. + JobWatchTimeout = 1 * time.Hour + + // PodPollInterval is how often we look for the ingestor Job's + // Pod once we have the Job name. Same 2s as the Job-level + // poll; same rationale. + PodPollInterval = 2 * time.Second + + // PodReadyTimeout caps how long the Job's Pod has to be + // schedulable + Running before we give up looking for it. + // 5 min covers image pull on a slow registry; beyond that + // the ingestion isn't going to start at all and the customer + // wants the diagnostic. + PodReadyTimeout = 5 * time.Minute +) + +// JobOutcome enumerates the terminal states the watch loop reports. +// The orchestrator (submit.go) maps these to exit codes. +type JobOutcome int + +const ( + // JobOutcomeUnknown is the zero value — never returned, used + // only as a switch-default sentinel. + JobOutcomeUnknown JobOutcome = iota + + // JobOutcomeSucceeded means the ingestor Job's Pod exited 0. + // Maps to exit code 0 in the orchestrator. + JobOutcomeSucceeded + + // JobOutcomeFailed means the ingestor Job's Pod exited + // non-zero (any cause: ingestion runtime error, OOM, image + // crashloop). Maps to the "ingest" exit code (9) in the + // orchestrator. The Pod-side summary banner may or may not + // have been printed depending on how far the run got — the + // orchestrator parses what it can. + JobOutcomeFailed + + // JobOutcomeDetached means the customer Ctrl-C'd mid-watch. + // jobs-manager already accepted the run; the cluster will + // continue without us. Maps to exit code 0 with a "reconnect + // with kubectl logs" hint in the orchestrator output. + JobOutcomeDetached +) + +func (o JobOutcome) String() string { + switch o { + case JobOutcomeSucceeded: + return "Succeeded" + case JobOutcomeFailed: + return "Failed" + case JobOutcomeDetached: + return "Detached" + default: + return "Unknown" + } +} + +// WatchResult bundles everything the orchestrator wants from the +// watch loop. Outcome drives the exit code; PodName is what the +// detach-hint prints; Summary is the structured INGESTION SUMMARY +// (nil if the run didn't produce one — early failure, OOM at +// startup, etc.). +type WatchResult struct { + Outcome JobOutcome + PodName string + + // Summary is the parsed 📊 banner, nil on early failure. + // The orchestrator decides whether to render the panel + // (success path) or include it in the failure framing + // (failed-after-summary path). + Summary *Summary + + // DetachReason qualifies the Detached outcome — set only + // when Outcome == JobOutcomeDetached. Lets the orchestrator + // print accurate diagnostics ("Detached on signal" vs + // "Pod didn't become Ready within timeout" vs "Watch cap + // exceeded") instead of the previous one-size-fits-all + // message. Bugbot PR #10 r7 flagged the misleading "signal" + // framing for the timeout-detach paths. + DetachReason DetachReason +} + +// DetachReason enumerates the conditions that produce a Detached +// outcome. Used by the orchestrator's diagnostic output only — +// the exit-code mapping treats all detach reasons as success (0) +// because the cluster keeps running regardless of why we stopped +// watching. +type DetachReason int + +const ( + // DetachReasonNone is the zero value, used when Outcome is + // not Detached. + DetachReasonNone DetachReason = iota + + // DetachReasonSignal: customer pressed Ctrl-C (or a parent + // process sent SIGTERM). The original Detach semantic. + DetachReasonSignal + + // DetachReasonPodWaitTimeout: PodReadyTimeout (5 min) + // exhausted before the ingestor Pod reached a useful + // phase. Slow image pull, scheduling backlog, PSA rejection. + DetachReasonPodWaitTimeout + + // DetachReasonWatchCap: JobWatchTimeout (1 hour) exceeded + // during log streaming. Long-running ingestion that + // outlasted the observation window. + DetachReasonWatchCap +) + +// WatchJob is the top-level watch loop: poll the Job until it +// reaches a terminal phase, stream the Pod's logs while it's +// running, and return a WatchResult. +// +// SIGINT contract (Bugbot-r9 echo for the previous package): +// the caller (cli/main.go via signal.NotifyContext) is expected +// to cancel ctx on Ctrl-C. WatchJob detects ctx.Err() == Canceled +// and returns Outcome=Detached rather than treating it as a poll +// failure. The customer who Ctrl-C'd during the watch sees the +// "your ingestion is still running in the cluster; reconnect with +// kubectl logs " hint. +// +// out is the customer-facing log stream (typically os.Stdout). +// Logs are written verbatim — no prefix, no munging — so the +// stream looks identical to `kubectl logs -f `. +func WatchJob( + ctx context.Context, + cs kubernetes.Interface, + namespace, jobName string, + out io.Writer, +) (*WatchResult, error) { + // Keep the customer's original ctx separately so finalJobStatus + // can derive a FRESH 30s context from it (rather than inheriting + // a possibly-depleted JobWatchTimeout). Bugbot PR #10 r2: the + // previous "wrap everything in JobWatchTimeout" approach starved + // finalJobStatus's budget when streaming used most of the hour, + // so a successful slow ingestion misreported as Unknown → exit 9. + customerCtx := ctx + + // JobWatchTimeout caps the pod-wait + log-stream phases (the + // time-spend dominant parts of the watch). finalJobStatus gets + // its own ctx below. + watchCtx, cancel := context.WithTimeout(customerCtx, JobWatchTimeout) + defer cancel() + + // 1. Wait for the ingestor Job's Pod to exist + reach Running. + // jobs-manager creates the Job and Kubernetes spawns the + // Pod asynchronously, so the Pod usually isn't there the + // moment after the 201 comes back. + podName, err := waitForJobPod(watchCtx, cs, namespace, jobName) + if err != nil { + if errors.Is(err, context.Canceled) { + // SIGINT before the Pod even appeared. jobs-manager + // has accepted the run, the cluster will run it, + // the CLI is just not watching anymore. + return &WatchResult{ + Outcome: JobOutcomeDetached, + DetachReason: DetachReasonSignal, + }, nil + } + // PodReadyTimeout (5min) exhausted = slow image pull / + // scheduling backlog / PSA still rejecting. The submit + // was accepted, the run will (eventually) execute in the + // cluster — the CLI just gave up observing within the + // timeout. Treat as Detached, not ingest-failed: bumping + // to exit 9 would falsely claim the ingestion failed. + // Bugbot PR #10 r5 flagged the false-positive exit code. + if errors.Is(err, context.DeadlineExceeded) { + return &WatchResult{ + Outcome: JobOutcomeDetached, + DetachReason: DetachReasonPodWaitTimeout, + }, nil + } + return nil, fmt.Errorf("waiting for ingestor Pod: %w", err) + } + + // 2. Stream Pod logs. This blocks until the Pod terminates + // or ctx is cancelled. We don't need a separate Job-status + // poll here because the Pod terminating drains the log + // stream — when GetLogs(Follow=true) returns EOF, the + // Pod has completed (success or failure). + // + // Any text the ingestor prints (including the 📊 banner + // at the end) flows verbatim through `out`. We also feed + // a side-channel to the summary parser so we end up with + // a structured representation of the banner without + // requiring a second log fetch post-completion. + summary, logErr := streamPodLogsAndParse(watchCtx, cs, namespace, podName, out) + + // 3. Detach branches — checked FIRST, since the customer's SIGINT + // or the watch-cap expiry is the operative intent and takes + // precedence over any stream error: + // - customerCtx canceled = SIGINT + // - watchCtx expired (DeadlineExceeded) = JobWatchTimeout cap + // hit during streaming (1-hour observation window exceeded) + // + // Both are "the cluster keeps running; the CLI just gave up + // observing." Same UX as the PodReadyTimeout case from r5 + // above — exit 0 with the kubectl-logs reconnect hint. + // Bugbot PR #10 r6 flagged the inconsistency: r5 detached + // on PodReady timeout but the watch-cap exit still mapped + // to exit 9. + if errors.Is(customerCtx.Err(), context.Canceled) || errors.Is(watchCtx.Err(), context.DeadlineExceeded) { + reason := DetachReasonSignal + if errors.Is(watchCtx.Err(), context.DeadlineExceeded) && + !errors.Is(customerCtx.Err(), context.Canceled) { + // Pure watchCtx-only expiry = JobWatchTimeout. The + // customerCtx-canceled case takes precedence (if both + // fired, the customer's intent was SIGINT). + reason = DetachReasonWatchCap + } + return &WatchResult{ + Outcome: JobOutcomeDetached, + PodName: podName, + Summary: summary, // may be partial + DetachReason: reason, + }, nil + } + + // 4. Final status check with a FRESH 30s budget derived from + // the customer's ctx (not watchCtx, which may be near- + // expired after a long log stream). Bugbot PR #10 r2: + // inheriting watchCtx's depleted budget caused successful + // slow ingestions to misreport as Unknown. + // + // The Job — not the log stream — is the source of truth for + // success/failure, so we ALWAYS consult it here, INCLUDING when + // the log stream broke for a non-ctx reason (#28: the watched + // Pod was replaced / restarted / deleted mid-follow, e.g. a + // backoffLimit retry). A broken stream is only fatal if we also + // can't determine the Job's outcome. + // + // The fresh ctx still propagates SIGINT (parent is customerCtx, + // which carries signal.NotifyContext's cancel); a Ctrl-C in this + // window falls into the detach branches below. + finalCtx, finalCancel := context.WithTimeout(customerCtx, 30*time.Second) + defer finalCancel() + outcome, statusErr := finalJobStatus(finalCtx, cs, namespace, jobName) + + // A non-ctx log-stream error is incidental if the Job still + // reached a terminal state. Previously ANY such error (e.g. + // "container is terminated" once the Pod was replaced by a retry) + // returned exit 9 even when the Job ultimately succeeded. #28. + streamFailed := logErr != nil && + !errors.Is(logErr, context.Canceled) && + !errors.Is(logErr, context.DeadlineExceeded) + if streamFailed { + // SIGINT during the final-status poll → graceful detach. + if errors.Is(customerCtx.Err(), context.Canceled) { + return &WatchResult{ + Outcome: JobOutcomeDetached, + PodName: podName, + Summary: summary, + DetachReason: DetachReasonSignal, + }, nil + } + // Job reached a terminal state → the stream error was + // incidental (Pod replaced/restarted). Report the real + // outcome the customer cares about. + if statusErr == nil && (outcome == JobOutcomeSucceeded || outcome == JobOutcomeFailed) { + return &WatchResult{Outcome: outcome, PodName: podName, Summary: summary}, nil + } + // Couldn't confirm a terminal Job state → the stream failure + // is the actionable signal; surface it. + return nil, fmt.Errorf("streaming logs from Pod %s/%s: %w", namespace, podName, logErr) + } + + // 5. Clean-stream path: classify on the Job status alone. + if statusErr != nil { + // Treat SIGINT during finalJobStatus as graceful detach + // (jobs-manager already accepted the run; the customer is + // just stopping the observation). Bugbot PR #10 r2 flagged + // the "exit 9 on post-stream SIGINT" inconsistency. + if errors.Is(customerCtx.Err(), context.Canceled) { + return &WatchResult{ + Outcome: JobOutcomeDetached, + PodName: podName, + Summary: summary, + DetachReason: DetachReasonSignal, + }, nil + } + return nil, fmt.Errorf("reading final Job status for %s/%s: %w", namespace, jobName, statusErr) + } + return &WatchResult{ + Outcome: outcome, + PodName: podName, + Summary: summary, + }, nil +} + +// waitForJobPod polls until the Job has spawned its Pod and that +// Pod has reached Phase=Running. The selection key is the +// `job-name=` label that batch/v1 controllers attach to +// every Pod they create. +func waitForJobPod(ctx context.Context, cs kubernetes.Interface, namespace, jobName string) (string, error) { + var podName string + err := wait.PollUntilContextTimeout(ctx, PodPollInterval, PodReadyTimeout, true, + func(ctx context.Context) (bool, error) { + pods, err := cs.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: "job-name=" + jobName, + }) + if err != nil { + // Terminal errors short-circuit (echoes the + // Phase-3 r3 fix in push/pod.go). + if apierrors.IsForbidden(err) { + return false, err + } + return false, nil // transient + } + if len(pods.Items) == 0 { + return false, nil // Pod hasn't been created yet + } + // Pick the MOST RECENT useful-phase Pod, not just + // items[0]. A Job with backoffLimit > 0 (or a Job + // where jobs-manager re-spawned the Pod for any + // reason) can have multiple Pods bearing the same + // `job-name=` label. The List API doesn't + // guarantee order, so items[0] could be the old + // Failed Pod from a prior retry instead of the + // current Running one. Bugbot PR #10 r4 caught this. + // + // "Useful phase" = Running (happy path) | Succeeded + // (fast-completing ingestion we missed) | Failed + // (terminated; we still want its logs). Pending Pods + // don't count — they have no logs to stream yet, so + // we keep polling until they either transition or + // become irrelevant. + var bestPod *corev1.Pod + for i := range pods.Items { + p := &pods.Items[i] + switch p.Status.Phase { + case corev1.PodRunning, corev1.PodSucceeded, corev1.PodFailed: + if bestPod == nil || + p.CreationTimestamp.After(bestPod.CreationTimestamp.Time) { + bestPod = p + } + } + } + if bestPod == nil { + return false, nil // all Pods still Pending + } + podName = bestPod.Name + return true, nil + }) + if err != nil { + return "", err + } + return podName, nil +} + +// streamPodLogsAndParse opens a streaming log read on the Pod and +// pipes it through (a) a TeeReader to `out` for verbatim display +// and (b) a Summary parser for the 📊 banner. Returns the parsed +// Summary (or nil if the Pod never produced one) + the underlying +// stream error. +// +// Streaming model: Follow=true means the API server keeps the +// connection open until the Pod terminates. When the Pod's +// container exits, the stream returns EOF and we drop out. This +// avoids the "poll twice" anti-pattern where Phase 4 would have to +// re-fetch logs after the Job is done to see the summary. +func streamPodLogsAndParse( + ctx context.Context, + cs kubernetes.Interface, + namespace, podName string, + out io.Writer, +) (*Summary, error) { + req := cs.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{ + Follow: true, + // Container omitted — Job Pods have exactly one container + // (the ingestor). If a future ingestor adds a sidecar + // (e.g. for metrics scrape), this needs to specify + // `Container: "ingestor"`. + }) + stream, err := req.Stream(ctx) + if err != nil { + return nil, err + } + defer func() { _ = stream.Close() }() + + // Wrap the stream in a TeeReader so each line flows through + // both customer-facing output AND the summary parser. The + // parser keeps a small ring buffer internally; it doesn't + // need to see the full stream in memory. + parser := NewSummaryParser() + tee := io.TeeReader(stream, parserWriter{parser: parser}) + + // Line-by-line copy so the customer sees output progressively. + // io.Copy would also work but would buffer chunks at the + // transport layer, making the output feel laggy on a fast + // ingestion. + scanner := bufio.NewScanner(tee) + // Default scanner buffer is 64 KB per line — fine for log + // lines but bump to 1 MB to handle the (rare) case where a + // single ingestion-error stacktrace has a multi-KB Python + // traceback line. + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024) + for scanner.Scan() { + line := scanner.Bytes() + // Print the line + a newline (scanner strips the trailing + // '\n'). errcheck-friendly: we discard the writer error + // because the exit code is the customer-facing contract. + _, _ = out.Write(line) + _, _ = out.Write([]byte("\n")) + } + if err := scanner.Err(); err != nil { + // EOF is normal end-of-stream; other errors (network drop + // mid-stream, ctx cancel) get propagated. + if !errors.Is(err, io.EOF) { + // Flush any buffered partial line before returning so + // the parser sees content even on mid-line failure. + parser.FlushLine() + return parser.Result(), err + } + } + // Flush at the end of stream too. A Pod that exited without + // a trailing newline on its final stdout write would otherwise + // lose the final line — including potentially the closing + // ═-rule that finalizes the banner. Bugbot flagged on PR #10. + parser.FlushLine() + return parser.Result(), nil +} + +// parserWriter adapts a SummaryParser into an io.Writer for use +// with io.TeeReader. The TeeReader writes everything to its +// secondary sink as bytes flow through; the parser's Feed method +// accepts them line-by-line internally. +type parserWriter struct { + parser *SummaryParser +} + +func (pw parserWriter) Write(b []byte) (int, error) { + pw.parser.Feed(b) + return len(b), nil +} + +// finalJobStatus does a bounded poll on the Job's status to +// determine Succeeded vs Failed after log streaming ends. This is +// a separate step because the log-stream-end doesn't always race +// the Job-status-update; we need to wait briefly for the +// apiserver to post the terminal phase. +func finalJobStatus(ctx context.Context, cs kubernetes.Interface, namespace, jobName string) (JobOutcome, error) { + var outcome JobOutcome + err := wait.PollUntilContextTimeout(ctx, JobPollInterval, 30*time.Second, true, + func(ctx context.Context) (bool, error) { + job, err := cs.BatchV1().Jobs(namespace).Get(ctx, jobName, metav1.GetOptions{}) + if err != nil { + if apierrors.IsForbidden(err) || apierrors.IsNotFound(err) { + return false, err + } + return false, nil + } + // batch/v1 Job conditions: Complete (success) or + // Failed. Poll until one is set; in practice this + // resolves within ~1s of log stream EOF. + for _, c := range job.Status.Conditions { + if c.Status != corev1.ConditionTrue { + continue + } + switch c.Type { + case batchv1.JobComplete: + outcome = JobOutcomeSucceeded + return true, nil + case batchv1.JobFailed: + outcome = JobOutcomeFailed + return true, nil + } + } + return false, nil + }) + if err != nil { + // If the poll timed out without seeing a terminal + // condition, the apiserver is being slow. Treat as + // Unknown rather than failing the whole push — the + // orchestrator can render a useful diagnostic from the + // streamed logs. + if errors.Is(err, context.DeadlineExceeded) { + return JobOutcomeUnknown, nil + } + return JobOutcomeUnknown, err + } + return outcome, nil +} diff --git a/internal/submit/watch_test.go b/internal/submit/watch_test.go new file mode 100644 index 0000000..4da6fb9 --- /dev/null +++ b/internal/submit/watch_test.go @@ -0,0 +1,355 @@ +package submit + +import ( + "bytes" + "context" + "errors" + "strings" + "testing" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" +) + +// jobPod constructs a Pod owned by `jobName` via the standard +// batch/v1 job-name label. Used to seed the fake clientset for +// waitForJobPod tests. +func jobPod(name, jobName string, phase corev1.PodPhase) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "tracebloc", + Labels: map[string]string{"job-name": jobName}, + }, + Status: corev1.PodStatus{Phase: phase}, + } +} + +// TestWaitForJobPod_RunningPodSurfaces: a Pod with job-name label +// in Phase=Running is returned. Pin the label-selector contract + +// the happy-path return. +func TestWaitForJobPod_RunningPodSurfaces(t *testing.T) { + cs := fake.NewClientset(jobPod("ingestor-abc-xyz", "ingestor-abc", corev1.PodRunning)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + name, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor-abc") + if err != nil { + t.Fatalf("waitForJobPod: %v", err) + } + if name != "ingestor-abc-xyz" { + t.Errorf("name = %q, want ingestor-abc-xyz", name) + } +} + +// TestWaitForJobPod_PicksMostRecentNotFirst: Jobs with retries +// (or any multi-Pod scenario) produce multiple Pods with the +// same `job-name=` label. The List API doesn't guarantee +// order — picking items[0] could grab the old Failed Pod from a +// prior retry while the current Running one waits. Bugbot +// PR #10 r4 caught the missing tie-break. +func TestWaitForJobPod_PicksMostRecentNotFirst(t *testing.T) { + now := time.Now() + older := jobPod("ingestor-old-failed", "ingestor", corev1.PodFailed) + older.CreationTimestamp = metav1.NewTime(now.Add(-10 * time.Minute)) + + newer := jobPod("ingestor-new-running", "ingestor", corev1.PodRunning) + newer.CreationTimestamp = metav1.NewTime(now.Add(-1 * time.Minute)) + + cs := fake.NewClientset(older, newer) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + name, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor") + if err != nil { + t.Fatalf("waitForJobPod: %v", err) + } + if name != "ingestor-new-running" { + t.Errorf("name = %q, want ingestor-new-running "+ + "(most-recent useful-phase Pod, not items[0])", name) + } +} + +// TestWaitForJobPod_AllPendingKeepsPolling: if every Pod is still +// Pending (image pulling, scheduling), the function keeps polling +// rather than returning a Pending Pod's name (Pods with no log +// stream yet aren't useful to attach to). +func TestWaitForJobPod_AllPendingKeepsPolling(t *testing.T) { + cs := fake.NewClientset(jobPod("pending-1", "ingestor", corev1.PodPending)) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor") + if err == nil { + t.Fatal("waitForJobPod returned nil on all-Pending; expected DeadlineExceeded") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("error doesn't wrap DeadlineExceeded: %v", err) + } +} + +// TestWaitForJobPod_FastCompletionPath: ingestions that finish +// faster than the poll interval might be in Phase=Succeeded by +// the time the watch loop checks. We still want the Pod's name so +// we can fetch its (post-mortem) logs. +func TestWaitForJobPod_FastCompletionPath(t *testing.T) { + cs := fake.NewClientset(jobPod("ingestor-fast", "ingestor", corev1.PodSucceeded)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + name, err := waitForJobPod(ctx, cs, "tracebloc", "ingestor") + if err != nil { + t.Fatalf("waitForJobPod on Succeeded: %v", err) + } + if name != "ingestor-fast" { + t.Errorf("name = %q, want ingestor-fast", name) + } +} + +// TestWaitForJobPod_ForbiddenIsTerminal: an RBAC denial on +// `list pods` must short-circuit the wait — otherwise the +// customer sits through PodReadyTimeout for an error that +// doesn't change. +func TestWaitForJobPod_ForbiddenIsTerminal(t *testing.T) { + cs := fake.NewClientset() + cs.PrependReactor("list", "pods", + func(_ k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewForbidden( + corev1.Resource("pods"), "", + errors.New("user cannot list pods")) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + start := time.Now() + _, err := waitForJobPod(ctx, cs, "tracebloc", "j") + elapsed := time.Since(start) + if err == nil { + t.Fatal("waitForJobPod returned nil on Forbidden") + } + if elapsed > 3*time.Second { + t.Errorf("waitForJobPod waited %s on Forbidden; expected immediate return", elapsed) + } +} + +// TestWaitForJobPod_NoPodEverShows: ingestor Job that doesn't +// spawn its Pod (image pull stuck, scheduling impossible) hits +// the PodReadyTimeout. Bound the test's ctx so it doesn't wait +// the full 5min. +func TestWaitForJobPod_NoPodEverShows(t *testing.T) { + cs := fake.NewClientset() // empty + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + _, err := waitForJobPod(ctx, cs, "tracebloc", "missing-job") + if err == nil { + t.Fatal("waitForJobPod returned nil when no Pod ever appeared") + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Errorf("error doesn't wrap DeadlineExceeded: %v", err) + } +} + +// jobWithCondition constructs a batch/v1 Job whose status reports +// the given terminal condition. Used to seed finalJobStatus tests. +func jobWithCondition(name string, cond batchv1.JobConditionType) *batchv1.Job { + return &batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: "tracebloc"}, + Status: batchv1.JobStatus{ + Conditions: []batchv1.JobCondition{{ + Type: cond, + Status: corev1.ConditionTrue, + }}, + }, + } +} + +// TestFinalJobStatus_Complete: Job with Condition=Complete maps +// to JobOutcomeSucceeded. +func TestFinalJobStatus_Complete(t *testing.T) { + cs := fake.NewClientset(jobWithCondition("done", batchv1.JobComplete)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + out, err := finalJobStatus(ctx, cs, "tracebloc", "done") + if err != nil { + t.Fatalf("finalJobStatus: %v", err) + } + if out != JobOutcomeSucceeded { + t.Errorf("Outcome = %v, want Succeeded", out) + } +} + +// TestFinalJobStatus_Failed: Job with Condition=Failed maps to +// JobOutcomeFailed (drives the exit-9 "ingest failed" path). +func TestFinalJobStatus_Failed(t *testing.T) { + cs := fake.NewClientset(jobWithCondition("crashed", batchv1.JobFailed)) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + out, err := finalJobStatus(ctx, cs, "tracebloc", "crashed") + if err != nil { + t.Fatalf("finalJobStatus: %v", err) + } + if out != JobOutcomeFailed { + t.Errorf("Outcome = %v, want Failed", out) + } +} + +// TestFinalJobStatus_TimeoutIsUnknown: if no terminal condition +// posts within 30s of the log stream ending, we return Unknown +// rather than blocking the customer forever. The orchestrator +// renders a useful diagnostic from the streamed logs. +func TestFinalJobStatus_TimeoutIsUnknown(t *testing.T) { + cs := fake.NewClientset(&batchv1.Job{ + ObjectMeta: metav1.ObjectMeta{Name: "stuck", Namespace: "tracebloc"}, + // No conditions — Job in a weird mid-state. + }) + + // Tight ctx so the test doesn't take 30s. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + out, err := finalJobStatus(ctx, cs, "tracebloc", "stuck") + if err != nil { + t.Fatalf("finalJobStatus: %v", err) + } + if out != JobOutcomeUnknown { + t.Errorf("Outcome = %v, want Unknown", out) + } +} + +// TestWatchJob_PodWaitTimeoutMapsToDetach: when waitForJobPod +// exhausts its 5-min budget (slow image pull, PSA backlog), the +// submit has already been accepted by jobs-manager — the +// ingestion will run, the CLI just gave up watching within the +// timeout. WatchJob must return Outcome=Detached, not an error +// that bubbles up as exit 9 ("ingestion failed"). Bugbot PR #10 +// r5 caught the false-positive exit code. +func TestWatchJob_PodWaitTimeoutMapsToDetach(t *testing.T) { + cs := fake.NewClientset() // no Pod backing the job-name + + // Tight ctx so the test doesn't actually wait PodReadyTimeout (5m). + // The DeadlineExceeded from the parent ctx propagates the same + // way the inner PodReadyTimeout would, exercising the same + // errors.Is(DeadlineExceeded) branch in WatchJob. + ctx, cancel := context.WithTimeout(context.Background(), 1500*time.Millisecond) + defer cancel() + + var out bytes.Buffer + wr, err := WatchJob(ctx, cs, "tracebloc", "ingestor-stuck", &out) + if err != nil { + t.Fatalf("WatchJob returned error on Pod-wait timeout; want nil + Detached: %v", err) + } + if wr == nil { + t.Fatal("WatchJob returned nil result on Pod-wait timeout") + } + if wr.Outcome != JobOutcomeDetached { + t.Errorf("Outcome = %v, want Detached (the cluster keeps running; the CLI just gave up observing)", wr.Outcome) + } +} + +// TestWatchJob_TerminalJobStatusWins is the #28 regression pin: the +// Job — not the log stream — is the source of truth for the outcome. +// With a Running Pod and a Job already reporting Complete, WatchJob +// must return Succeeded. This holds whether the (fake) log stream +// yields data or breaks: if the stream errored, the new +// "streamFailed but Job terminal → report outcome" branch still +// resolves to the Job's verdict instead of bubbling exit-9. Before +// the fix, a broken stream (e.g. a Pod replaced by a retry, or +// deleted mid-follow) returned an error even on a successful Job. +func TestWatchJob_TerminalJobStatusWins(t *testing.T) { + cs := fake.NewClientset( + jobPod("ingestor-xyz", "ingestor", corev1.PodRunning), + jobWithCondition("ingestor", batchv1.JobComplete), + ) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var out bytes.Buffer + wr, err := WatchJob(ctx, cs, "tracebloc", "ingestor", &out) + if err != nil { + t.Fatalf("WatchJob returned error; want nil + Succeeded: %v", err) + } + if wr == nil { + t.Fatal("WatchJob returned nil result") + } + if wr.Outcome != JobOutcomeSucceeded { + t.Fatalf("Outcome = %v, want Succeeded (Job condition is the source of truth)", wr.Outcome) + } +} + +// TestWatchJob_TerminalFailedJobReported: the mirror of the above for +// a Failed Job — the watch reports Failed (→ exit 9), not a generic +// watch error, even if the stream broke. +func TestWatchJob_TerminalFailedJobReported(t *testing.T) { + cs := fake.NewClientset( + jobPod("ingestor-xyz", "ingestor", corev1.PodRunning), + jobWithCondition("ingestor", batchv1.JobFailed), + ) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + var out bytes.Buffer + wr, err := WatchJob(ctx, cs, "tracebloc", "ingestor", &out) + if err != nil { + t.Fatalf("WatchJob returned error; want nil + Failed: %v", err) + } + if wr == nil { + t.Fatal("WatchJob returned nil result") + } + if wr.Outcome != JobOutcomeFailed { + t.Fatalf("Outcome = %v, want Failed", wr.Outcome) + } +} + +// TestJobOutcome_String: stringer pin so diagnostic output stays +// stable. +func TestJobOutcome_String(t *testing.T) { + cases := map[JobOutcome]string{ + JobOutcomeSucceeded: "Succeeded", + JobOutcomeFailed: "Failed", + JobOutcomeDetached: "Detached", + JobOutcomeUnknown: "Unknown", + } + for o, want := range cases { + if got := o.String(); got != want { + t.Errorf("%v.String() = %q, want %q", o, got, want) + } + } +} + +// TestParserWriter_FeedsParser: the io.Writer adapter that hooks +// the log-stream TeeReader to the SummaryParser. Pin that writes +// flow through to Feed correctly. +func TestParserWriter_FeedsParser(t *testing.T) { + p := NewSummaryParser() + pw := parserWriter{parser: p} + chunks := strings.Split(realIngestorBanner, "\n") + for _, line := range chunks { + _, _ = pw.Write([]byte(line + "\n")) + } + got := p.Result() + if got == nil { + t.Fatal("parserWriter didn't feed parser; Result is nil") + } + if got.TotalRecords != 1234 { + t.Errorf("TotalRecords = %d, want 1234 (via parserWriter)", got.TotalRecords) + } +} diff --git a/internal/submit/watcherr_test.go b/internal/submit/watcherr_test.go new file mode 100644 index 0000000..4ddad1c --- /dev/null +++ b/internal/submit/watcherr_test.go @@ -0,0 +1,33 @@ +package submit + +import ( + "errors" + "fmt" + "testing" +) + +// TestWatchError_UnwrapAndClassify pins WatchError's wrapping: it +// reports the inner message, unwraps to the cause (so errors.Is works +// through it), and IsWatchError recognizes it while rejecting other +// errors. This drives the orchestrator's exit-9 (ingest-side) vs +// exit-8 (submit-side) mapping, so the classification must be exact. +func TestWatchError_UnwrapAndClassify(t *testing.T) { + inner := errors.New("pod log stream broke") + we := &WatchError{Err: inner} + + if we.Error() != inner.Error() { + t.Errorf("Error() = %q, want %q", we.Error(), inner.Error()) + } + // errors.Is traverses Unwrap — covers WatchError.Unwrap. + if !errors.Is(we, inner) { + t.Error("errors.Is(WatchError, inner) = false; Unwrap not wired") + } + // And through an extra wrap layer. + wrapped := fmt.Errorf("watching ingestor Job: %w", we) + if !IsWatchError(wrapped) { + t.Error("IsWatchError(wrapped WatchError) = false, want true") + } + if IsWatchError(errors.New("unrelated")) { + t.Error("IsWatchError(plain error) = true, want false") + } +} diff --git a/scripts/RELEASE_CHECKLIST.md b/scripts/RELEASE_CHECKLIST.md new file mode 100644 index 0000000..2011431 --- /dev/null +++ b/scripts/RELEASE_CHECKLIST.md @@ -0,0 +1,107 @@ +# Release checklist + +The release workflow is fully automated — pushing a `v*.*.*` tag +triggers it. This document covers the per-release manual steps that +either ARE or AREN'T automated, so the on-call engineer doesn't +have to reverse-engineer the surface area on release day. + +## What runs automatically on `git push origin v0.1.0` + +1. `.github/workflows/release.yml` fires. +2. Matrix build of 5 platform binaries (linux/{amd64,arm64}, + darwin/{amd64,arm64}, windows/amd64). +3. Each binary is signed with cosign keyless OIDC; the per-binary + `.cert` + `.sig` files are produced. +4. SHA256SUMS file is aggregated across the matrix. +5. `install.sh` + `install.ps1` from `scripts/` are staged into + the release. +6. A GitHub Release is created with the tag, auto-generated notes, + and all artifacts attached. `prerelease=true` if the tag + contains a `-` (e.g. `v0.1.0-rc1`). +7. (If the `bump-homebrew-tap` job is enabled — see below) + the Homebrew tap formula in `tracebloc/homebrew-tap` is + updated with the new version + per-platform SHAs. + +## What needs manual setup ONCE (not per-release) + +These steps land outside the `tracebloc/cli` repo and are NOT +on the per-release checklist; they're foundational. + +### Homebrew tap repo + +1. Create `tracebloc/homebrew-tap` (must use the `homebrew-` + prefix for `brew tap tracebloc/tap` to resolve). +2. Add an initial `Formula/tracebloc.rb` (the release workflow + will overwrite this on the first tag — any valid-Ruby + placeholder is fine). +3. Mint a fine-grained PAT with `Contents: read+write` on the + tap repo. Save as `HOMEBREW_TAP_TOKEN` repo secret on + `tracebloc/cli`. +4. Flip `bump-homebrew-tap`'s `if: false` to `if: true` in + `release.yml`. + +### Vanity install URL (deferred) + +`install.tracebloc.io` is the eventual customer-facing URL but +isn't on the v0.1 critical path — the GitHub Release raw URL +(`https://github.com/tracebloc/cli/releases/latest/download/install.sh`) +serves v0.1 customers. Migrating to the vanity URL is a v0.2 +follow-up (DNS + CNAME or Cloudflare worker). + +## Per-release manual steps + +The release-cutter runs these on tag day. + +### 1. Pre-flight + +- [ ] All v0.1 phase tickets closed (`#147-#153`) +- [ ] `develop` is green on CI + Bugbot +- [ ] Local smoke: `go test -race ./...` passes +- [ ] Real EKS smoke: `tracebloc dataset push ./cats-dogs ...` + end-to-end reports the expected row count + +### 2. Tag + push + +```bash +# From develop, with a clean working tree +git tag -a v0.1.0 -m "tracebloc CLI v0.1.0" +git push origin v0.1.0 +``` + +The workflow takes 5-10 minutes. Monitor at +`https://github.com/tracebloc/cli/actions`. + +### 3. Verify the release + +- [ ] All 5 binaries attached to the GitHub Release +- [ ] SHA256SUMS file present +- [ ] install.sh + install.ps1 present +- [ ] Each binary has a `.cert` + `.sig` pair +- [ ] (If tap is set up) the `tracebloc/homebrew-tap` repo has + a new commit bumping `Formula/tracebloc.rb` + +### 4. Sanity-test each install path on a clean host + +```bash +# Linux: +curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh +tracebloc version + +# macOS: +curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh +# OR (if tap is set up): +brew install tracebloc/tap/tracebloc +tracebloc version + +# Windows (PowerShell): +irm https://github.com/tracebloc/cli/releases/latest/download/install.ps1 | iex +tracebloc version +``` + +### 5. Announce + +- [ ] Bump `images.ingestor.digest` in `tracebloc/client` if a new + ingestor release is coupled to this CLI release +- [ ] Update `README.md` install instructions to reference the new + version (or leave at `latest` — preferred) +- [ ] Post in the team channel + customer Slack diff --git a/scripts/homebrew-formula.rb.tmpl b/scripts/homebrew-formula.rb.tmpl new file mode 100644 index 0000000..b755b6a --- /dev/null +++ b/scripts/homebrew-formula.rb.tmpl @@ -0,0 +1,63 @@ +# Homebrew formula template for the tracebloc CLI. +# +# The release workflow (release.yml > bump-homebrew-tap) renders this +# template per release by substituting: +# +# __VERSION__ → the bare semver (e.g. "0.1.0") +# __TAG__ → the full tag (e.g. "v0.1.0") +# __SHA_DARWIN_AMD64__ → SHA256 of the darwin/amd64 binary +# __SHA_DARWIN_ARM64__ → SHA256 of the darwin/arm64 binary +# __SHA_LINUX_AMD64__ → SHA256 of the linux/amd64 binary +# __SHA_LINUX_ARM64__ → SHA256 of the linux/arm64 binary +# +# The rendered file is committed to tracebloc/homebrew-tap as +# Formula/tracebloc.rb. Customers install via: +# +# brew install tracebloc/tap/tracebloc +# +# Brew's `tap/tracebloc` shorthand maps to github.com/tracebloc/homebrew-tap +# automatically (the `homebrew-` prefix is brew convention). +class Tracebloc < Formula + desc "Declarative data ingestion CLI for the tracebloc platform" + homepage "https://github.com/tracebloc/cli" + version "__VERSION__" + license "Apache-2.0" + + # Per-platform bottle: brew picks the right one for the host OS+arch + # automatically. We ship pre-built binaries (not source) because the + # CLI's Go module pulls ~80MB of k8s.io transitive deps — source + # builds would be uncomfortably slow on customer laptops. + on_macos do + if Hardware::CPU.arm? + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-darwin-arm64" + sha256 "__SHA_DARWIN_ARM64__" + else + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-darwin-amd64" + sha256 "__SHA_DARWIN_AMD64__" + end + end + + on_linux do + if Hardware::CPU.arm? + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-linux-arm64" + sha256 "__SHA_LINUX_ARM64__" + else + url "https://github.com/tracebloc/cli/releases/download/__TAG__/tracebloc-__TAG__-linux-amd64" + sha256 "__SHA_LINUX_AMD64__" + end + end + + def install + # The release artifact is a bare binary, not an archive. Just + # rename it to "tracebloc" and drop it into HOMEBREW_PREFIX/bin. + bin.install Dir["*"].first => "tracebloc" + end + + test do + # Sanity smoke: brew runs this on every install if --with-test is + # set. The version subcommand exits 0 and prints SOMETHING with + # the version in it — that's enough to confirm the binary's not + # corrupted. + assert_match version.to_s, shell_output("#{bin}/tracebloc version") + end +end diff --git a/scripts/install.ps1 b/scripts/install.ps1 new file mode 100644 index 0000000..4b41768 --- /dev/null +++ b/scripts/install.ps1 @@ -0,0 +1,221 @@ +# tracebloc CLI installer for Windows (PowerShell 5.1+). +# +# Usage: +# irm https://github.com/tracebloc/cli/releases/latest/download/install.ps1 | iex +# # Or pin a version: +# $env:RELEASE_VERSION='v0.1.0'; irm | iex +# +# What it does: +# 1. Detects arch (amd64 only on Windows; arm64 not yet shipped) +# 2. Resolves the latest release tag (or honors $env:RELEASE_VERSION) +# 3. Downloads tracebloc--windows-amd64.exe + SHA256SUMS +# 4. Verifies SHA256 +# 5. (Optional) Verifies cosign signature if cosign.exe is on PATH +# 6. Installs to $env:USERPROFILE\AppData\Local\Programs\tracebloc\tracebloc.exe +# and PATH-adds it via user-scope env var +# +# PowerShell 5.1 is the floor (ships with Windows 10 21H1+). PS7+ also +# works. Older PS versions miss `Invoke-WebRequest -UseBasicParsing`'s +# default, but the script forces it. + +# Strict mode + halt on errors — the customer sees a clear error if +# anything fails, not a half-installed binary. +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +# Fail prints a clean error message + exits. With Stop preference, +# Write-Error throws a terminating error BEFORE any subsequent exit +# runs, surfacing as a verbose PowerShell error record with stack +# trace + line numbers — confusing for a customer-facing installer. +# Bugbot PR #11 r3 flagged the original Write-Error + exit 1 +# patterns as both dead code (exit never reached) AND ugly UX. +# Fail's Write-Host + explicit exit produces a one-line red error +# the customer can actually act on. +function Fail([string]$msg) { + Write-Host "Error: $msg" -ForegroundColor Red + exit 1 +} + +# --------------------------------------------------------------------- +# Config knobs (env-overridable, mirrors install.sh). +# --------------------------------------------------------------------- +$ReleaseVersion = if ($env:RELEASE_VERSION) { $env:RELEASE_VERSION } else { 'latest' } +$InstallPrefix = if ($env:INSTALL_PREFIX) { $env:INSTALL_PREFIX } ` + else { Join-Path $env:LOCALAPPDATA 'Programs\tracebloc' } +$GitHubRepo = 'tracebloc/cli' +$BinaryName = 'tracebloc.exe' + +# --------------------------------------------------------------------- +# Detect arch. +# --------------------------------------------------------------------- +function Get-Arch { + # PROCESSOR_ARCHITECTURE is the canonical Windows arch env var: + # AMD64 → x64 binary needed + # ARM64 → Windows on ARM (not yet shipped — see below) + $proc = $env:PROCESSOR_ARCHITECTURE + switch ($proc) { + 'AMD64' { return 'amd64' } + 'ARM64' { + Fail "Windows ARM64 binary isn't released yet. File an issue at https://github.com/tracebloc/cli if you need it." + } + default { + Fail "Unsupported PROCESSOR_ARCHITECTURE: $proc" + } + } +} + +$arch = Get-Arch + +# --------------------------------------------------------------------- +# Resolve the release tag if "latest". +# --------------------------------------------------------------------- +function Resolve-Tag { + if ($script:ReleaseVersion -ne 'latest') { + return $script:ReleaseVersion + } + # Follow the /releases/latest redirect to find the tag, same trick + # install.sh uses. -MaximumRedirection 0 makes Invoke-WebRequest + # surface the Location header instead of following. + try { + $resp = Invoke-WebRequest ` + -Uri "https://github.com/$script:GitHubRepo/releases/latest" ` + -MaximumRedirection 0 ` + -UseBasicParsing ` + -ErrorAction SilentlyContinue + } catch { + # PowerShell treats 3xx as an error when MaximumRedirection=0. + # The Exception.Response carries the redirect we want. + $resp = $_.Exception.Response + } + $loc = $null + if ($resp -and $resp.Headers) { + # PS5.1: Headers is a Dictionary[string,string]; PS7: a + # HttpResponseHeaders that needs different access. Try both. + try { $loc = $resp.Headers['Location'] } catch {} + if (-not $loc) { try { $loc = $resp.Headers.Location } catch {} } + } + if (-not $loc) { + Fail "Couldn't resolve the 'latest' release tag from GitHub. Pass `$env:RELEASE_VERSION explicitly." + } + # Location: https://github.com/tracebloc/cli/releases/tag/vX.Y.Z + return Split-Path -Leaf $loc +} + +$tag = Resolve-Tag +Write-Host "Installing tracebloc CLI $tag (windows/$arch)..." + +# --------------------------------------------------------------------- +# Download artifacts. +# --------------------------------------------------------------------- +$binaryFile = "tracebloc-$tag-windows-$arch.exe" +$baseUrl = "https://github.com/$GitHubRepo/releases/download/$tag" +$tmpDir = New-Item -ItemType Directory -Path (Join-Path $env:TEMP "tracebloc-install-$tag-$([guid]::NewGuid())") -Force + +try { + Write-Host "Downloading binary..." + Invoke-WebRequest -Uri "$baseUrl/$binaryFile" -OutFile (Join-Path $tmpDir $binaryFile) -UseBasicParsing + + Write-Host "Downloading SHA256SUMS..." + Invoke-WebRequest -Uri "$baseUrl/SHA256SUMS" -OutFile (Join-Path $tmpDir 'SHA256SUMS') -UseBasicParsing + + # ------------------------------------------------------------- + # Verify SHA256. + # ------------------------------------------------------------- + Write-Host "Verifying SHA256..." + $sumsContent = Get-Content (Join-Path $tmpDir 'SHA256SUMS') + $expected = ($sumsContent | Where-Object { $_ -match " $([regex]::Escape($binaryFile))$" } | + Select-Object -First 1) -replace ' .*$','' + if (-not $expected) { + Fail "SHA256SUMS doesn't contain an entry for $binaryFile — release artifacts may be incomplete." + } + $actual = (Get-FileHash -Algorithm SHA256 -Path (Join-Path $tmpDir $binaryFile)).Hash.ToLower() + if ($actual -ne $expected) { + Fail "SHA256 mismatch!`n expected: $expected`n actual: $actual`n refusing to install." + } + Write-Host " ✓ checksum matches" + + # ------------------------------------------------------------- + # Cosign signature verification (optional). + # ------------------------------------------------------------- + if (Get-Command cosign -ErrorAction SilentlyContinue) { + Write-Host "Verifying cosign signature..." + # Separate "download .sig/.cert" (recoverable if absent — old + # releases predate signing) from "verify the downloaded sig" + # (NOT recoverable — a failed verification means the binary + # is potentially tampered, refuse to install). Bugbot PR #11 + # caught the prior structure: with $ErrorActionPreference = + # 'Stop', Write-Error inside the try-block was thrown and + # caught by the same catch that handled missing-sig, so a + # failed verify silently downgraded to "skip + continue." + $sigDownloaded = $false + try { + Invoke-WebRequest -Uri "$baseUrl/$binaryFile.sig" -OutFile (Join-Path $tmpDir "$binaryFile.sig") -UseBasicParsing + Invoke-WebRequest -Uri "$baseUrl/$binaryFile.cert" -OutFile (Join-Path $tmpDir "$binaryFile.cert") -UseBasicParsing + $sigDownloaded = $true + } catch { + Write-Host " ⚠ couldn't download .sig/.cert — release may pre-date signing." + } + if ($sigDownloaded) { + # Verify OUTSIDE the try/catch: a non-zero $LASTEXITCODE + # from cosign is a hard refusal, not a swallowed + # exception. & invokes cosign as an external process, + # which doesn't interact with $ErrorActionPreference. + & cosign verify-blob ` + --certificate-identity-regexp "https://github.com/$GitHubRepo/.github/workflows/release.yml@.*" ` + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' ` + --certificate (Join-Path $tmpDir "$binaryFile.cert") ` + --signature (Join-Path $tmpDir "$binaryFile.sig") ` + (Join-Path $tmpDir $binaryFile) 2>$null + if ($LASTEXITCODE -ne 0) { + Write-Host "Error: cosign signature verification FAILED — refusing to install." -ForegroundColor Red + exit 1 + } + Write-Host " ✓ cosign signature valid" + } + } else { + Write-Host " (cosign not installed; SHA256 verified, signature skipped)" + } + + # ------------------------------------------------------------- + # Install to $InstallPrefix. + # ------------------------------------------------------------- + New-Item -ItemType Directory -Path $InstallPrefix -Force | Out-Null + $target = Join-Path $InstallPrefix $BinaryName + Move-Item -Path (Join-Path $tmpDir $binaryFile) -Destination $target -Force + + Write-Host "" + Write-Host "✓ tracebloc CLI installed: $target" + Write-Host "" + Write-Host "Verify with:" + Write-Host " $target version" + + # PATH advice. User-scope PATH edit so this survives reboots. + # + # Null-guard the existing $userPath before concatenation. On + # fresh Windows installs (or accounts that never set user-scope + # PATH), GetEnvironmentVariable returns $null. The naive + # `"$userPath;$InstallPrefix"` interpolation would then produce + # `";C:\..."` — a leading semicolon = empty PATH entry, which on + # Windows resolves to the CURRENT WORKING DIRECTORY. That's a + # well-known PATH-injection vector (binary planted in cwd runs + # ahead of real ones). Bugbot PR #11 r3 flagged the security + # concern. + $userPath = [Environment]::GetEnvironmentVariable('Path', 'User') + $existingEntries = if ($userPath) { $userPath -split ';' } else { @() } + if ($existingEntries -notcontains $InstallPrefix) { + Write-Host "" + Write-Host "Note: $InstallPrefix is not on `$env:Path. Adding it for your user:" + $newPath = if ($userPath) { "$userPath;$InstallPrefix" } else { $InstallPrefix } + [Environment]::SetEnvironmentVariable('Path', $newPath, 'User') + Write-Host " (open a new PowerShell window to pick up the change)" + } + + Write-Host "" + Write-Host "First steps:" + Write-Host " tracebloc cluster info # confirm CLI can reach your cluster" + Write-Host " tracebloc dataset push --help # see the dominant flow" +} +finally { + # Always clean up the temp dir, even on early exit / Ctrl-C. + Remove-Item -Recurse -Force -Path $tmpDir -ErrorAction SilentlyContinue +} diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 0000000..d2a194e --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env sh +# tracebloc CLI installer for Linux + macOS. +# +# Usage: +# curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh +# curl -fsSL https://github.com/tracebloc/cli/releases/latest/download/install.sh | sh -s -- --version v0.1.0 +# +# What it does: +# 1. Detects OS (linux/darwin) + arch (amd64/arm64) of the host +# 2. Resolves the latest release tag (or honors --version) +# 3. Downloads tracebloc--- from the GitHub Release +# 4. Verifies SHA256 against the release's SHA256SUMS file +# 5. (Optional) Verifies cosign signature if cosign is on PATH +# 6. Installs to /usr/local/bin/tracebloc (falls back to $HOME/.local/bin +# with PATH advice if /usr/local/bin isn't writable) +# +# Why /bin/sh + POSIX-only constructs: +# The customer's distro might not have bash. /bin/sh is POSIX-mandated. +# No bashisms (no [[ ]], no <(), no ${var/...}). Tested against dash, +# busybox sh, and bash. + +set -eu + +# -------------------------------------------------------------------- +# Configuration knobs (override via env or args). +# -------------------------------------------------------------------- +INSTALL_PREFIX="${INSTALL_PREFIX:-/usr/local/bin}" +RELEASE_VERSION="${RELEASE_VERSION:-latest}" +GITHUB_REPO="tracebloc/cli" +BINARY_NAME="tracebloc" + +usage() { + cat <] [--prefix ] [--help] + +Options: + --version Install a specific version (e.g. v0.1.0). Default: latest. + --prefix Install directory. Default: /usr/local/bin (falls back to + \$HOME/.local/bin if not writable). + --help Show this help. + +Environment overrides: + RELEASE_VERSION Same as --version. + INSTALL_PREFIX Same as --prefix. +EOF +} + +# -------------------------------------------------------------------- +# Arg parsing — minimal POSIX-shell loop, no getopt (not portable). +# -------------------------------------------------------------------- +while [ $# -gt 0 ]; do + case "$1" in + --version) + RELEASE_VERSION="$2" + shift 2 + ;; + --prefix) + INSTALL_PREFIX="$2" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Error: unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +# -------------------------------------------------------------------- +# Detect OS + arch. +# -------------------------------------------------------------------- +detect_os() { + uname_s="$(uname -s)" + case "$uname_s" in + Linux) echo "linux" ;; + Darwin) echo "darwin" ;; + *) + echo "Error: unsupported OS: $uname_s" >&2 + echo "tracebloc CLI is released for linux + darwin via this script;" >&2 + echo "Windows users can run install.ps1 from a PowerShell prompt." >&2 + exit 1 + ;; + esac +} + +detect_arch() { + uname_m="$(uname -m)" + case "$uname_m" in + x86_64|amd64) echo "amd64" ;; + arm64|aarch64) echo "arm64" ;; + *) + echo "Error: unsupported arch: $uname_m" >&2 + echo "tracebloc CLI is released for amd64 + arm64; if you need" >&2 + echo "another arch, please file an issue at github.com/tracebloc/cli." >&2 + exit 1 + ;; + esac +} + +OS="$(detect_os)" +ARCH="$(detect_arch)" + +# -------------------------------------------------------------------- +# Resolve the release tag if "latest". +# -------------------------------------------------------------------- +resolve_tag() { + if [ "$RELEASE_VERSION" != "latest" ]; then + echo "$RELEASE_VERSION" + return + fi + # Use the redirect-trail of /releases/latest to learn the tag — + # avoids hitting the rate-limited /api/repos endpoint for the + # zero-auth one-liner case. + redirect_url="$(curl -fsSI \ + "https://github.com/${GITHUB_REPO}/releases/latest" \ + | awk '/^[Ll]ocation:/ { print $2 }' \ + | tr -d '\r')" + if [ -z "$redirect_url" ]; then + echo "Error: couldn't resolve the 'latest' release tag from GitHub." >&2 + echo "Pass --version to install a specific release." >&2 + exit 1 + fi + # The redirect URL ends in /tag/; basename gives us the tag. + basename "$redirect_url" +} + +TAG="$(resolve_tag)" +echo "Installing tracebloc CLI $TAG ($OS/$ARCH)..." + +# -------------------------------------------------------------------- +# Download binary + SHA256SUMS + (optional) cosign sig/cert. +# -------------------------------------------------------------------- +BINARY_FILE="${BINARY_NAME}-${TAG}-${OS}-${ARCH}" +BASE_URL="https://github.com/${GITHUB_REPO}/releases/download/${TAG}" + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT INT TERM + +echo "Downloading binary..." +if ! curl -fsSL "$BASE_URL/$BINARY_FILE" -o "$TMP/$BINARY_FILE"; then + echo "Error: failed to download $BASE_URL/$BINARY_FILE" >&2 + exit 1 +fi + +echo "Downloading SHA256SUMS..." +if ! curl -fsSL "$BASE_URL/SHA256SUMS" -o "$TMP/SHA256SUMS"; then + echo "Error: failed to download SHA256SUMS — release may be malformed" >&2 + exit 1 +fi + +# -------------------------------------------------------------------- +# Verify SHA256. +# -------------------------------------------------------------------- +echo "Verifying SHA256..." +expected="$(grep " $BINARY_FILE$" "$TMP/SHA256SUMS" | awk '{print $1}')" +if [ -z "$expected" ]; then + echo "Error: SHA256SUMS doesn't contain an entry for $BINARY_FILE" >&2 + echo " — release artifacts may be incomplete." >&2 + exit 1 +fi +# sha256sum (GNU coreutils) vs shasum -a 256 (macOS): detect which is on PATH. +# If neither is available, refuse to install — running an unverified +# binary from the internet is exactly what this script exists to +# prevent. Bugbot PR #11 caught the previous "warn + continue + still +# print ✓ matches" branch as both a security regression AND a +# misleading-log issue. Almost every modern Linux distro ships coreutils +# (sha256sum) by default; macOS ships /usr/bin/shasum as part of the +# base Perl install. A host with neither is unusual enough that +# erroring out is the right call — the customer can install coreutils +# / xcode-select / similar and re-run. +if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$TMP/$BINARY_FILE" | awk '{print $1}')" +elif command -v shasum >/dev/null 2>&1; then + actual="$(shasum -a 256 "$TMP/$BINARY_FILE" | awk '{print $1}')" +else + echo "Error: neither sha256sum nor shasum is on PATH — can't verify the" >&2 + echo " downloaded binary's integrity. Install one of:" >&2 + echo " apt install coreutils # Debian/Ubuntu" >&2 + echo " dnf install coreutils # Fedora/RHEL" >&2 + echo " apk add coreutils # Alpine" >&2 + echo " (macOS ships /usr/bin/shasum by default — PATH issue?)" >&2 + echo " and re-run." >&2 + exit 1 +fi +if [ "$actual" != "$expected" ]; then + echo "Error: SHA256 mismatch!" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + echo " refusing to install." >&2 + exit 1 +fi +echo " ✓ checksum matches" + +# -------------------------------------------------------------------- +# Verify cosign signature if cosign is on PATH (optional). +# -------------------------------------------------------------------- +if command -v cosign >/dev/null 2>&1; then + echo "Verifying cosign signature..." + if ! curl -fsSL "$BASE_URL/$BINARY_FILE.sig" -o "$TMP/$BINARY_FILE.sig"; then + echo " ⚠ couldn't download .sig — release may pre-date signing." >&2 + elif ! curl -fsSL "$BASE_URL/$BINARY_FILE.cert" -o "$TMP/$BINARY_FILE.cert"; then + echo " ⚠ couldn't download .cert — release may pre-date signing." >&2 + elif ! cosign verify-blob \ + --certificate-identity-regexp \ + "https://github.com/${GITHUB_REPO}/.github/workflows/release.yml@.*" \ + --certificate-oidc-issuer 'https://token.actions.githubusercontent.com' \ + --certificate "$TMP/$BINARY_FILE.cert" \ + --signature "$TMP/$BINARY_FILE.sig" \ + "$TMP/$BINARY_FILE" 2>/dev/null; then + echo "Error: cosign signature verification FAILED — refusing to install." >&2 + exit 1 + else + echo " ✓ cosign signature valid" + fi +else + # Not having cosign isn't fatal — the SHA256 check above is the + # baseline. Recommend cosign for higher-trust installs. + echo " (cosign not installed; SHA256 verified, signature skipped)" +fi + +# -------------------------------------------------------------------- +# Install to a writable prefix. +# +# Bugbot PR #11 r2 caught a UX bug in the previous flow: POSIX +# `test -w` is false for paths that don't exist, so a custom +# --prefix /opt/tracebloc (legitimate, just not created yet) was +# silently overridden with the ~/.local/bin fallback. The right +# semantic: try to mkdir the customer's chosen prefix; if THAT +# fails (no write perms on parent), THEN fall back. +# -------------------------------------------------------------------- +PREFIX="$INSTALL_PREFIX" +if ! mkdir -p "$PREFIX" 2>/dev/null || [ ! -w "$PREFIX" ]; then + # The customer's chosen prefix isn't usable (no write perms on + # parent, /usr/local/bin without sudo, etc.). Fall back to a + # per-user dir; the PATH-advice block below tells them how to + # pick it up. + FALLBACK="$HOME/.local/bin" + echo "Note: $PREFIX isn't writable (couldn't mkdir or no -w); falling back to $FALLBACK" + mkdir -p "$FALLBACK" + PREFIX="$FALLBACK" +fi + +chmod +x "$TMP/$BINARY_FILE" +mv "$TMP/$BINARY_FILE" "$PREFIX/$BINARY_NAME" + +echo "" +echo "✓ tracebloc CLI installed: $PREFIX/$BINARY_NAME" +echo "" +echo "Verify with:" +echo " $PREFIX/$BINARY_NAME version" +echo "" + +# PATH guidance for the fallback case. +case ":$PATH:" in + *":$PREFIX:"*) ;; # already on PATH + *) + echo "Note: $PREFIX is not on \$PATH. Add this to your shell rc file:" + echo " export PATH=\"\$PATH:$PREFIX\"" + echo "" + ;; +esac + +echo "First steps:" +echo " $BINARY_NAME cluster info # confirm CLI can reach your cluster" +echo " $BINARY_NAME dataset push --help # see the dominant flow" diff --git a/scripts/sync-schema.sh b/scripts/sync-schema.sh new file mode 100755 index 0000000..a7f13d4 --- /dev/null +++ b/scripts/sync-schema.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# Sync ingest.v1.json from tracebloc/data-ingestors into the CLI's +# embedded copy at internal/schema/ingest.v1.json. +# +# The CLI validates locally using this schema. Drift between the +# CLI's copy and data-ingestors' canonical source is a real +# correctness hazard — a customer's YAML could pass `tracebloc ingest +# validate` locally but be rejected by jobs-manager (or vice versa). +# +# Run this script when bumping the schema version. CI invokes it in +# check-mode (`--check`) to fail builds that have drifted without +# running the sync first. +# +# Usage: +# scripts/sync-schema.sh # write into internal/schema/ +# scripts/sync-schema.sh --check # verify in-tree copy matches upstream; exit non-zero on drift +# +# Env knobs: +# SCHEMA_SOURCE_URL override the upstream URL (default: data-ingestors' master) +# SCHEMA_OUT override the in-tree destination (default: internal/schema/ingest.v1.json) +# +# Future: when we cut a v2 schema, this script will need to learn +# about multiple versions (e.g. embed v1 AND v2 side-by-side, picked +# at runtime based on the ingest.yaml's apiVersion). Keeping the path +# explicit in SCHEMA_OUT makes that extension easy. + +set -euo pipefail + +readonly DEFAULT_URL="https://raw.githubusercontent.com/tracebloc/data-ingestors/master/tracebloc_ingestor/schema/ingest.v1.json" +readonly DEFAULT_OUT="internal/schema/ingest.v1.json" + +SCHEMA_SOURCE_URL="${SCHEMA_SOURCE_URL:-$DEFAULT_URL}" +SCHEMA_OUT="${SCHEMA_OUT:-$DEFAULT_OUT}" + +CHECK_MODE=false +if [[ "${1:-}" == "--check" ]]; then + CHECK_MODE=true +fi + +# Stage the fetched schema in a temp file so a half-failed curl doesn't +# leave a truncated file in the repo. +tmp=$(mktemp) +trap 'rm -f "$tmp"' EXIT + +echo "==> fetching $SCHEMA_SOURCE_URL" +curl -fsSL "$SCHEMA_SOURCE_URL" -o "$tmp" + +# Make sure what came back is valid JSON before we trust it. +if ! python3 -m json.tool < "$tmp" > /dev/null 2>&1; then + echo "error: upstream response is not valid JSON" >&2 + echo "first 200 bytes of response:" >&2 + head -c 200 "$tmp" >&2 + exit 2 +fi + +mkdir -p "$(dirname "$SCHEMA_OUT")" + +if $CHECK_MODE; then + if [[ ! -f "$SCHEMA_OUT" ]]; then + echo "error: $SCHEMA_OUT does not exist" >&2 + echo "run \`scripts/sync-schema.sh\` (without --check) to seed it." >&2 + exit 1 + fi + if ! diff -q "$tmp" "$SCHEMA_OUT" >/dev/null; then + echo "error: $SCHEMA_OUT has drifted from upstream." >&2 + echo "diff (upstream → in-tree):" >&2 + diff -u "$SCHEMA_OUT" "$tmp" | head -40 >&2 || true + echo >&2 + echo "to fix, run \`scripts/sync-schema.sh\` and commit the result." >&2 + exit 1 + fi + echo "==> $SCHEMA_OUT matches upstream — no drift" + exit 0 +fi + +# Write mode. Only touch the destination if the content actually +# changed, so re-running the script on an already-current schema +# produces no file-mtime churn. +if [[ -f "$SCHEMA_OUT" ]] && diff -q "$tmp" "$SCHEMA_OUT" >/dev/null; then + echo "==> $SCHEMA_OUT already matches upstream — no change" + exit 0 +fi + +mv "$tmp" "$SCHEMA_OUT" +trap - EXIT # the temp file is now in place; don't try to rm it +echo "==> wrote $SCHEMA_OUT ($(wc -c < "$SCHEMA_OUT" | tr -d ' ') bytes)" diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 0000000..b53135b --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,36 @@ +# CLI integration tests + +Build-tagged (`//go:build integration`) tests that run against a **real +Kubernetes cluster** — covering the real-I/O seams the unit suite mocks +out (and where every live bug this project hit actually lived). + +## What they cover +- `cluster.Load` + `cluster.NewClientset` — kubeconfig → live API server. +- `push.SPDYExecutor.Exec` + `push.StreamLayout` — the **tar-over-exec + stream** against a live Pod + PVC (the highest-risk, 0%-unit-covered + path), verified by exec-ing back into the pod to confirm the bytes + landed. + +## Running +```sh +make test-integration # uses your current kubeconfig context +# or: +go test -tags integration -count=1 -v ./test/integration/... +``` +Requires a reachable cluster (kind, k3d, or any dev cluster) with a +default StorageClass and the ability to pull the digest-pinned alpine +stage-pod image. Each test creates a throwaway namespace and cleans up +after itself. + +In CI these run via `.github/workflows/e2e.yml` (kind), gated to +nightly / manual dispatch / `e2e`-labeled PRs (kind boot is too heavy +for every PR). + +## Follow-ups (not yet covered) +- `PortForwardJobsManager` against a live Service (needs a small HTTP + pod + Service fixture). +- A full `dataset push` through a real jobs-manager + ingestor (needs + the `tracebloc/client` chart installed in-cluster) — the + highest-fidelity test; gated on a reproducible in-CI chart install. + This would catch cross-component issues like the jobs-manager↔ingestor + schema skew end-to-end. diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go new file mode 100644 index 0000000..daa34bb --- /dev/null +++ b/test/integration/integration_test.go @@ -0,0 +1,176 @@ +//go:build integration + +// Package integration holds CLI integration tests that run against a +// REAL Kubernetes cluster (kind in CI; any KUBECONFIG-reachable +// cluster locally). They cover the real-I/O seams the mock-based unit +// suite can't reach: kubeconfig→clientset connectivity, and — the big +// one — the SPDYExecutor tar-over-exec stream against a live Pod + PVC +// (internal/push.SPDYExecutor.Exec, 0% in unit coverage). +// +// Run: +// +// make test-integration +// # or: go test -tags integration ./test/integration/ -v +// +// Requires a reachable cluster with a default StorageClass and the +// ability to pull the digest-pinned alpine stage-pod image. Each test +// creates its own throwaway namespace and cleans up after itself. +package integration + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/tracebloc/cli/internal/cluster" + "github.com/tracebloc/cli/internal/push" +) + +// loadCluster builds a clientset + rest.Config from the ambient +// kubeconfig — exercising the real cluster.Load + cluster.NewClientset +// path (NewClientset is 0% in unit coverage). +func loadCluster(t *testing.T) (kubernetes.Interface, *rest.Config) { + t.Helper() + resolved, err := cluster.Load(cluster.KubeconfigOptions{}) + if err != nil { + t.Fatalf("cluster.Load (need a reachable kubeconfig): %v", err) + } + cs, err := cluster.NewClientset(resolved) + if err != nil { + t.Fatalf("cluster.NewClientset: %v", err) + } + return cs, resolved.RestConfig +} + +// TestIntegration_Connectivity proves the kubeconfig→clientset path +// reaches a live API server. +func TestIntegration_Connectivity(t *testing.T) { + cs, _ := loadCluster(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + nss, err := cs.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatalf("list namespaces: %v", err) + } + if len(nss.Items) == 0 { + t.Fatal("cluster reports zero namespaces — unexpected") + } + t.Logf("connected: %d namespaces", len(nss.Items)) +} + +// TestIntegration_StageAndVerify is the core integration test: it +// stages a tiny dataset onto a real PVC via the real SPDYExecutor +// tar-over-exec stream, then exec's back into the pod to prove the +// files actually landed. This is the seam (push.SPDYExecutor.Exec + +// StreamLayout) that has zero unit coverage and where every live bug +// this project hit actually lived. +func TestIntegration_StageAndVerify(t *testing.T) { + cs, restConfig := loadCluster(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + // Throwaway namespace, auto-cleaned (cascades the PVC + pod). + ns, err := cs.CoreV1().Namespaces().Create(ctx, &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{GenerateName: "tb-it-"}, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create namespace: %v", err) + } + nsName := ns.Name + t.Cleanup(func() { + cctx, ccancel := context.WithTimeout(context.Background(), 60*time.Second) + defer ccancel() + _ = cs.CoreV1().Namespaces().Delete(cctx, nsName, metav1.DeleteOptions{}) + }) + + // Shared PVC (RWO + default StorageClass), mirroring the chart's + // client-pvc that the stage pod mounts at /data/shared. + const pvcName = "client-pvc" + const mountPath = "/data/shared" + _, err = cs.CoreV1().PersistentVolumeClaims(nsName).Create(ctx, &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{Name: pvcName}, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}, + Resources: corev1.VolumeResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceStorage: resource.MustParse("1Gi")}, + }, + }, + }, metav1.CreateOptions{}) + if err != nil { + t.Fatalf("create PVC: %v", err) + } + + // A minimal local dataset to stage. + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "labels.csv"), + []byte("filename,label\n001.jpg,cat\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, "images"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "images", "001.jpg"), + []byte("\xff\xd8\xff\xe0-integration-marker"), 0o644); err != nil { + t.Fatal(err) + } + layout, err := push.Discover(dir) + if err != nil { + t.Fatalf("push.Discover: %v", err) + } + + const table = "ittest" + exec := &push.SPDYExecutor{Config: restConfig, Client: cs} + + // Stage pod → Ready → tar-over-exec stream. + podName, err := push.CreateStagePod(ctx, cs, push.PodSpecOptions{ + Namespace: nsName, + PVCClaimName: pvcName, + PVCMountPath: mountPath, + Table: table, + ServiceAccountName: "default", + }) + if err != nil { + t.Fatalf("CreateStagePod: %v", err) + } + defer func() { + dctx, dcancel := context.WithTimeout(context.Background(), 60*time.Second) + defer dcancel() + _ = push.DeleteStagePod(dctx, cs, nsName, podName) + }() + + if _, err := push.WaitForStagePodReady(ctx, cs, nsName, podName); err != nil { + t.Fatalf("WaitForStagePodReady: %v", err) + } + + if err := push.StreamLayout(ctx, exec, nsName, podName, "stage", layout, table, push.NoOpProgress{}); err != nil { + t.Fatalf("StreamLayout (real SPDYExecutor): %v", err) + } + + // Exec back into the still-running pod to prove the bytes landed. + dest := push.StagedPrefix(table) + var stdout, stderr bytes.Buffer + if err := exec.Exec(ctx, nsName, podName, "stage", + []string{"/bin/sh", "-c", fmt.Sprintf("cat %q/labels.csv; echo; ls %q/images", dest, dest)}, + nil, &stdout, &stderr); err != nil { + t.Fatalf("verify exec: %v (stderr: %s)", err, stderr.String()) + } + got := stdout.String() + for _, want := range []string{"filename,label", "001.jpg"} { + if !strings.Contains(got, want) { + t.Errorf("staged content missing %q at %s; got:\n%s", want, dest, got) + } + } + t.Logf("staged + verified at %s:\n%s", dest, got) +} diff --git a/testdata/smoke/invalid-missing-images.yaml b/testdata/smoke/invalid-missing-images.yaml new file mode 100644 index 0000000..75706c6 --- /dev/null +++ b/testdata/smoke/invalid-missing-images.yaml @@ -0,0 +1,12 @@ +# Smoke-test fixture: an image_classification spec missing the +# required `images` directory. `tracebloc ingest validate` must REJECT +# it with exit 2 (schema violation). Guards against a regression where +# the embedded schema or the validator silently stops enforcing +# per-category requirements. +apiVersion: tracebloc.io/v1 +kind: IngestConfig +category: image_classification +table: smoke_test +intent: train +csv: /data/shared/smoke_test/labels.csv +label: label diff --git a/testdata/smoke/valid-image-classification.yaml b/testdata/smoke/valid-image-classification.yaml new file mode 100644 index 0000000..ac803c7 --- /dev/null +++ b/testdata/smoke/valid-image-classification.yaml @@ -0,0 +1,12 @@ +# Smoke-test fixture: a minimal, valid image_classification ingest +# spec. `tracebloc ingest validate` must accept it (exit 0). Used by +# the CI smoke step to exercise the embedded schema + validation wiring +# on the real built binary — no cluster required. +apiVersion: tracebloc.io/v1 +kind: IngestConfig +category: image_classification +table: smoke_test +intent: train +csv: /data/shared/smoke_test/labels.csv +images: /data/shared/smoke_test/images/ +label: label