From 211c4510b2f351c004b98e416bc569e56baa6933 Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Thu, 29 Oct 2026 19:33:11 -0400 Subject: [PATCH 1/3] ci(loadgen): publish buzz-loadgen as ghcr.io/block/buzz-loadgen buzz-loadgen has no image, so it cannot run in a cluster. Add a separate Dockerfile.loadgen (static musl binary on alpine, non-root) and a loadgen-image.yml publisher modeled on sprig-image.yml: native amd64 and arm64 builds pushed by digest, merged into a multi-arch manifest with provenance. Main pushes that touch the loadgen's crates publish :main and :sha-<7>; PRs that touch the Dockerfile or workflow build without pushing. Kept out of the relay image so production relay images don't carry a load tool and test-client-only commits don't count as relay changes. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Luke Tornquist --- .github/workflows/loadgen-image.yml | 220 ++++++++++++++++++++++++++++ Dockerfile.loadgen | 33 +++++ perf/LOADGEN.md | 6 + 3 files changed, 259 insertions(+) create mode 100644 .github/workflows/loadgen-image.yml create mode 100644 Dockerfile.loadgen diff --git a/.github/workflows/loadgen-image.yml b/.github/workflows/loadgen-image.yml new file mode 100644 index 00000000000..21679f5bc16 --- /dev/null +++ b/.github/workflows/loadgen-image.yml @@ -0,0 +1,220 @@ +name: Loadgen image + +# Builds and publishes the relay load generator image as +# ghcr.io/block/buzz-loadgen (see Dockerfile.loadgen and perf/LOADGEN.md). +# +# Strategy mirrors sprig-image.yml: each architecture builds on its native +# runner, pushes to GHCR by digest, then a merge job stitches the per-arch +# digests into one multi-arch manifest and attests provenance. +# +# Triggers: +# - push to main (paths-filtered) → :main + :sha-<7> +# - pull_request (paths-filtered) → build only, no push +# - workflow_dispatch → manual publish at the current ref +# +# NOTE: the first push creates the GHCR package PRIVATE by default. An org +# admin must flip ghcr.io/block/buzz-loadgen to public once (Package settings → +# Change visibility). Subsequent pushes keep the visibility. + +on: + push: + branches: [main] + paths: + - "Dockerfile.loadgen" + - ".github/workflows/loadgen-image.yml" + - "Cargo.toml" + - "Cargo.lock" + - "rust-toolchain.toml" + - "crates/buzz-test-client/**" + - "crates/buzz-ws-client/**" + - "crates/buzz-core/**" + pull_request: + paths: + - "Dockerfile.loadgen" + - ".github/workflows/loadgen-image.yml" + workflow_dispatch: {} + +concurrency: + group: loadgen-image-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: {} + +env: + IMAGE_NAME: ghcr.io/block/buzz-loadgen + +jobs: + build: + name: Build (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + permissions: + contents: read + packages: write + id-token: write + attestations: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-24.04 + arch: amd64 + - platform: linux/arm64 + runner: ubuntu-24.04-arm + arch: arm64 + + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + with: + # Same OOM cap as docker.yml and sprig-image.yml. + buildkitd-config-inline: | + [worker.oci] + max-parallelism = 2 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + with: + images: ${{ env.IMAGE_NAME }} + flavor: | + latest=false + tags: | + type=ref,event=branch + type=sha,prefix=sha-,format=short + labels: | + org.opencontainers.image.title=Buzz Loadgen + org.opencontainers.image.description=Scenario-driven Buzz relay load generator + org.opencontainers.image.licenses=Apache-2.0 + + - name: Build and push by digest + id: build + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + file: ./Dockerfile.loadgen + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=${{ github.event_name != 'pull_request' }} + cache-from: | + type=registry,ref=${{ env.IMAGE_NAME }}-buildcache:${{ matrix.arch }} + cache-to: | + ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && format('type=registry,ref={0}-buildcache:{1},mode=max,compression=zstd', env.IMAGE_NAME, matrix.arch) || '' }} + + - name: Export digest + if: github.event_name != 'pull_request' + env: + DIGEST: ${{ steps.build.outputs.digest }} + run: | + mkdir -p /tmp/digests + touch "/tmp/digests/${DIGEST#sha256:}" + + - name: Upload digest + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: loadgen-digest-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: Merge multi-arch manifest + if: github.event_name != 'pull_request' + runs-on: ubuntu-24.04 + needs: build + timeout-minutes: 15 + permissions: + contents: read + packages: write + id-token: write + attestations: write + + steps: + - name: Download per-arch digests + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: /tmp/digests + pattern: loadgen-digest-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Log in to GHCR + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + with: + images: ${{ env.IMAGE_NAME }} + # Must mirror the build job's tag matrix exactly. + flavor: | + latest=false + tags: | + type=ref,event=branch + type=sha,prefix=sha-,format=short + + - name: Create and push manifest list + id: manifest + working-directory: /tmp/digests + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + META_TAGS: ${{ steps.meta.outputs.tags }} + run: | + set -euo pipefail + tags=() + while IFS= read -r tag; do + [ -n "$tag" ] && tags+=("-t" "$tag") + done <<< "$META_TAGS" + + digests=() + for digest in *; do + digests+=("${IMAGE_NAME}@sha256:${digest}") + done + + docker buildx imagetools create "${tags[@]}" "${digests[@]}" + + first_tag=$(echo "$META_TAGS" | head -n1) + merged_digest=$(docker buildx imagetools inspect "$first_tag" \ + --format '{{json .Manifest}}' | jq -r '.digest') + echo "digest=${merged_digest}" >> "$GITHUB_OUTPUT" + + - name: Attest provenance for the merged image + # Verify with: gh attestation verify oci://ghcr.io/block/buzz-loadgen: --owner block + uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373 # v4.1.1 + with: + subject-name: ${{ env.IMAGE_NAME }} + subject-digest: ${{ steps.manifest.outputs.digest }} + push-to-registry: true + + - name: Summary + env: + IMAGE_NAME: ${{ env.IMAGE_NAME }} + DIGEST: ${{ steps.manifest.outputs.digest }} + run: | + { + echo "### Loadgen image published" + echo '```' + echo "${IMAGE_NAME}@${DIGEST}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/Dockerfile.loadgen b/Dockerfile.loadgen new file mode 100644 index 00000000000..240f40078d7 --- /dev/null +++ b/Dockerfile.loadgen @@ -0,0 +1,33 @@ +# syntax=docker/dockerfile:1.7 +# +# Relay load generator image — published as ghcr.io/block/buzz-loadgen. +# Runs `buzz-loadgen` (crates/buzz-test-client, see perf/LOADGEN.md), for +# example as a Kubernetes Indexed Job with one `run --shard i/n` per pod. +# +# Multi-arch is produced by building this file on native amd64 and arm64 +# runners (see .github/workflows/loadgen-image.yml). Keep both bases pinned to +# manifest-list digests so either architecture resolves to immutable bytes. +FROM rust:1.98-alpine3.22@sha256:a1796ca6fa216d6727b5f61c69e4c665b120b1a4dcb969639e2f25f1ed309456 AS builder + +RUN apk add --no-cache \ + build-base \ + cmake \ + musl-dev \ + perl \ + pkgconf +WORKDIR /build +COPY . . +RUN cargo build --locked --release -p buzz-test-client --bin buzz-loadgen \ + && strip target/release/buzz-loadgen + +FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce + +LABEL org.opencontainers.image.source="https://github.com/block/buzz" + +RUN apk add --no-cache ca-certificates \ + && adduser -D -H loadgen + +COPY --from=builder --chmod=0755 /build/target/release/buzz-loadgen /usr/local/bin/buzz-loadgen + +USER loadgen +ENTRYPOINT ["/usr/local/bin/buzz-loadgen"] diff --git a/perf/LOADGEN.md b/perf/LOADGEN.md index cc26fa5ebd2..214d382df48 100644 --- a/perf/LOADGEN.md +++ b/perf/LOADGEN.md @@ -75,6 +75,12 @@ processes, such as a Kubernetes Indexed Job, with the same scenario. Each shard reports its own clients; add counts and take the worst percentiles across shards. +`ghcr.io/block/buzz-loadgen:main` (and `:sha-<7>`) packages the binary as the +image entrypoint, published by `.github/workflows/loadgen-image.yml` from +`Dockerfile.loadgen`. Mount the scenario and pass the subcommand as the +container arguments, for example `run /etc/loadgen/scenario.toml --shard +$(JOB_COMPLETION_INDEX)/30`. + ## Abort Each step reports every operation from the start of its ramp to the end of From 2c18186a87377debb6252180434bca40efb29912 Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Thu, 29 Oct 2026 19:38:09 -0400 Subject: [PATCH 2/3] ci(loadgen): run the image as numeric UID 10001 USER loadgen is a name, so a pod with runAsNonRoot: true fails to start: the kubelet cannot verify a non-numeric user is non-root. Create the user with a fixed UID/GID and set USER to the numbers. Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Luke Tornquist --- Dockerfile.loadgen | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile.loadgen b/Dockerfile.loadgen index 240f40078d7..94bcdfee07d 100644 --- a/Dockerfile.loadgen +++ b/Dockerfile.loadgen @@ -24,10 +24,12 @@ FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc LABEL org.opencontainers.image.source="https://github.com/block/buzz" +# Numeric UID/GID so Kubernetes runAsNonRoot can verify the user. RUN apk add --no-cache ca-certificates \ - && adduser -D -H loadgen + && addgroup -g 10001 loadgen \ + && adduser -D -H -u 10001 -G loadgen loadgen COPY --from=builder --chmod=0755 /build/target/release/buzz-loadgen /usr/local/bin/buzz-loadgen -USER loadgen +USER 10001:10001 ENTRYPOINT ["/usr/local/bin/buzz-loadgen"] From c24ee76b0378950cb16e21f1fb568468892047c4 Mon Sep 17 00:00:00 2001 From: Luke Tornquist Date: Thu, 29 Oct 2026 19:54:23 -0400 Subject: [PATCH 3/3] ci(loadgen): smoke the binary in the build, cover .cargo/config.toml Review follow-ups. Run `buzz-loadgen --help` after strip in the builder so each native-arch CI build proves the binary starts. Add .cargo/config.toml to the publish path filter since its [env] affects release builds. Note in the workflow that same-repo PRs still write the build cache, and in LOADGEN.md to pin :sha-<7> for comparable runs and point --report at a writable mount (the container has no writable working directory). Co-authored-by: Elrond <28d6302a099e5225b02c4155ac4236e4912603df2ab08dbfc2f4fef08ce598c8@buzz.block.builderlab.xyz> Co-Authored-By: Claude Opus 5.5 Signed-off-by: Luke Tornquist --- .github/workflows/loadgen-image.yml | 4 +++- Dockerfile.loadgen | 4 +++- perf/LOADGEN.md | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/loadgen-image.yml b/.github/workflows/loadgen-image.yml index 21679f5bc16..7b546653467 100644 --- a/.github/workflows/loadgen-image.yml +++ b/.github/workflows/loadgen-image.yml @@ -9,7 +9,8 @@ name: Loadgen image # # Triggers: # - push to main (paths-filtered) → :main + :sha-<7> -# - pull_request (paths-filtered) → build only, no push +# - pull_request (paths-filtered) → build only, no image push (same-repo +# PRs still write the registry build cache, as in sprig-image.yml) # - workflow_dispatch → manual publish at the current ref # # NOTE: the first push creates the GHCR package PRIVATE by default. An org @@ -22,6 +23,7 @@ on: paths: - "Dockerfile.loadgen" - ".github/workflows/loadgen-image.yml" + - ".cargo/config.toml" - "Cargo.toml" - "Cargo.lock" - "rust-toolchain.toml" diff --git a/Dockerfile.loadgen b/Dockerfile.loadgen index 94bcdfee07d..181b28fa9e8 100644 --- a/Dockerfile.loadgen +++ b/Dockerfile.loadgen @@ -17,8 +17,10 @@ RUN apk add --no-cache \ pkgconf WORKDIR /build COPY . . +# --help proves the stripped binary starts on each native architecture. RUN cargo build --locked --release -p buzz-test-client --bin buzz-loadgen \ - && strip target/release/buzz-loadgen + && strip target/release/buzz-loadgen \ + && target/release/buzz-loadgen --help >/dev/null FROM alpine:3.22@sha256:14358309a308569c32bdc37e2e0e9694be33a9d99e68afb0f5ff33cc1f695dce diff --git a/perf/LOADGEN.md b/perf/LOADGEN.md index 214d382df48..4fd6e39a8c2 100644 --- a/perf/LOADGEN.md +++ b/perf/LOADGEN.md @@ -79,7 +79,9 @@ across shards. image entrypoint, published by `.github/workflows/loadgen-image.yml` from `Dockerfile.loadgen`. Mount the scenario and pass the subcommand as the container arguments, for example `run /etc/loadgen/scenario.toml --shard -$(JOB_COMPLETION_INDEX)/30`. +$(JOB_COMPLETION_INDEX)/30`. Pin `:sha-<7>` or a digest for runs you intend +to compare; `:main` moves. The container runs as UID 10001 with `/` as its +working directory and no home, so point `--report` at a writable mount. ## Abort