Make PrimeCaches the only Rust cache writer and remove sccache (#1963)

Makes PrimeCaches the only workflow that writes Rust caches, and removes
sccache.

PR CI hit sccache rate limits (429/503), and caches on main kept growing
until they filled the 10 GB repository quota. rust-cache keeps every
dependency version it restores, so entries saved after builds on
restored caches grow with each lockfile change.

* Remove sccache. The jobs it served (fuzz, non-PIE guests, bench
baseline) get rust-cache entries.
* PrimeCaches looks up each key and builds from a cold target only on a
miss. Every other workflow restores only.
* Scheduled, Miri and Copilot jobs no longer write caches.
* A final PrimeCaches job deletes superseded `v1-rust-*` and `sysroot-*`
entries on main.
* Bump rust-cache to v2.9.3 and start under the `v1-rust` prefix, so the
first run replaces the grown `v0-rust` entries.

A cold PrimeCaches run on this branch saved about 8.6 GB in total. The
largest entries are code-checks (1.4 GB Linux, 1.0 GB Windows) and the
eight host entries (540–620 MB each).

Follow-ups: caches for Miri X64, Windows arm64, macOS and trace fuzz
guests.

Easiest to review commit-by-commit

---------

Signed-off-by: Ludvig Liljenberg <4257730+ludfjig@users.noreply.github.com>

Author: ludfjig

.github/actions/setup-sccache/action.yml

@@ -20,6 +20,7 @@ jobs:
       - uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f  # v2.0.0
         with:
           toolchain: "1.94"
+          cache: false
 
       - uses: rustsec/audit-check@69366f33c96575abad1ee0dba8212993eecbe998  # v2.0.0
         with:

.github/workflows/CargoAudit.yml

@@ -44,18 +44,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Rust cache
-        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
-        with:
-          shared-key: "${{ runner.os }}-debug"
-          cache-on-failure: "true"
-          # Only save on main as caches are not shared across branches.
-          # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Set up sccache
-        uses: ./.github/actions/setup-sccache
-
       - name: Build guest binaries
         run: just guests
 

.github/workflows/Coverage.yml

@@ -1,13 +1,16 @@
 # yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json
 
-# Builds on main to populate the GitHub Actions cache for all Swatinem/rust-cache
-# shared-keys used by PR workflows. Caches saved on `refs/heads/main` are
-# restorable from every PR via the base-branch fallback. PR workflows are
-# configured with `save-if: github.ref == 'refs/heads/main'`, so they only ever
-# read from this cache.
+# The only workflow that writes the Rust caches pull requests restore. Entries
+# saved on main are restorable from every pull request. Every other workflow
+# only reads them.
 #
-# No tests are run here. The goal is just to populate dependency build artifacts
-# so that PR jobs start warm.
+# Each job looks up the entry for its current lockfiles and toolchain. If it
+# is missing, the job builds from a cold target and saves. rust-cache keeps
+# every dependency version it restores, so a build on a restored entry would
+# grow it with each lockfile change. Tests build but do not run.
+#
+# Keys hash lockfiles and toolchains, not build commands. Bump `prefix-key` in
+# every rust-cache step after changing what a job here builds.
 
 name: Prime Caches
 
@@ -16,6 +19,12 @@ on:
     branches: [main]
   workflow_dispatch:
 
+# A new run replaces a pending one and never cancels the running one, so the
+# newest commit always saves.
+concurrency:
+  group: prime-caches-${{ github.ref }}
+  cancel-in-progress: false
+
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: full
@@ -40,17 +49,31 @@ jobs:
     with:
       arch: ${{ matrix.arch }}
       config: ${{ matrix.config }}
+      prime_cache: true
 
-  # Populate v0-rust-code-checks-{linux,windows} caches.
+  # Populate v1-rust-code-checks-{linux,windows} caches.
   code-checks:
     uses: ./.github/workflows/dep_code_checks.yml
     secrets: inherit
+    with:
+      prime_cache: true
+
+  # Populate the fuzz cache (key fuzz-{arch}) with every fuzz target.
+  fuzz:
+    uses: ./.github/workflows/dep_fuzzing.yml
+    secrets: inherit
+    with:
+      prime_cache: true
+      arch: X64
+      target: all
+      max_total_time: 0
+      timeout_minutes: 30
 
   # Populate the host caches (key {arch}-{os}-{hypervisor}-{config}) used by
   # dep_build_test.yml. Linux is primed once per hypervisor because the kvm
   # pools run Ubuntu and the mshv3 pools run Azure Linux 3, and we want each
-  # distro to restore a cache produced under its own libc. Only the host
-  # dependency graph is built here, no tests are run.
+  # distro to restore a cache produced under its own libc. The host, its tests
+  # and hyperlight-ci are built here. No tests run.
   host:
     strategy:
       fail-fast: false
@@ -102,17 +125,44 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      # Keep the steps before this one in sync with dep_build_test.yml. The key
+      # hashes the installed toolchains and the Rust and C build env vars.
       - name: Rust cache
-        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        id: rust-cache
+        uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0  # v2.9.3
         with:
+          prefix-key: v1-rust
           shared-key: "${{ matrix.arch }}-${{ runner.os }}-${{ matrix.hypervisor }}-${{ matrix.config }}"
-          cache-on-failure: "true"
-          # Only save on main as caches are not shared across branches.
-          # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache
-          save-if: ${{ github.ref == 'refs/heads/main' }}
+          lookup-only: true
 
-      - name: Set up sccache
-        uses: ./.github/actions/setup-sccache
+      - name: Build host and tests
+        if: steps.rust-cache.outputs.cache-hit != 'true'
+        run: just prime-caches ${{ matrix.config }}
 
-      - name: Build host
-        run: just build ${{ matrix.config }}
+  # Rust keys end in a toolchain hash and a lockfile hash. Sysroot keys end in
+  # one hash. Each change leaves the old entry behind. Keep the newest entry
+  # per key and delete the rest to stay under the repository cache quota.
+  cleanup:
+    needs: [guests, code-checks, fuzz, host]
+    if: ${{ !cancelled() }}
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    env:
+      GH_TOKEN: ${{ github.token }}
+      GH_REPO: ${{ github.repository }}
+    steps:
+      - name: Delete superseded caches
+        run: |
+          # Filter by key prefix. The listing caps at 1000 entries.
+          for prefix in v1-rust- sysroot-; do
+            gh cache list --ref "$GITHUB_REF" --key "$prefix" --limit 1000 --json id,key,createdAt
+          done | jq -s -r '
+            add
+            | group_by(.key | if startswith("sysroot-") then sub("-[^-]*$"; "") else sub("-[^-]*-[^-]*$"; "") end)
+            | map(sort_by(.createdAt) | reverse | .[1:][])
+            | .[] | "\(.id) \(.key)"' |
+          while read -r id key; do
+            echo "Deleting $key"
+            gh cache delete "$id" || echo "::warning::Failed to delete $key"
+          done

.github/workflows/PrimeCaches.yml

@@ -50,29 +50,6 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-      # cargo-hyperlight builds a custom sysroot for x86_64-hyperlight-none target.
-      # rust-cache cleans "anything not a dependency" from target dirs, removing the sysroot.
-      # We cache sysroot separately to avoid rebuilding it (~10s) on every run.
-      - name: Sysroot cache
-        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
-        with:
-          path: |
-            src/tests/rust_guests/target/sysroot
-            src/tests/rust_guests/target-non-pie/sysroot
-          key: sysroot-linux-${{ matrix.config }}-${{ hashFiles('rust-toolchain.toml') }}
-
-      - name: Rust cache
-        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
-        with:
-          shared-key: "nightly-${{ matrix.config }}"
-          cache-on-failure: "true"
-          workspaces: |
-            . -> target
-            src/tests/rust_guests -> target
-
-      - name: Set up sccache
-        uses: ./.github/actions/setup-sccache
-
       - name: Install cargo-hyperlight
         run: just ensure-cargo-hyperlight
 

.github/workflows/RustNightly.yml

@@ -27,6 +27,8 @@ jobs:
         with:
           toolchain: nightly
           components: rustfmt
+          # The agent runs on branches, where saved caches serve only that branch and evict those of main.
+          cache: false
 
 
       - uses: hyperlight-dev/ci-setup-workflow@2f4142ba17cf573af44fc1e1f1ffc743daded5b3  # v1.10.0

.github/workflows/copilot-setup-steps.yml

@@ -82,14 +82,12 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Rust cache
-        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0  # v2.9.3
         with:
+          prefix-key: v1-rust
           shared-key: "${{ inputs.arch }}-${{ runner.os }}-${{ inputs.hypervisor }}-release"
           save-if: "false"
 
-      - name: Set up sccache
-        uses: ./.github/actions/setup-sccache
-
       - name: Download Rust guests
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2
         with:
@@ -116,6 +114,13 @@ jobs:
           name: c-guests-${{ inputs.arch }}-release-baseline
           path: ${{ runner.temp }}/baseline-guests/c/
 
+      # The baseline builds in its own target directory. Cargo fingerprints
+      # path dependencies by mtime and workspace path, so two checkouts cannot
+      # share one. A copy of the restored cache gives it the dependencies.
+      - name: Seed baseline target directory
+        if: inputs.baseline_ref != ''
+        run: if [ -d target ]; then cp -a target "$RUNNER_TEMP/baseline-target"; fi
+
       # Both commits are measured on this machine, so the comparison reflects
       # them rather than the difference between two runners. The branch point
       # is read off the merge this run validates, so on anything else there is
@@ -130,7 +135,7 @@ jobs:
               echo "::error::HEAD is not a merge, so ${{ inputs.baseline_ref }} is not the branch point"
               exit 1
             }
-            just bench-against ${{ inputs.baseline_ref }} "$RUNNER_TEMP/baseline-guests"
+            just bench-against ${{ inputs.baseline_ref }} "$RUNNER_TEMP/baseline-guests" "" "$RUNNER_TEMP/baseline-target"
           else
             just bench-ci main
           fi

.github/workflows/dep_benchmarks.yml

@@ -30,6 +30,13 @@ on:
         required: false
         type: string
         default: ""
+      prime_cache:
+        description: >-
+          Build the Rust guests from a cold target and save the caches. Skips
+          the build when they are up to date, and skips C guests and uploads.
+        required: false
+        type: boolean
+        default: false
 
 env:
   CARGO_TERM_COLOR: always
@@ -87,45 +94,51 @@ jobs:
         id: sysroot-cache
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
         with:
-          path: |
-            src/tests/rust_guests/target/sysroot
-            src/tests/rust_guests/target-non-pie/sysroot
+          path: src/tests/rust_guests/target/sysroot
           key: sysroot-linux-${{ inputs.arch }}-${{ inputs.config }}-${{ hashFiles('rust-toolchain.toml') }}
 
+      # Priming builds cold, so the entry holds only what this commit needs.
       - name: Rust cache
-        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        id: rust-cache
+        uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0  # v2.9.3
         with:
+          prefix-key: v1-rust
           shared-key: "guests-${{ inputs.arch }}-${{ inputs.config }}"
-          cache-on-failure: "true"
-          # Only save on main as caches are not shared across branches.
-          # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache
-          save-if: ${{ github.ref == 'refs/heads/main' }}
+          lookup-only: ${{ inputs.prime_cache }}
+          save-if: ${{ inputs.prime_cache }}
           workspaces: |
-            . -> target
             src/tests/rust_guests -> target
+            src/tests/rust_guests -> target-non-pie/build
 
-      - name: Set up sccache
-        uses: ./.github/actions/setup-sccache
+      - name: Check whether the caches are up to date
+        id: prime
+        if: inputs.prime_cache
+        run: echo "skip=${{ steps.rust-cache.outputs.cache-hit == 'true' && steps.sysroot-cache.outputs.cache-hit == 'true' }}" >> "$GITHUB_OUTPUT"
 
       - name: Install cargo-hyperlight
+        if: steps.prime.outputs.skip != 'true'
         run: just ensure-cargo-hyperlight
 
       - name: Build Rust guests
+        if: steps.prime.outputs.skip != 'true'
         run: |
           just build-rust-guests ${{ inputs.config }}
           just move-rust-guests ${{ inputs.config }}
 
       - name: Build non-PIE Rust guests
+        if: steps.prime.outputs.skip != 'true'
         run: |
           just build-rust-guests-non-pie ${{ inputs.config }}
           just move-rust-guests-non-pie ${{ inputs.config }}
 
       - name: Build C guests
+        if: ${{ !inputs.prime_cache }}
         run: |
           just build-c-guests ${{ inputs.config }}
           just move-c-guests ${{ inputs.config }}
 
       - name: Upload Rust guests
+        if: ${{ !inputs.prime_cache }}
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
         with:
           name: rust-guests-${{ inputs.arch }}-${{ inputs.config }}${{ inputs.artifact_suffix }}
@@ -135,6 +148,7 @@ jobs:
           if-no-files-found: error
 
       - name: Upload C guests
+        if: ${{ !inputs.prime_cache }}
         uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9  # v7.0.2
         with:
           name: c-guests-${{ inputs.arch }}-${{ inputs.config }}${{ inputs.artifact_suffix }}
@@ -143,13 +157,10 @@ jobs:
           if-no-files-found: error
 
       # A job step, not a post-job step, so the save runs before rust-cache's
-      # post-job cleanup deletes the sysroot. Only main saves, as caches are not
-      # shared across branches.
+      # post-job cleanup deletes the sysroot.
       - name: Save sysroot cache
-        if: ${{ github.ref == 'refs/heads/main' && steps.sysroot-cache.outputs.cache-hit != 'true' }}
+        if: ${{ inputs.prime_cache && steps.sysroot-cache.outputs.cache-hit != 'true' }}
         uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9  # v6.1.0
         with:
-          path: |
-            src/tests/rust_guests/target/sysroot
-            src/tests/rust_guests/target-non-pie/sysroot
+          path: src/tests/rust_guests/target/sysroot
           key: ${{ steps.sysroot-cache.outputs.cache-primary-key }}

.github/workflows/dep_build_guests.yml

@@ -106,18 +106,15 @@ jobs:
           $llvmPath >> $env:GITHUB_PATH
 
       - name: Rust cache
-        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6  # v2.9.2
+        uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0  # v2.9.3
         with:
+          prefix-key: v1-rust
           # Concurrent macOS runners share ~/.cargo/bin, so cleanup can remove Cargo from another job.
           cache-bin: ${{ runner.os != 'macOS' }}
           shared-key: "${{ inputs.arch }}-${{ runner.os }}-${{ inputs.hypervisor }}-${{ inputs.config }}"
-          cache-on-failure: "true"
-          # Only save on main as caches are not shared across branches.
-          # https://docs.github.com/en/actions/reference/workflows-and-actions/dependency-caching#restrictions-for-accessing-a-cache
-          save-if: ${{ github.ref == 'refs/heads/main' }}
-
-      - name: Set up sccache
-        uses: ./.github/actions/setup-sccache
+          # PrimeCaches writes this entry. Keep the steps before this one in
+          # sync with its host job.
+          save-if: "false"
 
       - name: Download Rust guests
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333  # v8.0.2