Cache the fixture adapters' Cargo registry and build tree from main

.github/workflows/server-parity.yml

@@ -11,6 +11,8 @@ on:
       - scripts/conformance/server-parity*
       - scripts/conformance/server-parity/**
       - scripts/ci/pull-images.sh
+      - scripts/ci/patched-pip.sh
+      - scripts/ci/sdk-adapter-cache-key.sh
       - tests/Fixtures/ServerParity/**
       - tests/Fixtures/ServerParityBaselines/**
       - tests/Fixtures/ServerParityNormalization/**
@@ -67,6 +69,7 @@ on:
       - tests/Unit/ServerParityUpdateValidationRetryTest.mjs
       - tests/Unit/ServerParityKilledAuthorPollTest.mjs
       - tests/Unit/ServerParityImagePullTest.mjs
+      - tests/Unit/ServerParityAdapterCacheTest.mjs
       - tests/Unit/ServerParityMessageStreamsContractTest.mjs
       - tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs
       - tests/Unit/ServerParityWaitingHistoryContractTest.mjs
@@ -142,6 +145,24 @@ jobs:
             mirror.gcr.io/library/rust@sha256:300ec56abce8cc9448ddea2172747d048ed902a3090e6b57babb2bf19f754081 \
             mirror.gcr.io/library/python@sha256:bf44cdfcb76cd3b41e879bc058fc37ec5872002ccfde7fcb765e218cde0cd79c \
             "$(baseline php-2.11.1)"
+      # The adapters' pinned, --locked dependencies build the same way every
+      # run. Restore their public Cargo registry data and build tree, keyed
+      # by the toolchain image, the runner and every tracked adapter input.
+      # Only successful pushes to main save them, so no pull request or merge
+      # group writes what a later run reads. Every run still builds, stamps
+      # and hashes its own adapters, and the final adapters are never cached.
+      - name: Key the adapters' Cargo cache by every tracked adapter input
+        id: adapter_key
+        run: bash scripts/ci/sdk-adapter-cache-key.sh
+      - name: Restore the adapters' Cargo registry and build tree
+        id: adapter_cargo
+        uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            sdk-cargo/registry
+            sdk-cargo/git
+            sdk-target
+          key: ${{ steps.adapter_key.outputs.key }}
       - name: Build the locked published SDK adapter once
         run: bash scripts/conformance/server-parity/published-sdk-patch-adapters.sh
       - name: Build genuine legacy-marker adapters with the existing cache
@@ -156,6 +177,15 @@ jobs:
         run: bash scripts/conformance/server-parity/corrected-marker-adapters.sh
       - name: Build the separate current query consumers once
         run: bash scripts/conformance/server-parity/query-authority-adapters.sh
+      - name: Save the adapters' Cargo registry and build tree from main
+        if: ${{ success() && github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.adapter_cargo.outputs.cache-hit != 'true' }}
+        uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: |
+            sdk-cargo/registry
+            sdk-cargo/git
+            sdk-target
+          key: ${{ steps.adapter_key.outputs.key }}
       - name: Share current query consumers without compiler caches
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
         with:
@@ -1873,7 +1903,7 @@ jobs:
           bash scripts/ci/run-composer.sh --rm -v "$PWD:/source" -w /source -- \
             install --working-dir=scripts/conformance/server-parity --no-interaction --no-progress
           docker run --rm --user=1000:1000 --entrypoint node -v "$PWD:/source:ro" -w /source \
-            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt
+            "$image" --test tests/Unit/ServerParityRunnerTest.mjs tests/Unit/ServerParitySdkDirectionLanesTest.mjs tests/Unit/ServerParityCodecDeadlineCellsTest.mjs tests/Unit/ServerParityCooperativeContractTest.mjs tests/Unit/ServerParityChildCancellationContractTest.mjs tests/Unit/ServerParityScheduleContractTest.mjs tests/Unit/ServerParityVisibilityContractTest.mjs tests/Unit/ServerParityAdmissionContractTest.mjs tests/Unit/ServerParityRoleTokenContractTest.mjs tests/Unit/ServerParityWorkerDeregistrationContractTest.mjs tests/Unit/ServerParitySearchAttributeContractTest.mjs tests/Unit/ServerParityWorkerListingContractTest.mjs tests/Unit/ServerParityWorkerReplacementContractTest.mjs tests/Unit/ServerParityPatchDeploymentContractTest.mjs tests/Unit/ServerParityWaitingHistoryContractTest.mjs tests/Unit/ServerParityTaskRetryContractTest.mjs tests/Unit/ServerParityPublishedPatchContractTest.mjs tests/Unit/ServerParityPublishedPatchTransportTest.mjs tests/Unit/ServerParityRecordFileTest.mjs tests/Unit/ServerParityFailureLocationTest.mjs tests/Unit/ServerParityFailureExcerptTest.mjs tests/Unit/ServerParityLateWorkflowCompletionContractTest.mjs tests/Unit/ServerParityStorageTestContractTest.mjs tests/Unit/ServerParitySchedulePoliciesContractTest.mjs tests/Unit/ServerParitySignalsBeforeWorkerContractTest.mjs tests/Unit/ServerParityScheduleCatalogContractTest.mjs tests/Unit/ServerParityOperatorMetricsContractTest.mjs tests/Unit/ServerParityActivityLeaseRenewalContractTest.mjs tests/Unit/ServerParityRecordLocalActivityContractTest.mjs tests/Unit/ServerParityActivityCancellationAckContractTest.mjs tests/Unit/ServerParityServiceCatalogContractTest.mjs tests/Unit/ServerParityServiceEndpointsContractTest.mjs tests/Unit/ServerParityServiceCallsContractTest.mjs tests/Unit/ServerParityWorkflowImportContractTest.mjs tests/Unit/ServerParityWaterlineImportContractTest.mjs tests/Unit/ServerParitySystemMetricsContractTest.mjs tests/Unit/ServerParityOperatorDashboardTypesContractTest.mjs tests/Unit/ServerParityPrometheusMetricsContractTest.mjs tests/Unit/ServerParityPayloadCleanupContractTest.mjs tests/Unit/ServerParityRepairPassContractTest.mjs tests/Unit/ServerParityClusterInfoContractTest.mjs tests/Unit/ServerParityNamespaceDeleteContractTest.mjs tests/Unit/ServerParityMessageStreamsContractTest.mjs tests/Unit/ServerParityCancellationScopesContractTest.mjs tests/Unit/ServerParityWorkflowPollRequestContractTest.mjs tests/Unit/ServerParityUpdateValidationContractTest.mjs tests/Unit/ServerParityFixtureSelectionTest.mjs tests/Unit/ServerParityUpdateValidationRetryTest.mjs tests/Unit/ServerParityKilledAuthorPollTest.mjs tests/Unit/ServerParityImagePullTest.mjs tests/Unit/ServerParityAdapterCacheTest.mjs tests/Unit/ServerParityWorkerSessionsContractTest.mjs tests/Unit/ServerParityWorkerSessionActivitiesContractTest.mjs tests/Unit/ServerParityNexusOperationsContractTest.mjs tests/Unit/ServerParityBridgeAdaptersContractTest.mjs tests/Unit/ServerParityWorkflowStreamWritesContractTest.mjs > parity-evidence/comparator-tests.txt
           docker run --rm --user=1000:1000 --cpus=1 --memory=256m --memory-swap=256m \
             --network=none --entrypoint node -v "$PWD:/source:ro" -w /source \
             "$image" --test tests/Unit/ServerParityCodecDeadlineLoadTest.mjs > parity-evidence/codec-load-tests.txt

scripts/ci/sdk-adapter-cache-key.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Print the exact cache key for the published SDK adapters' public Cargo
+# registry data and build tree (Shared Server fixtures, Server #325), and
+# write it to $GITHUB_OUTPUT as `key` when that is set.
+#
+# The key names the runner OS and architecture, every pinned Rust image the
+# adapter builds use and a digest of every tracked adapter input: the five
+# adapter crates, their build scripts, the pip wrapper and the SDK profiles.
+# It is an exact key with no restore prefix. Only pushes to main save it.
+set -euo pipefail
+: "${RUNNER_OS:?RUNNER_OS must be set}" "${RUNNER_ARCH:?RUNNER_ARCH must be set}"
+
+parity=scripts/conformance/server-parity
+profiles=tests/Fixtures/ServerParityProfiles
+mapfile -t inputs < <(git ls-files -- \
+  "$parity/published-sdk-patch" \
+  "$parity/sdk-directions" \
+  "$parity/legacy-marker-aliases" \
+  "$parity/corrected-markers" \
+  "$parity/query-authority" \
+  ":(glob)$parity/*-adapters.sh" \
+  scripts/ci/patched-pip.sh \
+  "$profiles/published-sdk-patch-replay" \
+  "$profiles/published-sdk-directions" \
+  "$profiles/legacy-marker-aliases" \
+  "$profiles/corrected-markers" \
+  "$profiles/query-authority" \
+  "$profiles/version-range-repair" | LC_ALL=C sort)
+
+# Never a partial key: all five adapter crates and build scripts must be
+# inputs. A sixth adapter must join the list above.
+test "$(printf '%s\n' "${inputs[@]}" | grep -c '/rust/Cargo\.lock$')" -eq 5
+test "$(printf '%s\n' "${inputs[@]}" | grep -c -- '-adapters\.sh$')" -eq 5
+
+# The key names every pinned Rust image the inputs build with: the build
+# scripts pin one and the query-authority profile another. The input digest
+# below covers them too; naming them keeps a toolchain change readable.
+mapfile -t toolchains < <(grep -ho 'library/rust@sha256:[0-9a-f]\{64\}' -- "${inputs[@]}" | sed 's/.*@sha256://' | LC_ALL=C sort -u)
+test "${#toolchains[@]}" -ge 1
+rust=$(IFS=-; printf '%s' "${toolchains[*]}")
+
+digest=$(sha256sum -- "${inputs[@]}" | sha256sum | cut -c1-64)
+key="sdk-adapter-cargo-v2-$RUNNER_OS-$RUNNER_ARCH-rust-$rust-inputs-$digest"
+if test -n "${GITHUB_OUTPUT:-}"; then
+  printf 'key=%s\n' "$key" >> "$GITHUB_OUTPUT"
+fi
+printf '%s\n' "$key"

tests/Unit/ServerParityAdapterCacheTest.mjs

@@ -0,0 +1,78 @@
+// The published SDK adapters' public Cargo registry data and build tree are
+// cached between runs, but only successful pushes to main save them: a pull
+// request or merge group can restore main's cache and never write one a
+// later run reads. The key covers the toolchain, the runner and every
+// tracked adapter input, and an empty input list fails the job.
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {readFileSync} from 'node:fs';
+import {resolve} from 'node:path';
+
+const workflow = readFileSync(resolve('.github/workflows/server-parity.yml'), 'utf8');
+
+function job(name) {
+  const start = workflow.indexOf(`\n  ${name}:\n`);
+  assert.ok(start >= 0, name);
+  const next = workflow.slice(start + 1).search(/\n {2}[a-z0-9-]+:\n/);
+  return next < 0 ? workflow.slice(start) : workflow.slice(start, start + 1 + next);
+}
+
+const steps = block => block.split(/\n {6}(?=- (?:name|uses):)/).slice(1);
+
+test('only the adapter job caches, and only public registry data and the build tree', () => {
+  const uses = workflow.match(/uses: actions\/cache[^\s]*/g) ?? [];
+  assert.deepEqual(uses, [
+    'uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9',
+    'uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9',
+  ]);
+  const adapters = steps(job('published-sdk-adapters'));
+  const restore = adapters.findIndex(step => step.includes('actions/cache/restore@'));
+  const save = adapters.findIndex(step => step.includes('actions/cache/save@'));
+  const builds = adapters.map((step, index) => [step, index]).filter(([step]) => /-adapters\.sh\n?$/m.test(step)).map(([, index]) => index);
+  assert.equal(builds.length, 5);
+  assert.ok(restore >= 0 && restore < Math.min(...builds), 'restored before the first build');
+  assert.ok(save > Math.max(...builds), 'saved after the last build');
+  for (const index of [restore, save]) {
+    // Never the whole CARGO_HOME, where credentials would live.
+    assert.match(adapters[index], /path: \|\n {12}sdk-cargo\/registry\n {12}sdk-cargo\/git\n {12}sdk-target\n/);
+    assert.match(adapters[index], /key: \$\{\{ steps\.adapter_key\.outputs\.key \}\}$/m);
+  }
+  // No restore-keys: a run gets this exact key's tree or builds from nothing.
+  assert.doesNotMatch(adapters[restore], /restore-keys/);
+  assert.match(adapters[save], /if: \$\{\{ success\(\) && github\.event_name == 'push' && github\.ref == 'refs\/heads\/main' && /);
+});
+
+test('the cache key follows the toolchains, the runner and every tracked adapter input', () => {
+  const block = job('published-sdk-adapters');
+  const key = steps(block).find(step => step.includes('id: adapter_key'));
+  assert.ok(key, 'a key step');
+  // The key comes from one repository helper, never inline shell.
+  assert.match(key, /\n {8}run: bash scripts\/ci\/sdk-adapter-cache-key\.sh\n?$/);
+  const helper = readFileSync(resolve('scripts/ci/sdk-adapter-cache-key.sh'), 'utf8');
+  assert.match(helper, /^set -euo pipefail$/m);
+  assert.match(helper, /git ls-files --/);
+  for (const adapter of ['published-sdk-patch', 'sdk-directions', 'legacy-marker-aliases', 'corrected-markers', 'query-authority']) {
+    assert.ok(helper.includes(`"$parity/${adapter}" \\\n`), adapter);
+  }
+  assert.ok(helper.includes('":(glob)$parity/*-adapters.sh" \\\n'));
+  assert.ok(helper.includes('scripts/ci/patched-pip.sh \\\n'));
+  // An empty or partial input list fails instead of keying a cache.
+  assert.match(helper, /grep -c '\/rust\/Cargo\\\.lock\$'\)" -eq 5/);
+  assert.match(helper, /grep -c -- '-adapters\\\.sh\$'\)" -eq 5/);
+  // Every pinned Rust image among the inputs is named, as are the runner
+  // and a digest of every input's name and content.
+  assert.match(helper, /grep -ho 'library\/rust@sha256:\[0-9a-f\]\\\{64\\\}' -- "\$\{inputs\[@\]\}"/);
+  assert.match(helper, /test "\$\{#toolchains\[@\]\}" -ge 1/);
+  assert.match(helper, /digest=\$\(sha256sum -- "\$\{inputs\[@\]\}" \| sha256sum \| cut -c1-64\)/);
+  assert.ok(helper.includes('key="sdk-adapter-cargo-v2-$RUNNER_OS-$RUNNER_ARCH-rust-$rust-inputs-$digest"'));
+  // The five build scripts are the job's adapter build steps.
+  const builds = steps(block).filter(step => /-adapters\.sh\n?$/m.test(step));
+  assert.equal(builds.length, 5);
+});
+
+test('changing the key helper or the pip wrapper runs the workflow on main', () => {
+  const push = workflow.slice(workflow.indexOf('\n  push:\n'), workflow.indexOf('\njobs:\n'));
+  for (const path of ['scripts/ci/sdk-adapter-cache-key.sh', 'scripts/ci/patched-pip.sh', 'tests/Unit/ServerParityAdapterCacheTest.mjs']) {
+    assert.ok(push.includes(`\n      - ${path}\n`), path);
+  }
+});