From 4854d25188e807fb8e42ce679293d7c01e8e5630 Mon Sep 17 00:00:00 2001 From: Erik Scorelle Date: Wed, 4 Nov 2026 09:45:30 -0500 Subject: [PATCH 1/7] Cap archived macOS sccache storage at 30 GB Add trusted-main serialized maintenance with baseline-aware eviction and dry-run regression coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-sccache/README.md | 34 ++++ .github/scripts/prune-macos-sccache.cjs | 130 +++++++++++++ .github/scripts/prune-macos-sccache.test.cjs | 181 ++++++++++++++++++ .../workflows/macos-sccache-maintenance.yml | 70 +++++++ 4 files changed, 415 insertions(+) create mode 100644 .github/scripts/prune-macos-sccache.cjs create mode 100644 .github/scripts/prune-macos-sccache.test.cjs create mode 100644 .github/workflows/macos-sccache-maintenance.yml diff --git a/.github/actions/setup-sccache/README.md b/.github/actions/setup-sccache/README.md index 00007bab35293..d3edfb426b498 100644 --- a/.github/actions/setup-sccache/README.md +++ b/.github/actions/setup-sccache/README.md @@ -39,3 +39,37 @@ Actions cache ref-scoping still applies; a cache created on a feature branch is The actions do not select a CMake generator or enable compiler launchers themselves. Keep `--use_cache --cache_tool sccache` in build flags. Windows launcher support in `build.py` currently requires Ninja; validate the selected compiler/debug-symbol configuration when adopting it. + +## macOS archived-storage budget + +`macos-sccache-maintenance.yml` enforces a **30 GB decimal (30,000,000,000 bytes) aggregate** +budget for GitHub's archived macOS sccache snapshots, separate from the unchanged per-run `10G` +local cache. It runs after main's `MacOS CI Pipeline` completes (even if tests fail after cache +publication), with an hourly scheduled fallback. GitHub may delay scheduled runs. + +The budget includes well-formed `sccache-v1-macOS-{ARM64,X64}--` snapshots +on `refs/heads/main` and legacy `macos-sccache-v1-`/`macos-sccache-v2-` snapshots on the exact +pilot ref `refs/heads/eserscor-dev/erscor/2026/10/30-mac-compiler-cachi`. Other refs/namespaces, +vcpkg, Linux and Windows caches are untouched. New key formats require updating the scope matcher. + +Cleanup inventories all pages in both namespaces, then removes pilot snapshots first and older +main generations oldest-first, preferentially retaining the newest snapshot per compatibility +prefix. If those baselines alone exceed the budget, it warns and removes the oldest baselines +too (including a single oversized snapshot), rather than silently exceeding the limit. +Exact immutable cache IDs are deleted, and API errors fail the maintenance job. + +This is a **soft cap**, not an atomic GitHub storage quota: cleanup jobs serialize with each other, +not with cache uploads. Storage can exceed 30 GB between maintenance runs or during concurrent +publication. Cleanup rechecks storage afterward and fails visibly if still over budget; the next +run retries. Cache eviction may cause a cold build but does not change build/test coverage. + +Only the separate trusted-main cleanup job has `actions: write`. PRs run regression tests with +read-only permissions; cleanup never checks out PR code or consumes triggering-run artifacts. +Manual dispatch defaults to dry run. For a read-only local preview with an authenticated `gh`: + +```bash +GITHUB_REPOSITORY=microsoft/onnxruntime node .github/scripts/prune-macos-sccache.cjs +node --test .github/scripts/prune-macos-sccache.test.cjs +``` + +Omitting `--apply` never deletes caches. No live cleanup is needed to run the regression tests. diff --git a/.github/scripts/prune-macos-sccache.cjs b/.github/scripts/prune-macos-sccache.cjs new file mode 100644 index 0000000000000..5b3ca4353d276 --- /dev/null +++ b/.github/scripts/prune-macos-sccache.cjs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +const { execFileSync } = require('node:child_process'); + +const BUDGET_BYTES = 30_000_000_000; +const MAIN_REF = 'refs/heads/main'; +const PILOT_REF = 'refs/heads/eserscor-dev/erscor/2026/10/30-mac-compiler-cachi'; +const MAIN_KEY = /^(sccache-v1-macOS-(?:ARM64|X64)-[a-f0-9]{64}-)[a-f0-9]{40}$/; +const PILOT_KEY = /^macos-sccache-v[12]-.+-[a-f0-9]{64}-[a-f0-9]{40}$/; + +function compatibility(cache) { + if (cache.ref === MAIN_REF) { + return MAIN_KEY.exec(cache.key)?.[1]; + } + return undefined; +} + +function inScope(cache) { + return Boolean(compatibility(cache) || (cache.ref === PILOT_REF && PILOT_KEY.test(cache.key))); +} + +function selectEvictions(inventory, budget = BUDGET_BYTES) { + if (!Number.isSafeInteger(budget) || budget < 0) { + throw new Error('Cache budget must be a nonnegative safe integer'); + } + const caches = inventory.filter(inScope); + for (const cache of caches) { + if ( + !Number.isSafeInteger(cache.id) || cache.id <= 0 || + !Number.isSafeInteger(cache.size_in_bytes) || cache.size_in_bytes < 0 || + !Number.isFinite(Date.parse(cache.created_at)) + ) { + throw new Error(`Invalid cache metadata for ${cache.key}`); + } + } + caches.sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at) || b.id - a.id); + const baselines = new Set(); + const prefixes = new Set(); + for (const cache of caches) { + const prefix = compatibility(cache); + if (prefix && !prefixes.has(prefix)) { + prefixes.add(prefix); + baselines.add(cache.id); + } + } + // Retire pilot snapshots first, then old main generations, and only then main baselines. + const priority = (cache) => baselines.has(cache.id) ? 2 : compatibility(cache) ? 1 : 0; + const candidates = [...caches].sort((a, b) => + priority(a) - priority(b) || Date.parse(a.created_at) - Date.parse(b.created_at) || a.id - b.id); + const totalBytes = caches.reduce((sum, cache) => sum + cache.size_in_bytes, 0); + if (!Number.isSafeInteger(totalBytes)) { + throw new Error('Aggregate cache size exceeds safe integer range'); + } + let remainingBytes = totalBytes; + const deletions = []; + for (const cache of candidates) { + if (remainingBytes <= budget) { + break; + } + deletions.push(cache); + remainingBytes -= cache.size_in_bytes; + } + return { totalBytes, remainingBytes, deletions, evictedBaselines: deletions.filter((c) => baselines.has(c.id)) }; +} + +function ghApi(args) { + return execFileSync('gh', ['api', ...args], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); +} + +function listCaches(repository, api) { + const caches = new Map(); + for (const key of ['sccache-v1-macOS-', 'macos-sccache-v']) { + const pages = JSON.parse(api([ + '--paginate', '--slurp', `repos/${repository}/actions/caches?per_page=100&key=${key}`, + ])); + if (!Array.isArray(pages) || pages.length === 0) { + throw new Error('Invalid cache pagination response'); + } + for (const page of pages) { + if (!Array.isArray(page.actions_caches)) { + throw new Error('Missing actions_caches in API response'); + } + for (const cache of page.actions_caches) { + caches.set(cache.id, cache); + } + } + } + return [...caches.values()]; +} + +function run({ repository, apply = false, api = ghApi, log = console.log }) { + if (!/^[\w.-]+\/[\w.-]+$/.test(repository)) { + throw new Error('Expected repository in owner/name format'); + } + const plan = selectEvictions(listCaches(repository, api)); + log(`${apply ? 'Apply' : 'Dry run'}: ${plan.totalBytes} bytes; budget ${BUDGET_BYTES}; ` + + `${plan.deletions.length} evictions; planned remainder ${plan.remainingBytes} bytes`); + if (plan.evictedBaselines.length > 0) { + log(`::warning::Budget requires evicting ${plan.evictedBaselines.length} newest compatibility baselines`); + } + for (const cache of plan.deletions) { + // IDs are immutable; scope is checked again at the destructive boundary. + if (!inScope(cache) || !Number.isSafeInteger(cache.id) || cache.id <= 0) { + throw new Error('Refusing to delete an out-of-scope cache'); + } + log(`${apply ? 'Deleting' : 'Would delete'} cache ${cache.id}: ${cache.ref} ${cache.key} (${cache.size_in_bytes} bytes)`); + if (apply) { + api(['--method', 'DELETE', `repos/${repository}/actions/caches/${cache.id}`]); + } + } + if (apply) { + const observed = selectEvictions(listCaches(repository, api)); + log(`Observed storage after cleanup: ${observed.totalBytes} bytes`); + if (observed.totalBytes > BUDGET_BYTES) { + throw new Error('macOS sccache remains over budget (possibly concurrent uploads); next maintenance run will retry'); + } + } + return plan; +} + +if (require.main === module) { + const args = process.argv.slice(2); + if (args.length > 1 || (args.length === 1 && args[0] !== '--apply')) { + throw new Error('Usage: node prune-macos-sccache.cjs [--apply] (default: dry run)'); + } + run({ repository: process.env.GITHUB_REPOSITORY, apply: args[0] === '--apply' }); +} + +module.exports = { BUDGET_BYTES, MAIN_REF, PILOT_REF, selectEvictions, listCaches, run }; diff --git a/.github/scripts/prune-macos-sccache.test.cjs b/.github/scripts/prune-macos-sccache.test.cjs new file mode 100644 index 0000000000000..dfbc233b7201a --- /dev/null +++ b/.github/scripts/prune-macos-sccache.test.cjs @@ -0,0 +1,181 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +const assert = require('node:assert/strict'); +const { test } = require('node:test'); +const { BUDGET_BYTES, MAIN_REF, PILOT_REF, selectEvictions, listCaches, run } = require('./prune-macos-sccache.cjs'); + +function cache(id, size, variant = 'a', overrides = {}) { + return { + id, + key: `sccache-v1-macOS-ARM64-${variant.repeat(64)}-${id.toString(16).padStart(40, '0')}`, + ref: MAIN_REF, + size_in_bytes: size, + created_at: new Date(Date.UTC(2026, 10, 1, 0, id)).toISOString(), + ...overrides, + }; +} + +function pilot(id, size) { + return cache(id, size, 'a', { + ref: PILOT_REF, + key: `macos-sccache-v2-arm64-arm64-Debug-${'a'.repeat(64)}-${'b'.repeat(40)}`, + }); +} + +const ids = (plan) => plan.deletions.map((c) => c.id); +const pages = (entries) => JSON.stringify([{ actions_caches: entries }]); +const quiet = () => {}; + +test('scope excludes other OSes, vcpkg, unknown keys and all other refs', () => { + const entries = [ + cache(1, 10, 'a', { key: 'vcpkg-macos-cache' }), + cache(2, 10, 'a', { key: cache(2, 10).key.replace('macOS', 'Linux') }), + cache(3, 10, 'a', { key: cache(3, 10).key.replace('macOS', 'Windows') }), + cache(4, 10, 'a', { ref: 'refs/pull/42/merge' }), + cache(5, 10, 'a', { ref: 'refs/heads/feature' }), + cache(6, 10, 'a', { key: 'sccache-v1-macOS-unrecognized' }), + { ...pilot(7, 10), ref: MAIN_REF }, + { ...pilot(8, 10), ref: 'refs/heads/feature' }, + cache(9, 10), + pilot(10, 10), + ]; + assert.deepEqual(ids(selectEvictions(entries, 0)), [10, 9]); +}); + +test('empty inventory, exact budget and under budget need no eviction', () => { + assert.deepEqual(ids(selectEvictions([], 0)), []); + assert.deepEqual(ids(selectEvictions([cache(1, BUDGET_BYTES)])), []); + assert.deepEqual(ids(selectEvictions([cache(1, 1)])), []); +}); + +test('Intel main snapshots and both pilot key versions are included', () => { + const intel = cache(1, 10, 'a', { key: cache(1, 10).key.replace('ARM64', 'X64') }); + const oldPilot = { ...pilot(2, 10), key: pilot(2, 10).key.replace('-v2-', '-v1-') }; + assert.deepEqual(ids(selectEvictions([intel, oldPilot, pilot(3, 10)], 0)), [2, 3, 1]); +}); + +test('aggregate budget evicts pilot then oldest generations, preserving newest per prefix', () => { + const entries = [cache(1, 8), cache(2, 8, 'b'), cache(3, 8), cache(4, 8, 'b'), pilot(5, 8)]; + const plan = selectEvictions(entries.reverse(), 24); + assert.deepEqual(ids(plan), [5, 1]); + assert.equal(plan.totalBytes, 40); + assert.equal(plan.remainingBytes, 24); + assert.deepEqual(plan.evictedBaselines, []); +}); + +test('multiple generations and compatibility prefixes are independent', () => { + const entries = Array.from({ length: 170 }, (_, i) => cache(i + 1, 1_000_000_000, (i % 10).toString(16))); + const plan = selectEvictions(entries); + assert.equal(plan.deletions.length, 140); + assert.equal(plan.remainingBytes, BUDGET_BYTES); + assert.deepEqual(ids(plan), Array.from({ length: 140 }, (_, i) => i + 1)); + assert.deepEqual(plan.evictedBaselines, []); +}); + +test('oversized protected set evicts oldest baselines explicitly', () => { + const plan = selectEvictions([cache(1, 20), cache(2, 20, 'b')], 30); + assert.deepEqual(ids(plan), [1]); + assert.deepEqual(plan.evictedBaselines.map((c) => c.id), [1]); + assert.deepEqual(ids(selectEvictions([cache(1, 31)], 30)), [1]); +}); + +test('equal creation times break ties by ID, not inventory order or access time', () => { + const a = cache(1, 10); + const b = cache(2, 10, 'a', { created_at: a.created_at, last_accessed_at: '2000-01-01T00:00:00Z' }); + assert.deepEqual(ids(selectEvictions([b, a], 10)), [1]); +}); + +test('invalid budgets and scoped metadata fail closed', () => { + for (const budget of [-1, NaN, 1.5]) { + assert.throws(() => selectEvictions([], budget)); + } + for (const overrides of [{ id: -1 }, { size_in_bytes: -1 }, { created_at: 'invalid' }]) { + assert.throws(() => selectEvictions([cache(1, 1, 'a', overrides)])); + } +}); + +test('both namespaces use complete pagination and duplicate IDs are counted once', () => { + const calls = []; + const entries = listCaches('owner/repo', (args) => { + calls.push(args); + return JSON.stringify([{ actions_caches: [cache(1, 1)] }, { actions_caches: [cache(2, 1)] }]); + }); + assert.equal(entries.length, 2); + assert.equal(calls.length, 2); + assert.ok(calls.every((args) => args.includes('--paginate') && args.includes('--slurp'))); + assert.ok(calls[0].at(-1).includes('per_page=100&key=sccache-v1-macOS-')); + assert.ok(calls[1].at(-1).includes('per_page=100&key=macos-sccache-v')); +}); + +test('dry run never deletes and reports baseline loss', () => { + const messages = []; + run({ + repository: 'owner/repo', + api: (args) => { + assert.ok(!args.includes('DELETE')); + return pages([cache(1, BUDGET_BYTES + 1)]); + }, + log: (message) => messages.push(message), + }); + assert.ok(messages.some((message) => message.includes('::warning::'))); + assert.ok(messages.some((message) => message.includes('Would delete cache 1'))); +}); + +test('apply deletes only exact scoped IDs and rechecks observed aggregate', () => { + const calls = []; + let lists = 0; + run({ + repository: 'owner/repo', + apply: true, + api: (args) => { + calls.push(args); + if (args.includes('DELETE')) { + return ''; + } + return pages(++lists <= 2 ? [cache(1, BUDGET_BYTES), cache(2, 1)] : [cache(2, 1)]); + }, + log: quiet, + }); + assert.deepEqual(calls.filter((args) => args.includes('DELETE')), [ + ['--method', 'DELETE', 'repos/owner/repo/actions/caches/1'], + ]); + assert.equal(lists, 4); +}); + +test('listing, malformed response and deletion failures propagate', () => { + assert.throws(() => run({ repository: 'owner/repo', api: () => { throw new Error('403'); }, log: quiet }), /403/); + for (const response of ['bad JSON', '[]', '[{}]']) { + assert.throws(() => listCaches('owner/repo', () => response)); + } + assert.throws(() => run({ + repository: 'owner/repo', apply: true, log: quiet, + api: (args) => { + if (args.includes('DELETE')) { + throw new Error('503'); + } + return pages([cache(1, BUDGET_BYTES + 1)]); + }, + }), /503/); +}); + +test('failure listing the second namespace prevents all deletion', () => { + let calls = 0; + assert.throws(() => run({ + repository: 'owner/repo', apply: true, log: quiet, + api: (args) => { + assert.ok(!args.includes('DELETE')); + if (++calls === 2) { + throw new Error('Second namespace failed'); + } + return pages([cache(1, BUDGET_BYTES + 1)]); + }, + }), /Second namespace failed/); +}); + +test('concurrent uploads exceeding budget are reported as failure after deletion', () => { + assert.throws(() => run({ + repository: 'owner/repo', apply: true, log: quiet, + api: (args) => args.includes('DELETE') ? '' : pages([cache(1, BUDGET_BYTES + 1)]), + }), /remains over budget/); +}); diff --git a/.github/workflows/macos-sccache-maintenance.yml b/.github/workflows/macos-sccache-maintenance.yml new file mode 100644 index 0000000000000..cc54ebd7a1b0a --- /dev/null +++ b/.github/workflows/macos-sccache-maintenance.yml @@ -0,0 +1,70 @@ +name: macOS sccache storage maintenance + +on: + workflow_run: + workflows: ["MacOS CI Pipeline"] + types: [completed] + branches: [main] + schedule: + - cron: '37 * * * *' + workflow_dispatch: + inputs: + dry_run: + description: Report planned evictions without deleting caches + type: boolean + default: true + pull_request: + paths: + - '.github/scripts/prune-macos-sccache*' + - '.github/workflows/macos-sccache-maintenance.yml' + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Never execute the triggering workflow's checkout in a privileged workflow_run. + ref: ${{ github.event_name == 'pull_request' && github.ref || 'main' }} + - run: node --test .github/scripts/prune-macos-sccache.test.cjs + + cleanup: + needs: test + if: >- + github.repository == 'microsoft/onnxruntime' && + github.ref == 'refs/heads/main' && + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + (github.event_name == 'workflow_run' && + github.event.workflow_run.event == 'push' && + github.event.workflow_run.head_branch == 'main' && + github.event.workflow_run.head_repository.full_name == github.repository)) + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + actions: write + concurrency: + group: macos-sccache-storage-maintenance + cancel-in-progress: false + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: main + persist-credentials: false + - name: Enforce archived macOS compiler-cache budget + env: + GH_TOKEN: ${{ github.token }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} + shell: bash + run: | + set -euo pipefail + args=() + if [[ "$DRY_RUN" != "true" ]]; then + args+=(--apply) + fi + node .github/scripts/prune-macos-sccache.cjs "${args[@]}" From 529dc8b7f241449025dfaf6f919c9b595130cef2 Mon Sep 17 00:00:00 2001 From: Erik Scorelle Date: Wed, 4 Nov 2026 10:05:00 -0500 Subject: [PATCH 2/7] Use Python for macOS sccache maintenance Port cache selection, GitHub API integration and regression tests to Python standard library; update workflow and documentation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-sccache/README.md | 5 +- .github/scripts/prune-macos-sccache.cjs | 130 ------------- .github/scripts/prune-macos-sccache.test.cjs | 181 ------------------ .github/scripts/prune_macos_sccache.py | 158 +++++++++++++++ .github/scripts/test_prune_macos_sccache.py | 170 ++++++++++++++++ .../workflows/macos-sccache-maintenance.yml | 6 +- 6 files changed, 334 insertions(+), 316 deletions(-) delete mode 100644 .github/scripts/prune-macos-sccache.cjs delete mode 100644 .github/scripts/prune-macos-sccache.test.cjs create mode 100644 .github/scripts/prune_macos_sccache.py create mode 100644 .github/scripts/test_prune_macos_sccache.py diff --git a/.github/actions/setup-sccache/README.md b/.github/actions/setup-sccache/README.md index d3edfb426b498..90d9c1e4d5586 100644 --- a/.github/actions/setup-sccache/README.md +++ b/.github/actions/setup-sccache/README.md @@ -68,8 +68,9 @@ read-only permissions; cleanup never checks out PR code or consumes triggering-r Manual dispatch defaults to dry run. For a read-only local preview with an authenticated `gh`: ```bash -GITHUB_REPOSITORY=microsoft/onnxruntime node .github/scripts/prune-macos-sccache.cjs -node --test .github/scripts/prune-macos-sccache.test.cjs +GITHUB_REPOSITORY=microsoft/onnxruntime python .github/scripts/prune_macos_sccache.py +python -m unittest discover -s .github/scripts -p test_prune_macos_sccache.py -v ``` Omitting `--apply` never deletes caches. No live cleanup is needed to run the regression tests. +The cleanup script and tests use only Python's standard library and the preinstalled `gh` CLI. diff --git a/.github/scripts/prune-macos-sccache.cjs b/.github/scripts/prune-macos-sccache.cjs deleted file mode 100644 index 5b3ca4353d276..0000000000000 --- a/.github/scripts/prune-macos-sccache.cjs +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -const { execFileSync } = require('node:child_process'); - -const BUDGET_BYTES = 30_000_000_000; -const MAIN_REF = 'refs/heads/main'; -const PILOT_REF = 'refs/heads/eserscor-dev/erscor/2026/10/30-mac-compiler-cachi'; -const MAIN_KEY = /^(sccache-v1-macOS-(?:ARM64|X64)-[a-f0-9]{64}-)[a-f0-9]{40}$/; -const PILOT_KEY = /^macos-sccache-v[12]-.+-[a-f0-9]{64}-[a-f0-9]{40}$/; - -function compatibility(cache) { - if (cache.ref === MAIN_REF) { - return MAIN_KEY.exec(cache.key)?.[1]; - } - return undefined; -} - -function inScope(cache) { - return Boolean(compatibility(cache) || (cache.ref === PILOT_REF && PILOT_KEY.test(cache.key))); -} - -function selectEvictions(inventory, budget = BUDGET_BYTES) { - if (!Number.isSafeInteger(budget) || budget < 0) { - throw new Error('Cache budget must be a nonnegative safe integer'); - } - const caches = inventory.filter(inScope); - for (const cache of caches) { - if ( - !Number.isSafeInteger(cache.id) || cache.id <= 0 || - !Number.isSafeInteger(cache.size_in_bytes) || cache.size_in_bytes < 0 || - !Number.isFinite(Date.parse(cache.created_at)) - ) { - throw new Error(`Invalid cache metadata for ${cache.key}`); - } - } - caches.sort((a, b) => Date.parse(b.created_at) - Date.parse(a.created_at) || b.id - a.id); - const baselines = new Set(); - const prefixes = new Set(); - for (const cache of caches) { - const prefix = compatibility(cache); - if (prefix && !prefixes.has(prefix)) { - prefixes.add(prefix); - baselines.add(cache.id); - } - } - // Retire pilot snapshots first, then old main generations, and only then main baselines. - const priority = (cache) => baselines.has(cache.id) ? 2 : compatibility(cache) ? 1 : 0; - const candidates = [...caches].sort((a, b) => - priority(a) - priority(b) || Date.parse(a.created_at) - Date.parse(b.created_at) || a.id - b.id); - const totalBytes = caches.reduce((sum, cache) => sum + cache.size_in_bytes, 0); - if (!Number.isSafeInteger(totalBytes)) { - throw new Error('Aggregate cache size exceeds safe integer range'); - } - let remainingBytes = totalBytes; - const deletions = []; - for (const cache of candidates) { - if (remainingBytes <= budget) { - break; - } - deletions.push(cache); - remainingBytes -= cache.size_in_bytes; - } - return { totalBytes, remainingBytes, deletions, evictedBaselines: deletions.filter((c) => baselines.has(c.id)) }; -} - -function ghApi(args) { - return execFileSync('gh', ['api', ...args], { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }); -} - -function listCaches(repository, api) { - const caches = new Map(); - for (const key of ['sccache-v1-macOS-', 'macos-sccache-v']) { - const pages = JSON.parse(api([ - '--paginate', '--slurp', `repos/${repository}/actions/caches?per_page=100&key=${key}`, - ])); - if (!Array.isArray(pages) || pages.length === 0) { - throw new Error('Invalid cache pagination response'); - } - for (const page of pages) { - if (!Array.isArray(page.actions_caches)) { - throw new Error('Missing actions_caches in API response'); - } - for (const cache of page.actions_caches) { - caches.set(cache.id, cache); - } - } - } - return [...caches.values()]; -} - -function run({ repository, apply = false, api = ghApi, log = console.log }) { - if (!/^[\w.-]+\/[\w.-]+$/.test(repository)) { - throw new Error('Expected repository in owner/name format'); - } - const plan = selectEvictions(listCaches(repository, api)); - log(`${apply ? 'Apply' : 'Dry run'}: ${plan.totalBytes} bytes; budget ${BUDGET_BYTES}; ` + - `${plan.deletions.length} evictions; planned remainder ${plan.remainingBytes} bytes`); - if (plan.evictedBaselines.length > 0) { - log(`::warning::Budget requires evicting ${plan.evictedBaselines.length} newest compatibility baselines`); - } - for (const cache of plan.deletions) { - // IDs are immutable; scope is checked again at the destructive boundary. - if (!inScope(cache) || !Number.isSafeInteger(cache.id) || cache.id <= 0) { - throw new Error('Refusing to delete an out-of-scope cache'); - } - log(`${apply ? 'Deleting' : 'Would delete'} cache ${cache.id}: ${cache.ref} ${cache.key} (${cache.size_in_bytes} bytes)`); - if (apply) { - api(['--method', 'DELETE', `repos/${repository}/actions/caches/${cache.id}`]); - } - } - if (apply) { - const observed = selectEvictions(listCaches(repository, api)); - log(`Observed storage after cleanup: ${observed.totalBytes} bytes`); - if (observed.totalBytes > BUDGET_BYTES) { - throw new Error('macOS sccache remains over budget (possibly concurrent uploads); next maintenance run will retry'); - } - } - return plan; -} - -if (require.main === module) { - const args = process.argv.slice(2); - if (args.length > 1 || (args.length === 1 && args[0] !== '--apply')) { - throw new Error('Usage: node prune-macos-sccache.cjs [--apply] (default: dry run)'); - } - run({ repository: process.env.GITHUB_REPOSITORY, apply: args[0] === '--apply' }); -} - -module.exports = { BUDGET_BYTES, MAIN_REF, PILOT_REF, selectEvictions, listCaches, run }; diff --git a/.github/scripts/prune-macos-sccache.test.cjs b/.github/scripts/prune-macos-sccache.test.cjs deleted file mode 100644 index dfbc233b7201a..0000000000000 --- a/.github/scripts/prune-macos-sccache.test.cjs +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -const assert = require('node:assert/strict'); -const { test } = require('node:test'); -const { BUDGET_BYTES, MAIN_REF, PILOT_REF, selectEvictions, listCaches, run } = require('./prune-macos-sccache.cjs'); - -function cache(id, size, variant = 'a', overrides = {}) { - return { - id, - key: `sccache-v1-macOS-ARM64-${variant.repeat(64)}-${id.toString(16).padStart(40, '0')}`, - ref: MAIN_REF, - size_in_bytes: size, - created_at: new Date(Date.UTC(2026, 10, 1, 0, id)).toISOString(), - ...overrides, - }; -} - -function pilot(id, size) { - return cache(id, size, 'a', { - ref: PILOT_REF, - key: `macos-sccache-v2-arm64-arm64-Debug-${'a'.repeat(64)}-${'b'.repeat(40)}`, - }); -} - -const ids = (plan) => plan.deletions.map((c) => c.id); -const pages = (entries) => JSON.stringify([{ actions_caches: entries }]); -const quiet = () => {}; - -test('scope excludes other OSes, vcpkg, unknown keys and all other refs', () => { - const entries = [ - cache(1, 10, 'a', { key: 'vcpkg-macos-cache' }), - cache(2, 10, 'a', { key: cache(2, 10).key.replace('macOS', 'Linux') }), - cache(3, 10, 'a', { key: cache(3, 10).key.replace('macOS', 'Windows') }), - cache(4, 10, 'a', { ref: 'refs/pull/42/merge' }), - cache(5, 10, 'a', { ref: 'refs/heads/feature' }), - cache(6, 10, 'a', { key: 'sccache-v1-macOS-unrecognized' }), - { ...pilot(7, 10), ref: MAIN_REF }, - { ...pilot(8, 10), ref: 'refs/heads/feature' }, - cache(9, 10), - pilot(10, 10), - ]; - assert.deepEqual(ids(selectEvictions(entries, 0)), [10, 9]); -}); - -test('empty inventory, exact budget and under budget need no eviction', () => { - assert.deepEqual(ids(selectEvictions([], 0)), []); - assert.deepEqual(ids(selectEvictions([cache(1, BUDGET_BYTES)])), []); - assert.deepEqual(ids(selectEvictions([cache(1, 1)])), []); -}); - -test('Intel main snapshots and both pilot key versions are included', () => { - const intel = cache(1, 10, 'a', { key: cache(1, 10).key.replace('ARM64', 'X64') }); - const oldPilot = { ...pilot(2, 10), key: pilot(2, 10).key.replace('-v2-', '-v1-') }; - assert.deepEqual(ids(selectEvictions([intel, oldPilot, pilot(3, 10)], 0)), [2, 3, 1]); -}); - -test('aggregate budget evicts pilot then oldest generations, preserving newest per prefix', () => { - const entries = [cache(1, 8), cache(2, 8, 'b'), cache(3, 8), cache(4, 8, 'b'), pilot(5, 8)]; - const plan = selectEvictions(entries.reverse(), 24); - assert.deepEqual(ids(plan), [5, 1]); - assert.equal(plan.totalBytes, 40); - assert.equal(plan.remainingBytes, 24); - assert.deepEqual(plan.evictedBaselines, []); -}); - -test('multiple generations and compatibility prefixes are independent', () => { - const entries = Array.from({ length: 170 }, (_, i) => cache(i + 1, 1_000_000_000, (i % 10).toString(16))); - const plan = selectEvictions(entries); - assert.equal(plan.deletions.length, 140); - assert.equal(plan.remainingBytes, BUDGET_BYTES); - assert.deepEqual(ids(plan), Array.from({ length: 140 }, (_, i) => i + 1)); - assert.deepEqual(plan.evictedBaselines, []); -}); - -test('oversized protected set evicts oldest baselines explicitly', () => { - const plan = selectEvictions([cache(1, 20), cache(2, 20, 'b')], 30); - assert.deepEqual(ids(plan), [1]); - assert.deepEqual(plan.evictedBaselines.map((c) => c.id), [1]); - assert.deepEqual(ids(selectEvictions([cache(1, 31)], 30)), [1]); -}); - -test('equal creation times break ties by ID, not inventory order or access time', () => { - const a = cache(1, 10); - const b = cache(2, 10, 'a', { created_at: a.created_at, last_accessed_at: '2000-01-01T00:00:00Z' }); - assert.deepEqual(ids(selectEvictions([b, a], 10)), [1]); -}); - -test('invalid budgets and scoped metadata fail closed', () => { - for (const budget of [-1, NaN, 1.5]) { - assert.throws(() => selectEvictions([], budget)); - } - for (const overrides of [{ id: -1 }, { size_in_bytes: -1 }, { created_at: 'invalid' }]) { - assert.throws(() => selectEvictions([cache(1, 1, 'a', overrides)])); - } -}); - -test('both namespaces use complete pagination and duplicate IDs are counted once', () => { - const calls = []; - const entries = listCaches('owner/repo', (args) => { - calls.push(args); - return JSON.stringify([{ actions_caches: [cache(1, 1)] }, { actions_caches: [cache(2, 1)] }]); - }); - assert.equal(entries.length, 2); - assert.equal(calls.length, 2); - assert.ok(calls.every((args) => args.includes('--paginate') && args.includes('--slurp'))); - assert.ok(calls[0].at(-1).includes('per_page=100&key=sccache-v1-macOS-')); - assert.ok(calls[1].at(-1).includes('per_page=100&key=macos-sccache-v')); -}); - -test('dry run never deletes and reports baseline loss', () => { - const messages = []; - run({ - repository: 'owner/repo', - api: (args) => { - assert.ok(!args.includes('DELETE')); - return pages([cache(1, BUDGET_BYTES + 1)]); - }, - log: (message) => messages.push(message), - }); - assert.ok(messages.some((message) => message.includes('::warning::'))); - assert.ok(messages.some((message) => message.includes('Would delete cache 1'))); -}); - -test('apply deletes only exact scoped IDs and rechecks observed aggregate', () => { - const calls = []; - let lists = 0; - run({ - repository: 'owner/repo', - apply: true, - api: (args) => { - calls.push(args); - if (args.includes('DELETE')) { - return ''; - } - return pages(++lists <= 2 ? [cache(1, BUDGET_BYTES), cache(2, 1)] : [cache(2, 1)]); - }, - log: quiet, - }); - assert.deepEqual(calls.filter((args) => args.includes('DELETE')), [ - ['--method', 'DELETE', 'repos/owner/repo/actions/caches/1'], - ]); - assert.equal(lists, 4); -}); - -test('listing, malformed response and deletion failures propagate', () => { - assert.throws(() => run({ repository: 'owner/repo', api: () => { throw new Error('403'); }, log: quiet }), /403/); - for (const response of ['bad JSON', '[]', '[{}]']) { - assert.throws(() => listCaches('owner/repo', () => response)); - } - assert.throws(() => run({ - repository: 'owner/repo', apply: true, log: quiet, - api: (args) => { - if (args.includes('DELETE')) { - throw new Error('503'); - } - return pages([cache(1, BUDGET_BYTES + 1)]); - }, - }), /503/); -}); - -test('failure listing the second namespace prevents all deletion', () => { - let calls = 0; - assert.throws(() => run({ - repository: 'owner/repo', apply: true, log: quiet, - api: (args) => { - assert.ok(!args.includes('DELETE')); - if (++calls === 2) { - throw new Error('Second namespace failed'); - } - return pages([cache(1, BUDGET_BYTES + 1)]); - }, - }), /Second namespace failed/); -}); - -test('concurrent uploads exceeding budget are reported as failure after deletion', () => { - assert.throws(() => run({ - repository: 'owner/repo', apply: true, log: quiet, - api: (args) => args.includes('DELETE') ? '' : pages([cache(1, BUDGET_BYTES + 1)]), - }), /remains over budget/); -}); diff --git a/.github/scripts/prune_macos_sccache.py b/.github/scripts/prune_macos_sccache.py new file mode 100644 index 0000000000000..15c49d33ab720 --- /dev/null +++ b/.github/scripts/prune_macos_sccache.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +"""Enforce the archived macOS sccache budget; default to a read-only preview.""" + +import argparse +import json +import os +import re +import subprocess +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import TypedDict + +BUDGET_BYTES = 30_000_000_000 +MAIN_REF = "refs/heads/main" +PILOT_REF = "refs/heads/eserscor-dev/erscor/2026/10/30-mac-compiler-cachi" +MAIN_KEY = re.compile(r"(sccache-v1-macOS-(?:ARM64|X64)-[a-f0-9]{64}-)[a-f0-9]{40}") +PILOT_KEY = re.compile(r"macos-sccache-v[12]-.+-[a-f0-9]{64}-[a-f0-9]{40}") + + +class Cache(TypedDict): + id: int + key: str + ref: str + size_in_bytes: int + created_at: str + + +@dataclass +class EvictionPlan: + total_bytes: int + remaining_bytes: int + deletions: list[Cache] + evicted_baselines: list[Cache] + + +def compatibility(cache: Cache) -> str | None: + if cache["ref"] == MAIN_REF: + match = MAIN_KEY.fullmatch(cache["key"]) + if match: + return match[1] + return None + + +def in_scope(cache: Cache) -> bool: + return bool(compatibility(cache) or (cache["ref"] == PILOT_REF and PILOT_KEY.fullmatch(cache["key"]))) + + +def creation_time(cache: Cache) -> datetime: + timestamp = datetime.fromisoformat(cache["created_at"].replace("Z", "+00:00")) + if timestamp.tzinfo is None: + raise ValueError(f"Cache timestamp must have a timezone: {cache['key']}") + return timestamp + + +def select_evictions(inventory: Sequence[Cache], budget: int = BUDGET_BYTES) -> EvictionPlan: + if type(budget) is not int or budget < 0: + raise ValueError("Cache budget must be a nonnegative integer") + caches = [cache for cache in inventory if in_scope(cache)] + for cache in caches: + if ( + type(cache["id"]) is not int + or cache["id"] <= 0 + or type(cache["size_in_bytes"]) is not int + or cache["size_in_bytes"] < 0 + ): + raise ValueError(f"Invalid cache metadata for {cache['key']}") + creation_time(cache) + caches.sort(key=lambda cache: (creation_time(cache), cache["id"]), reverse=True) + baselines = set() + prefixes = set() + for cache in caches: + prefix = compatibility(cache) + if prefix and prefix not in prefixes: + prefixes.add(prefix) + baselines.add(cache["id"]) + + def priority(cache: Cache) -> int: + # Retire pilot snapshots first, then old main generations, and only then main baselines. + if cache["id"] in baselines: + return 2 + return 1 if compatibility(cache) else 0 + + candidates = sorted(caches, key=lambda cache: (priority(cache), creation_time(cache), cache["id"])) + total_bytes = sum(cache["size_in_bytes"] for cache in caches) + remaining_bytes = total_bytes + deletions = [] + for cache in candidates: + if remaining_bytes <= budget: + break + deletions.append(cache) + remaining_bytes -= cache["size_in_bytes"] + return EvictionPlan( + total_bytes, remaining_bytes, deletions, [cache for cache in deletions if cache["id"] in baselines] + ) + + +def gh_api(args: list[str]) -> str: + return subprocess.check_output(["gh", "api", *args], text=True, encoding="utf-8") + + +def list_caches(repository: str, api: Callable[[list[str]], str]) -> list[Cache]: + caches = {} + for key in ["sccache-v1-macOS-", "macos-sccache-v"]: + pages = json.loads(api(["--paginate", "--slurp", f"repos/{repository}/actions/caches?per_page=100&key={key}"])) + if not isinstance(pages, list) or not pages: + raise ValueError("Invalid cache pagination response") + for page in pages: + if not isinstance(page, dict) or not isinstance(page.get("actions_caches"), list): + raise ValueError("Missing actions_caches in API response") + for cache in page["actions_caches"]: + caches[cache["id"]] = cache + return list(caches.values()) + + +def run( + repository: str, + *, + apply: bool = False, + api: Callable[[list[str]], str] = gh_api, + log: Callable[[str], None] = print, +) -> EvictionPlan: + if not re.fullmatch(r"[\w.-]+/[\w.-]+", repository, flags=re.ASCII): + raise ValueError("Expected repository in owner/name format") + plan = select_evictions(list_caches(repository, api)) + log( + f"{'Apply' if apply else 'Dry run'}: {plan.total_bytes} bytes; budget {BUDGET_BYTES}; " + f"{len(plan.deletions)} evictions; planned remainder {plan.remaining_bytes} bytes" + ) + if plan.evicted_baselines: + log(f"::warning::Budget requires evicting {len(plan.evicted_baselines)} newest compatibility baselines") + for cache in plan.deletions: + # IDs are immutable; scope is checked again at the destructive boundary. + if not in_scope(cache) or type(cache["id"]) is not int or cache["id"] <= 0: + raise ValueError("Refusing to delete an out-of-scope cache") + log( + f"{'Deleting' if apply else 'Would delete'} cache {cache['id']}: " + f"{cache['ref']} {cache['key']} ({cache['size_in_bytes']} bytes)" + ) + if apply: + api(["--method", "DELETE", f"repos/{repository}/actions/caches/{cache['id']}"]) + if apply: + observed = select_evictions(list_caches(repository, api)) + log(f"Observed storage after cleanup: {observed.total_bytes} bytes") + if observed.total_bytes > BUDGET_BYTES: + raise RuntimeError( + "macOS sccache remains over budget (possibly concurrent uploads); next maintenance run will retry" + ) + return plan + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="Delete selected caches (default: dry run)") + args = parser.parse_args() + run(repository=os.environ["GITHUB_REPOSITORY"], apply=args.apply) diff --git a/.github/scripts/test_prune_macos_sccache.py b/.github/scripts/test_prune_macos_sccache.py new file mode 100644 index 0000000000000..90ecd251c1507 --- /dev/null +++ b/.github/scripts/test_prune_macos_sccache.py @@ -0,0 +1,170 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +import subprocess +import unittest +from datetime import datetime, timedelta, timezone +from unittest.mock import Mock, patch + +from prune_macos_sccache import BUDGET_BYTES, MAIN_REF, PILOT_REF, gh_api, list_caches, run, select_evictions + + +def cache(cache_id, size, variant="a", **overrides): + return { + "id": cache_id, + "key": f"sccache-v1-macOS-ARM64-{variant * 64}-{cache_id:040x}", + "ref": MAIN_REF, + "size_in_bytes": size, + "created_at": (datetime(2026, 11, 1, tzinfo=timezone.utc) + timedelta(minutes=cache_id)).isoformat(), + **overrides, + } + + +def pilot(cache_id, size): + return cache(cache_id, size, ref=PILOT_REF, key=f"macos-sccache-v2-arm64-arm64-Debug-{'a' * 64}-{'b' * 40}") + + +def ids(plan): + return [entry["id"] for entry in plan.deletions] + + +def pages(entries): + return json.dumps([{"actions_caches": entries}]) + + +class PruneMacosSccacheTest(unittest.TestCase): + def test_scope_excludes_other_namespaces_and_refs(self): + entries = [ + cache(1, 10, key="vcpkg-macos-cache"), + cache(2, 10, key=cache(2, 10)["key"].replace("macOS", "Linux")), + cache(3, 10, key=cache(3, 10)["key"].replace("macOS", "Windows")), + cache(4, 10, ref="refs/pull/42/merge"), + cache(5, 10, ref="refs/heads/feature"), + cache(6, 10, key="sccache-v1-macOS-unrecognized"), + {**pilot(7, 10), "ref": MAIN_REF}, + {**pilot(8, 10), "ref": "refs/heads/feature"}, + cache(9, 10), + pilot(10, 10), + ] + self.assertEqual(ids(select_evictions(entries, 0)), [10, 9]) + + def test_empty_exact_and_under_budget_need_no_eviction(self): + self.assertEqual(ids(select_evictions([], 0)), []) + self.assertEqual(ids(select_evictions([cache(1, BUDGET_BYTES)])), []) + self.assertEqual(ids(select_evictions([cache(1, 1)])), []) + + def test_intel_and_both_pilot_versions_are_included(self): + intel = cache(1, 10, key=cache(1, 10)["key"].replace("ARM64", "X64")) + old_pilot = {**pilot(2, 10), "key": pilot(2, 10)["key"].replace("-v2-", "-v1-")} + self.assertEqual(ids(select_evictions([intel, old_pilot, pilot(3, 10)], 0)), [2, 3, 1]) + + def test_aggregate_budget_preserves_newest_per_prefix(self): + entries = [cache(1, 8), cache(2, 8, "b"), cache(3, 8), cache(4, 8, "b"), pilot(5, 8)] + plan = select_evictions(entries[::-1], 24) + self.assertEqual(ids(plan), [5, 1]) + self.assertEqual(plan.total_bytes, 40) + self.assertEqual(plan.remaining_bytes, 24) + self.assertEqual(plan.evicted_baselines, []) + + def test_multiple_generations_and_prefixes_are_independent(self): + entries = [cache(i + 1, 1_000_000_000, f"{i % 10:x}") for i in range(170)] + plan = select_evictions(entries) + self.assertEqual(len(plan.deletions), 140) + self.assertEqual(plan.remaining_bytes, BUDGET_BYTES) + self.assertEqual(ids(plan), list(range(1, 141))) + self.assertEqual(plan.evicted_baselines, []) + + def test_oversized_protected_set_evicts_oldest_baselines(self): + plan = select_evictions([cache(1, 20), cache(2, 20, "b")], 30) + self.assertEqual(ids(plan), [1]) + self.assertEqual([entry["id"] for entry in plan.evicted_baselines], [1]) + self.assertEqual(ids(select_evictions([cache(1, 31)], 30)), [1]) + + def test_equal_creation_times_break_ties_by_id(self): + first = cache(1, 10) + second = cache(2, 10, created_at=first["created_at"], last_accessed_at="2000-01-01T00:00:00Z") + self.assertEqual(ids(select_evictions([second, first], 10)), [1]) + + def test_invalid_budget_and_scoped_metadata_fail_closed(self): + for budget in [-1, float("nan"), 1.5, True]: + with self.subTest(budget=budget), self.assertRaises(ValueError): + select_evictions([], budget) + for overrides in [ + {"id": -1}, + {"id": True}, + {"size_in_bytes": -1}, + {"size_in_bytes": False}, + {"created_at": "invalid"}, + {"created_at": "2026-11-01T00:00:00"}, + ]: + with self.subTest(overrides=overrides), self.assertRaises(ValueError): + select_evictions([cache(1, 1, **overrides)]) + + def test_pagination_and_duplicate_ids(self): + api = Mock(return_value=json.dumps([{"actions_caches": [cache(1, 1)]}, {"actions_caches": [cache(2, 1)]}])) + self.assertEqual(len(list_caches("owner/repo", api)), 2) + self.assertEqual(api.call_count, 2) + for call in api.call_args_list: + self.assertIn("--paginate", call.args[0]) + self.assertIn("--slurp", call.args[0]) + self.assertIn("per_page=100&key=sccache-v1-macOS-", api.call_args_list[0].args[0][-1]) + self.assertIn("per_page=100&key=macos-sccache-v", api.call_args_list[1].args[0][-1]) + + def test_dry_run_never_deletes_and_reports_baseline_loss(self): + api = Mock(return_value=pages([cache(1, BUDGET_BYTES + 1)])) + log = Mock() + run("owner/repo", api=api, log=log) + self.assertEqual(api.call_count, 2) + self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list)) + self.assertTrue(any("::warning::" in call.args[0] for call in log.call_args_list)) + self.assertTrue(any("Would delete cache 1" in call.args[0] for call in log.call_args_list)) + + def test_apply_deletes_exact_ids_and_rechecks_aggregate(self): + before = pages([cache(1, BUDGET_BYTES), cache(2, 1)]) + after = pages([cache(2, 1)]) + api = Mock(side_effect=[before, before, "", after, after]) + run("owner/repo", apply=True, api=api, log=Mock()) + self.assertEqual(api.call_count, 5) + self.assertEqual(api.call_args_list[2].args[0], ["--method", "DELETE", "repos/owner/repo/actions/caches/1"]) + self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list[:2])) + self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list[3:])) + + def test_listing_malformed_response_and_delete_errors_propagate(self): + with self.assertRaisesRegex(RuntimeError, "403"): + run("owner/repo", api=Mock(side_effect=RuntimeError("403")), log=Mock()) + for response in ["bad JSON", "[]", "[{}]", "[null]"]: + with self.subTest(response=response), self.assertRaises(ValueError): + list_caches("owner/repo", Mock(return_value=response)) + before = pages([cache(1, BUDGET_BYTES + 1)]) + with self.assertRaisesRegex(RuntimeError, "503"): + run("owner/repo", apply=True, api=Mock(side_effect=[before, before, RuntimeError("503")]), log=Mock()) + + def test_second_namespace_failure_prevents_deletion(self): + api = Mock(side_effect=[pages([cache(1, BUDGET_BYTES + 1)]), RuntimeError("Second namespace failed")]) + with self.assertRaisesRegex(RuntimeError, "Second namespace failed"): + run("owner/repo", apply=True, api=api, log=Mock()) + self.assertEqual(api.call_count, 2) + self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list)) + + def test_concurrent_uploads_fail_post_cleanup_check(self): + before = pages([cache(1, BUDGET_BYTES + 1)]) + with self.assertRaisesRegex(RuntimeError, "remains over budget"): + run("owner/repo", apply=True, api=Mock(side_effect=[before, before, "", before, before]), log=Mock()) + + def test_gh_api_invocation_and_process_errors(self): + with patch("prune_macos_sccache.subprocess.check_output", return_value="[]") as check_output: + self.assertEqual(gh_api(["--paginate"]), "[]") + check_output.assert_called_once_with(["gh", "api", "--paginate"], text=True, encoding="utf-8") + with ( + patch( + "prune_macos_sccache.subprocess.check_output", + side_effect=subprocess.CalledProcessError(1, ["gh", "api"]), + ), + self.assertRaises(subprocess.CalledProcessError), + ): + gh_api([]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/macos-sccache-maintenance.yml b/.github/workflows/macos-sccache-maintenance.yml index cc54ebd7a1b0a..86d9364dc8f42 100644 --- a/.github/workflows/macos-sccache-maintenance.yml +++ b/.github/workflows/macos-sccache-maintenance.yml @@ -15,7 +15,7 @@ on: default: true pull_request: paths: - - '.github/scripts/prune-macos-sccache*' + - '.github/scripts/*prune_macos_sccache.py' - '.github/workflows/macos-sccache-maintenance.yml' permissions: @@ -31,7 +31,7 @@ jobs: persist-credentials: false # Never execute the triggering workflow's checkout in a privileged workflow_run. ref: ${{ github.event_name == 'pull_request' && github.ref || 'main' }} - - run: node --test .github/scripts/prune-macos-sccache.test.cjs + - run: python -m unittest discover -s .github/scripts -p test_prune_macos_sccache.py -v cleanup: needs: test @@ -67,4 +67,4 @@ jobs: if [[ "$DRY_RUN" != "true" ]]; then args+=(--apply) fi - node .github/scripts/prune-macos-sccache.cjs "${args[@]}" + python .github/scripts/prune_macos_sccache.py "${args[@]}" From dc145fa9a390f79a63d0ed96fce23f3b64a0d286 Mon Sep 17 00:00:00 2001 From: Erik Scorelle Date: Wed, 4 Nov 2026 10:18:49 -0500 Subject: [PATCH 3/7] Generalize sccache storage budget across operating systems Share the 30 GB archived-storage cap across all main sccache-v1 OS, architecture and build identities. Run generic hourly maintenance independently of specific build workflows and exclude legacy pilot caches. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-sccache/README.md | 27 +++--- ...rune_macos_sccache.py => prune_sccache.py} | 39 ++++---- ...macos_sccache.py => test_prune_sccache.py} | 90 ++++++++++--------- ...aintenance.yml => sccache-maintenance.yml} | 25 ++---- 4 files changed, 88 insertions(+), 93 deletions(-) rename .github/scripts/{prune_macos_sccache.py => prune_sccache.py} (75%) rename .github/scripts/{test_prune_macos_sccache.py => test_prune_sccache.py} (65%) rename .github/workflows/{macos-sccache-maintenance.yml => sccache-maintenance.yml} (64%) diff --git a/.github/actions/setup-sccache/README.md b/.github/actions/setup-sccache/README.md index 90d9c1e4d5586..a86d043e283f8 100644 --- a/.github/actions/setup-sccache/README.md +++ b/.github/actions/setup-sccache/README.md @@ -40,20 +40,21 @@ The actions do not select a CMake generator or enable compiler launchers themsel `--use_cache --cache_tool sccache` in build flags. Windows launcher support in `build.py` currently requires Ninja; validate the selected compiler/debug-symbol configuration when adopting it. -## macOS archived-storage budget +## Shared archived-storage budget -`macos-sccache-maintenance.yml` enforces a **30 GB decimal (30,000,000,000 bytes) aggregate** -budget for GitHub's archived macOS sccache snapshots, separate from the unchanged per-run `10G` -local cache. It runs after main's `MacOS CI Pipeline` completes (even if tests fail after cache -publication), with an hourly scheduled fallback. GitHub may delay scheduled runs. +`sccache-maintenance.yml` enforces one **30 GB decimal (30,000,000,000 bytes) aggregate** +budget for GitHub's archived sccache snapshots across all operating systems, architectures, +and build variants using these shared actions, separate from the unchanged per-run `10G` +local cache. It runs hourly, independently of any particular build workflow, so new callers +participate automatically without adding workflow names. GitHub may delay scheduled runs. -The budget includes well-formed `sccache-v1-macOS-{ARM64,X64}--` snapshots -on `refs/heads/main` and legacy `macos-sccache-v1-`/`macos-sccache-v2-` snapshots on the exact -pilot ref `refs/heads/eserscor-dev/erscor/2026/10/30-mac-compiler-cachi`. Other refs/namespaces, -vcpkg, Linux and Windows caches are untouched. New key formats require updating the scope matcher. +The budget includes well-formed `sccache-v1----` snapshots on +`refs/heads/main`. OS and architecture identifiers are not restricted to the current runners. +Other refs/namespaces, vcpkg, CodeQL, and legacy `macos-sccache-v1-`/`v2-` pilot caches are +untouched and are not counted in this budget. New key formats require updating the scope matcher. -Cleanup inventories all pages in both namespaces, then removes pilot snapshots first and older -main generations oldest-first, preferentially retaining the newest snapshot per compatibility +Cleanup inventories all pages in the shared namespace, then removes older main generations +oldest-first, preferentially retaining the newest snapshot per compatibility prefix. If those baselines alone exceed the budget, it warns and removes the oldest baselines too (including a single oversized snapshot), rather than silently exceeding the limit. Exact immutable cache IDs are deleted, and API errors fail the maintenance job. @@ -68,8 +69,8 @@ read-only permissions; cleanup never checks out PR code or consumes triggering-r Manual dispatch defaults to dry run. For a read-only local preview with an authenticated `gh`: ```bash -GITHUB_REPOSITORY=microsoft/onnxruntime python .github/scripts/prune_macos_sccache.py -python -m unittest discover -s .github/scripts -p test_prune_macos_sccache.py -v +GITHUB_REPOSITORY=microsoft/onnxruntime python .github/scripts/prune_sccache.py +python -m unittest discover -s .github/scripts -p test_prune_sccache.py -v ``` Omitting `--apply` never deletes caches. No live cleanup is needed to run the regression tests. diff --git a/.github/scripts/prune_macos_sccache.py b/.github/scripts/prune_sccache.py similarity index 75% rename from .github/scripts/prune_macos_sccache.py rename to .github/scripts/prune_sccache.py index 15c49d33ab720..1bbfaa49d1a4b 100644 --- a/.github/scripts/prune_macos_sccache.py +++ b/.github/scripts/prune_sccache.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -"""Enforce the archived macOS sccache budget; default to a read-only preview.""" +"""Enforce the shared archived sccache budget; default to a read-only preview.""" import argparse import json @@ -15,9 +15,8 @@ BUDGET_BYTES = 30_000_000_000 MAIN_REF = "refs/heads/main" -PILOT_REF = "refs/heads/eserscor-dev/erscor/2026/10/30-mac-compiler-cachi" -MAIN_KEY = re.compile(r"(sccache-v1-macOS-(?:ARM64|X64)-[a-f0-9]{64}-)[a-f0-9]{40}") -PILOT_KEY = re.compile(r"macos-sccache-v[12]-.+-[a-f0-9]{64}-[a-f0-9]{40}") +KEY_PREFIX = "sccache-v1-" +MAIN_KEY = re.compile(r"(sccache-v1-[A-Za-z0-9_]+-[A-Za-z0-9_]+-[a-f0-9]{64}-)[a-f0-9]{40}") class Cache(TypedDict): @@ -45,7 +44,7 @@ def compatibility(cache: Cache) -> str | None: def in_scope(cache: Cache) -> bool: - return bool(compatibility(cache) or (cache["ref"] == PILOT_REF and PILOT_KEY.fullmatch(cache["key"]))) + return compatibility(cache) is not None def creation_time(cache: Cache) -> datetime: @@ -77,13 +76,8 @@ def select_evictions(inventory: Sequence[Cache], budget: int = BUDGET_BYTES) -> prefixes.add(prefix) baselines.add(cache["id"]) - def priority(cache: Cache) -> int: - # Retire pilot snapshots first, then old main generations, and only then main baselines. - if cache["id"] in baselines: - return 2 - return 1 if compatibility(cache) else 0 - - candidates = sorted(caches, key=lambda cache: (priority(cache), creation_time(cache), cache["id"])) + # Retire older generations before evicting any compatibility prefix's newest baseline. + candidates = sorted(caches, key=lambda cache: (cache["id"] in baselines, creation_time(cache), cache["id"])) total_bytes = sum(cache["size_in_bytes"] for cache in caches) remaining_bytes = total_bytes deletions = [] @@ -103,15 +97,16 @@ def gh_api(args: list[str]) -> str: def list_caches(repository: str, api: Callable[[list[str]], str]) -> list[Cache]: caches = {} - for key in ["sccache-v1-macOS-", "macos-sccache-v"]: - pages = json.loads(api(["--paginate", "--slurp", f"repos/{repository}/actions/caches?per_page=100&key={key}"])) - if not isinstance(pages, list) or not pages: - raise ValueError("Invalid cache pagination response") - for page in pages: - if not isinstance(page, dict) or not isinstance(page.get("actions_caches"), list): - raise ValueError("Missing actions_caches in API response") - for cache in page["actions_caches"]: - caches[cache["id"]] = cache + pages = json.loads( + api(["--paginate", "--slurp", f"repos/{repository}/actions/caches?per_page=100&key={KEY_PREFIX}"]) + ) + if not isinstance(pages, list) or not pages: + raise ValueError("Invalid cache pagination response") + for page in pages: + if not isinstance(page, dict) or not isinstance(page.get("actions_caches"), list): + raise ValueError("Missing actions_caches in API response") + for cache in page["actions_caches"]: + caches[cache["id"]] = cache return list(caches.values()) @@ -146,7 +141,7 @@ def run( log(f"Observed storage after cleanup: {observed.total_bytes} bytes") if observed.total_bytes > BUDGET_BYTES: raise RuntimeError( - "macOS sccache remains over budget (possibly concurrent uploads); next maintenance run will retry" + "sccache remains over budget (possibly concurrent uploads); next maintenance run will retry" ) return plan diff --git a/.github/scripts/test_prune_macos_sccache.py b/.github/scripts/test_prune_sccache.py similarity index 65% rename from .github/scripts/test_prune_macos_sccache.py rename to .github/scripts/test_prune_sccache.py index 90ecd251c1507..28d9c9f58e9ae 100644 --- a/.github/scripts/test_prune_macos_sccache.py +++ b/.github/scripts/test_prune_sccache.py @@ -7,13 +7,13 @@ from datetime import datetime, timedelta, timezone from unittest.mock import Mock, patch -from prune_macos_sccache import BUDGET_BYTES, MAIN_REF, PILOT_REF, gh_api, list_caches, run, select_evictions +from prune_sccache import BUDGET_BYTES, MAIN_REF, gh_api, list_caches, run, select_evictions -def cache(cache_id, size, variant="a", **overrides): +def cache(cache_id, size, variant="a", os_name="macOS", arch="ARM64", **overrides): return { "id": cache_id, - "key": f"sccache-v1-macOS-ARM64-{variant * 64}-{cache_id:040x}", + "key": f"sccache-v1-{os_name}-{arch}-{variant * 64}-{cache_id:040x}", "ref": MAIN_REF, "size_in_bytes": size, "created_at": (datetime(2026, 11, 1, tzinfo=timezone.utc) + timedelta(minutes=cache_id)).isoformat(), @@ -21,10 +21,6 @@ def cache(cache_id, size, variant="a", **overrides): } -def pilot(cache_id, size): - return cache(cache_id, size, ref=PILOT_REF, key=f"macos-sccache-v2-arm64-arm64-Debug-{'a' * 64}-{'b' * 40}") - - def ids(plan): return [entry["id"] for entry in plan.deletions] @@ -33,38 +29,47 @@ def pages(entries): return json.dumps([{"actions_caches": entries}]) -class PruneMacosSccacheTest(unittest.TestCase): +class PruneSccacheTest(unittest.TestCase): def test_scope_excludes_other_namespaces_and_refs(self): entries = [ cache(1, 10, key="vcpkg-macos-cache"), - cache(2, 10, key=cache(2, 10)["key"].replace("macOS", "Linux")), - cache(3, 10, key=cache(3, 10)["key"].replace("macOS", "Windows")), + cache(2, 10, key="codeql-cache"), + cache(3, 10, key=cache(3, 10)["key"].replace("sccache-v1-", "sccache-v2-")), cache(4, 10, ref="refs/pull/42/merge"), cache(5, 10, ref="refs/heads/feature"), cache(6, 10, key="sccache-v1-macOS-unrecognized"), - {**pilot(7, 10), "ref": MAIN_REF}, - {**pilot(8, 10), "ref": "refs/heads/feature"}, + cache(7, 10, key=f"macos-sccache-v1-arm64-Debug-{'a' * 64}-{'b' * 40}"), + cache(8, 10, key=f"macos-sccache-v2-arm64-Debug-{'a' * 64}-{'b' * 40}"), cache(9, 10), - pilot(10, 10), + cache(10, 10, os_name="Linux", ref="refs/heads/feature"), ] - self.assertEqual(ids(select_evictions(entries, 0)), [10, 9]) + self.assertEqual(ids(select_evictions(entries, 0)), [9]) def test_empty_exact_and_under_budget_need_no_eviction(self): self.assertEqual(ids(select_evictions([], 0)), []) self.assertEqual(ids(select_evictions([cache(1, BUDGET_BYTES)])), []) self.assertEqual(ids(select_evictions([cache(1, 1)])), []) - def test_intel_and_both_pilot_versions_are_included(self): - intel = cache(1, 10, key=cache(1, 10)["key"].replace("ARM64", "X64")) - old_pilot = {**pilot(2, 10), "key": pilot(2, 10)["key"].replace("-v2-", "-v1-")} - self.assertEqual(ids(select_evictions([intel, old_pilot, pilot(3, 10)], 0)), [2, 3, 1]) + def test_all_os_arch_and_build_variants_share_one_budget(self): + entries = [ + cache(1, 10, os_name="macOS", arch="X64"), + cache(2, 10, os_name="Linux", arch="ARM64"), + cache(3, 10, os_name="Windows", arch="X64"), + cache(4, 10, variant="b", os_name="Linux", arch="ARM64"), + cache(5, 10, os_name="FutureOS", arch="FutureArch"), + ] + plan = select_evictions(entries, 30) + self.assertEqual(plan.total_bytes, 50) + self.assertEqual(plan.remaining_bytes, 30) + self.assertEqual(ids(plan), [1, 2]) + self.assertEqual([entry["id"] for entry in plan.evicted_baselines], [1, 2]) def test_aggregate_budget_preserves_newest_per_prefix(self): - entries = [cache(1, 8), cache(2, 8, "b"), cache(3, 8), cache(4, 8, "b"), pilot(5, 8)] - plan = select_evictions(entries[::-1], 24) - self.assertEqual(ids(plan), [5, 1]) - self.assertEqual(plan.total_bytes, 40) - self.assertEqual(plan.remaining_bytes, 24) + entries = [cache(1, 8), cache(2, 8, os_name="Linux"), cache(3, 8), cache(4, 8, os_name="Linux")] + plan = select_evictions(entries[::-1], 16) + self.assertEqual(ids(plan), [1, 2]) + self.assertEqual(plan.total_bytes, 32) + self.assertEqual(plan.remaining_bytes, 16) self.assertEqual(plan.evicted_baselines, []) def test_multiple_generations_and_prefixes_are_independent(self): @@ -102,20 +107,21 @@ def test_invalid_budget_and_scoped_metadata_fail_closed(self): select_evictions([cache(1, 1, **overrides)]) def test_pagination_and_duplicate_ids(self): - api = Mock(return_value=json.dumps([{"actions_caches": [cache(1, 1)]}, {"actions_caches": [cache(2, 1)]}])) + api = Mock( + return_value=json.dumps([{"actions_caches": [cache(1, 1)]}, {"actions_caches": [cache(1, 1), cache(2, 1)]}]) + ) self.assertEqual(len(list_caches("owner/repo", api)), 2) - self.assertEqual(api.call_count, 2) + self.assertEqual(api.call_count, 1) for call in api.call_args_list: self.assertIn("--paginate", call.args[0]) self.assertIn("--slurp", call.args[0]) - self.assertIn("per_page=100&key=sccache-v1-macOS-", api.call_args_list[0].args[0][-1]) - self.assertIn("per_page=100&key=macos-sccache-v", api.call_args_list[1].args[0][-1]) + self.assertIn("per_page=100&key=sccache-v1-", api.call_args_list[0].args[0][-1]) def test_dry_run_never_deletes_and_reports_baseline_loss(self): api = Mock(return_value=pages([cache(1, BUDGET_BYTES + 1)])) log = Mock() run("owner/repo", api=api, log=log) - self.assertEqual(api.call_count, 2) + self.assertEqual(api.call_count, 1) self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list)) self.assertTrue(any("::warning::" in call.args[0] for call in log.call_args_list)) self.assertTrue(any("Would delete cache 1" in call.args[0] for call in log.call_args_list)) @@ -123,12 +129,12 @@ def test_dry_run_never_deletes_and_reports_baseline_loss(self): def test_apply_deletes_exact_ids_and_rechecks_aggregate(self): before = pages([cache(1, BUDGET_BYTES), cache(2, 1)]) after = pages([cache(2, 1)]) - api = Mock(side_effect=[before, before, "", after, after]) + api = Mock(side_effect=[before, "", after]) run("owner/repo", apply=True, api=api, log=Mock()) - self.assertEqual(api.call_count, 5) - self.assertEqual(api.call_args_list[2].args[0], ["--method", "DELETE", "repos/owner/repo/actions/caches/1"]) - self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list[:2])) - self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list[3:])) + self.assertEqual(api.call_count, 3) + self.assertEqual(api.call_args_list[1].args[0], ["--method", "DELETE", "repos/owner/repo/actions/caches/1"]) + self.assertNotIn("DELETE", api.call_args_list[0].args[0]) + self.assertNotIn("DELETE", api.call_args_list[2].args[0]) def test_listing_malformed_response_and_delete_errors_propagate(self): with self.assertRaisesRegex(RuntimeError, "403"): @@ -138,27 +144,29 @@ def test_listing_malformed_response_and_delete_errors_propagate(self): list_caches("owner/repo", Mock(return_value=response)) before = pages([cache(1, BUDGET_BYTES + 1)]) with self.assertRaisesRegex(RuntimeError, "503"): - run("owner/repo", apply=True, api=Mock(side_effect=[before, before, RuntimeError("503")]), log=Mock()) + run("owner/repo", apply=True, api=Mock(side_effect=[before, RuntimeError("503")]), log=Mock()) - def test_second_namespace_failure_prevents_deletion(self): - api = Mock(side_effect=[pages([cache(1, BUDGET_BYTES + 1)]), RuntimeError("Second namespace failed")]) - with self.assertRaisesRegex(RuntimeError, "Second namespace failed"): + def test_later_page_failure_prevents_deletion(self): + # gh --paginate fails the entire command if any page request fails. + error = subprocess.CalledProcessError(1, ["gh", "api"], output=pages([cache(1, BUDGET_BYTES + 1)])) + api = Mock(side_effect=error) + with self.assertRaises(subprocess.CalledProcessError): run("owner/repo", apply=True, api=api, log=Mock()) - self.assertEqual(api.call_count, 2) + self.assertEqual(api.call_count, 1) self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list)) def test_concurrent_uploads_fail_post_cleanup_check(self): before = pages([cache(1, BUDGET_BYTES + 1)]) with self.assertRaisesRegex(RuntimeError, "remains over budget"): - run("owner/repo", apply=True, api=Mock(side_effect=[before, before, "", before, before]), log=Mock()) + run("owner/repo", apply=True, api=Mock(side_effect=[before, "", before]), log=Mock()) def test_gh_api_invocation_and_process_errors(self): - with patch("prune_macos_sccache.subprocess.check_output", return_value="[]") as check_output: + with patch("prune_sccache.subprocess.check_output", return_value="[]") as check_output: self.assertEqual(gh_api(["--paginate"]), "[]") check_output.assert_called_once_with(["gh", "api", "--paginate"], text=True, encoding="utf-8") with ( patch( - "prune_macos_sccache.subprocess.check_output", + "prune_sccache.subprocess.check_output", side_effect=subprocess.CalledProcessError(1, ["gh", "api"]), ), self.assertRaises(subprocess.CalledProcessError), diff --git a/.github/workflows/macos-sccache-maintenance.yml b/.github/workflows/sccache-maintenance.yml similarity index 64% rename from .github/workflows/macos-sccache-maintenance.yml rename to .github/workflows/sccache-maintenance.yml index 86d9364dc8f42..8f1671d2e4407 100644 --- a/.github/workflows/macos-sccache-maintenance.yml +++ b/.github/workflows/sccache-maintenance.yml @@ -1,10 +1,6 @@ -name: macOS sccache storage maintenance +name: sccache storage maintenance on: - workflow_run: - workflows: ["MacOS CI Pipeline"] - types: [completed] - branches: [main] schedule: - cron: '37 * * * *' workflow_dispatch: @@ -15,8 +11,8 @@ on: default: true pull_request: paths: - - '.github/scripts/*prune_macos_sccache.py' - - '.github/workflows/macos-sccache-maintenance.yml' + - '.github/scripts/*prune_sccache.py' + - '.github/workflows/sccache-maintenance.yml' permissions: contents: read @@ -29,34 +25,29 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # Never execute the triggering workflow's checkout in a privileged workflow_run. ref: ${{ github.event_name == 'pull_request' && github.ref || 'main' }} - - run: python -m unittest discover -s .github/scripts -p test_prune_macos_sccache.py -v + - run: python -m unittest discover -s .github/scripts -p test_prune_sccache.py -v cleanup: needs: test if: >- github.repository == 'microsoft/onnxruntime' && github.ref == 'refs/heads/main' && - (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || - (github.event_name == 'workflow_run' && - github.event.workflow_run.event == 'push' && - github.event.workflow_run.head_branch == 'main' && - github.event.workflow_run.head_repository.full_name == github.repository)) + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read actions: write concurrency: - group: macos-sccache-storage-maintenance + group: sccache-storage-maintenance cancel-in-progress: false steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: main persist-credentials: false - - name: Enforce archived macOS compiler-cache budget + - name: Enforce shared archived compiler-cache budget env: GH_TOKEN: ${{ github.token }} DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} @@ -67,4 +58,4 @@ jobs: if [[ "$DRY_RUN" != "true" ]]; then args+=(--apply) fi - python .github/scripts/prune_macos_sccache.py "${args[@]}" + python .github/scripts/prune_sccache.py "${args[@]}" From 858e595cd3d7f1b22205d68d1e1d8ef7c1112b3d Mon Sep 17 00:00:00 2001 From: Erik Scorelle Date: Wed, 4 Nov 2026 10:23:19 -0500 Subject: [PATCH 4/7] Apply the 30 GB sccache storage budget independently per OS Aggregate architectures and build variants within each OS, preserve independent OS budgets during eviction and post-cleanup checks, and report per-OS storage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-sccache/README.md | 16 +++++---- .github/scripts/prune_sccache.py | 38 ++++++++++++++------ .github/scripts/test_prune_sccache.py | 42 ++++++++++++++++++++--- .github/workflows/sccache-maintenance.yml | 2 +- 4 files changed, 76 insertions(+), 22 deletions(-) diff --git a/.github/actions/setup-sccache/README.md b/.github/actions/setup-sccache/README.md index a86d043e283f8..fa94aeff53a9a 100644 --- a/.github/actions/setup-sccache/README.md +++ b/.github/actions/setup-sccache/README.md @@ -40,13 +40,15 @@ The actions do not select a CMake generator or enable compiler launchers themsel `--use_cache --cache_tool sccache` in build flags. Windows launcher support in `build.py` currently requires Ninja; validate the selected compiler/debug-symbol configuration when adopting it. -## Shared archived-storage budget +## Per-OS archived-storage budgets -`sccache-maintenance.yml` enforces one **30 GB decimal (30,000,000,000 bytes) aggregate** -budget for GitHub's archived sccache snapshots across all operating systems, architectures, -and build variants using these shared actions, separate from the unchanged per-run `10G` +`sccache-maintenance.yml` currently enforces a separate **30 GB decimal (30,000,000,000 bytes)** +budget per OS for GitHub's archived sccache snapshots. Each OS's architectures and build variants +using these shared actions share that OS's budget, separate from the unchanged per-run `10G` local cache. It runs hourly, independently of any particular build workflow, so new callers participate automatically without adding workflow names. GitHub may delay scheduled runs. +Unused space in one OS's budget does not increase another's; total managed storage can exceed +30 GB when multiple OSes participate. Logs report each OS's current and planned storage. The budget includes well-formed `sccache-v1----` snapshots on `refs/heads/main`. OS and architecture identifiers are not restricted to the current runners. @@ -55,13 +57,13 @@ untouched and are not counted in this budget. New key formats require updating t Cleanup inventories all pages in the shared namespace, then removes older main generations oldest-first, preferentially retaining the newest snapshot per compatibility -prefix. If those baselines alone exceed the budget, it warns and removes the oldest baselines +prefix. If an OS's baselines alone exceed its budget, it warns and removes its oldest baselines too (including a single oversized snapshot), rather than silently exceeding the limit. Exact immutable cache IDs are deleted, and API errors fail the maintenance job. This is a **soft cap**, not an atomic GitHub storage quota: cleanup jobs serialize with each other, -not with cache uploads. Storage can exceed 30 GB between maintenance runs or during concurrent -publication. Cleanup rechecks storage afterward and fails visibly if still over budget; the next +not with cache uploads. Each OS's storage can exceed 30 GB between maintenance runs or during concurrent +publication. Cleanup rechecks every OS afterward and fails visibly if any is still over budget; the next run retries. Cache eviction may cause a cold build but does not change build/test coverage. Only the separate trusted-main cleanup job has `actions: write`. PRs run regression tests with diff --git a/.github/scripts/prune_sccache.py b/.github/scripts/prune_sccache.py index 1bbfaa49d1a4b..f8ae3a5741399 100644 --- a/.github/scripts/prune_sccache.py +++ b/.github/scripts/prune_sccache.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -"""Enforce the shared archived sccache budget; default to a read-only preview.""" +"""Enforce per-OS archived sccache budgets; default to a read-only preview.""" import argparse import json @@ -33,6 +33,8 @@ class EvictionPlan: remaining_bytes: int deletions: list[Cache] evicted_baselines: list[Cache] + total_bytes_by_os: dict[str, int] + remaining_bytes_by_os: dict[str, int] def compatibility(cache: Cache) -> str | None: @@ -78,16 +80,25 @@ def select_evictions(inventory: Sequence[Cache], budget: int = BUDGET_BYTES) -> # Retire older generations before evicting any compatibility prefix's newest baseline. candidates = sorted(caches, key=lambda cache: (cache["id"] in baselines, creation_time(cache), cache["id"])) - total_bytes = sum(cache["size_in_bytes"] for cache in caches) - remaining_bytes = total_bytes + total_bytes_by_os: dict[str, int] = {} + for cache in caches: + os_name = cache["key"].split("-")[2] + total_bytes_by_os[os_name] = total_bytes_by_os.get(os_name, 0) + cache["size_in_bytes"] + remaining_bytes_by_os = total_bytes_by_os.copy() deletions = [] for cache in candidates: - if remaining_bytes <= budget: - break + os_name = cache["key"].split("-")[2] + if remaining_bytes_by_os[os_name] <= budget: + continue deletions.append(cache) - remaining_bytes -= cache["size_in_bytes"] + remaining_bytes_by_os[os_name] -= cache["size_in_bytes"] return EvictionPlan( - total_bytes, remaining_bytes, deletions, [cache for cache in deletions if cache["id"] in baselines] + sum(total_bytes_by_os.values()), + sum(remaining_bytes_by_os.values()), + deletions, + [cache for cache in deletions if cache["id"] in baselines], + total_bytes_by_os, + remaining_bytes_by_os, ) @@ -121,9 +132,14 @@ def run( raise ValueError("Expected repository in owner/name format") plan = select_evictions(list_caches(repository, api)) log( - f"{'Apply' if apply else 'Dry run'}: {plan.total_bytes} bytes; budget {BUDGET_BYTES}; " + f"{'Apply' if apply else 'Dry run'}: {plan.total_bytes} bytes; budget {BUDGET_BYTES} per OS; " f"{len(plan.deletions)} evictions; planned remainder {plan.remaining_bytes} bytes" ) + for os_name in sorted(plan.total_bytes_by_os): + log( + f"{os_name}: {plan.total_bytes_by_os[os_name]} bytes; " + f"planned remainder {plan.remaining_bytes_by_os[os_name]} bytes" + ) if plan.evicted_baselines: log(f"::warning::Budget requires evicting {len(plan.evicted_baselines)} newest compatibility baselines") for cache in plan.deletions: @@ -139,9 +155,11 @@ def run( if apply: observed = select_evictions(list_caches(repository, api)) log(f"Observed storage after cleanup: {observed.total_bytes} bytes") - if observed.total_bytes > BUDGET_BYTES: + over_budget = [os_name for os_name, size in observed.total_bytes_by_os.items() if size > BUDGET_BYTES] + if over_budget: raise RuntimeError( - "sccache remains over budget (possibly concurrent uploads); next maintenance run will retry" + f"sccache remains over budget for {', '.join(sorted(over_budget))} " + "(possibly concurrent uploads); next maintenance run will retry" ) return plan diff --git a/.github/scripts/test_prune_sccache.py b/.github/scripts/test_prune_sccache.py index 28d9c9f58e9ae..3056c9365fc51 100644 --- a/.github/scripts/test_prune_sccache.py +++ b/.github/scripts/test_prune_sccache.py @@ -50,7 +50,7 @@ def test_empty_exact_and_under_budget_need_no_eviction(self): self.assertEqual(ids(select_evictions([cache(1, BUDGET_BYTES)])), []) self.assertEqual(ids(select_evictions([cache(1, 1)])), []) - def test_all_os_arch_and_build_variants_share_one_budget(self): + def test_each_os_has_an_independent_budget(self): entries = [ cache(1, 10, os_name="macOS", arch="X64"), cache(2, 10, os_name="Linux", arch="ARM64"), @@ -60,13 +60,38 @@ def test_all_os_arch_and_build_variants_share_one_budget(self): ] plan = select_evictions(entries, 30) self.assertEqual(plan.total_bytes, 50) - self.assertEqual(plan.remaining_bytes, 30) + self.assertEqual(plan.remaining_bytes, 50) + self.assertEqual(ids(plan), []) + self.assertEqual(plan.total_bytes_by_os, {"macOS": 10, "Linux": 20, "Windows": 10, "FutureOS": 10}) + + def test_os_budget_combines_architectures_and_build_variants(self): + entries = [ + cache(1, 20, os_name="Linux", arch="X64"), + cache(2, 20, variant="b", os_name="Linux", arch="ARM64"), + cache(3, 10, os_name="Windows"), + cache(4, 10, os_name="macOS"), + ] + plan = select_evictions(entries, 30) + self.assertEqual(ids(plan), [1]) + self.assertEqual(plan.remaining_bytes_by_os, {"Linux": 20, "Windows": 10, "macOS": 10}) + self.assertEqual([entry["id"] for entry in plan.evicted_baselines], [1]) + + def test_each_over_budget_os_is_evicted_independently(self): + entries = [ + cache(1, 20, os_name="Linux"), + cache(2, 20, os_name="Windows"), + cache(3, 20, os_name="Linux"), + cache(4, 20, os_name="Windows"), + cache(5, 1, os_name="macOS"), + ] + plan = select_evictions(entries, 30) self.assertEqual(ids(plan), [1, 2]) - self.assertEqual([entry["id"] for entry in plan.evicted_baselines], [1, 2]) + self.assertEqual(plan.remaining_bytes_by_os, {"Linux": 20, "Windows": 20, "macOS": 1}) + self.assertEqual(plan.evicted_baselines, []) def test_aggregate_budget_preserves_newest_per_prefix(self): entries = [cache(1, 8), cache(2, 8, os_name="Linux"), cache(3, 8), cache(4, 8, os_name="Linux")] - plan = select_evictions(entries[::-1], 16) + plan = select_evictions(entries[::-1], 8) self.assertEqual(ids(plan), [1, 2]) self.assertEqual(plan.total_bytes, 32) self.assertEqual(plan.remaining_bytes, 16) @@ -160,6 +185,15 @@ def test_concurrent_uploads_fail_post_cleanup_check(self): with self.assertRaisesRegex(RuntimeError, "remains over budget"): run("owner/repo", apply=True, api=Mock(side_effect=[before, "", before]), log=Mock()) + def test_post_cleanup_allows_combined_storage_above_one_os_budget(self): + inventory = pages([cache(1, BUDGET_BYTES), cache(2, BUDGET_BYTES, os_name="Linux")]) + api = Mock(return_value=inventory) + plan = run("owner/repo", apply=True, api=api, log=Mock()) + self.assertEqual(plan.total_bytes, 2 * BUDGET_BYTES) + self.assertEqual(ids(plan), []) + self.assertEqual(api.call_count, 2) + self.assertTrue(all("DELETE" not in call.args[0] for call in api.call_args_list)) + def test_gh_api_invocation_and_process_errors(self): with patch("prune_sccache.subprocess.check_output", return_value="[]") as check_output: self.assertEqual(gh_api(["--paginate"]), "[]") diff --git a/.github/workflows/sccache-maintenance.yml b/.github/workflows/sccache-maintenance.yml index 8f1671d2e4407..7c436f112add4 100644 --- a/.github/workflows/sccache-maintenance.yml +++ b/.github/workflows/sccache-maintenance.yml @@ -47,7 +47,7 @@ jobs: with: ref: main persist-credentials: false - - name: Enforce shared archived compiler-cache budget + - name: Enforce per-OS archived compiler-cache budgets env: GH_TOKEN: ${{ github.token }} DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }} From 2b5afe04d2e6f662d98599a9152a4864c1cccb76 Mon Sep 17 00:00:00 2001 From: Erik Scorelle Date: Wed, 4 Nov 2026 10:24:44 -0500 Subject: [PATCH 5/7] Raise archived sccache budget to 150 GB per OS Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/actions/setup-sccache/README.md | 6 +++--- .github/scripts/prune_sccache.py | 2 +- .github/scripts/test_prune_sccache.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup-sccache/README.md b/.github/actions/setup-sccache/README.md index fa94aeff53a9a..8ff611c33231b 100644 --- a/.github/actions/setup-sccache/README.md +++ b/.github/actions/setup-sccache/README.md @@ -42,13 +42,13 @@ requires Ninja; validate the selected compiler/debug-symbol configuration when a ## Per-OS archived-storage budgets -`sccache-maintenance.yml` currently enforces a separate **30 GB decimal (30,000,000,000 bytes)** +`sccache-maintenance.yml` currently enforces a separate **150 GB decimal (150,000,000,000 bytes)** budget per OS for GitHub's archived sccache snapshots. Each OS's architectures and build variants using these shared actions share that OS's budget, separate from the unchanged per-run `10G` local cache. It runs hourly, independently of any particular build workflow, so new callers participate automatically without adding workflow names. GitHub may delay scheduled runs. Unused space in one OS's budget does not increase another's; total managed storage can exceed -30 GB when multiple OSes participate. Logs report each OS's current and planned storage. +150 GB when multiple OSes participate. Logs report each OS's current and planned storage. The budget includes well-formed `sccache-v1----` snapshots on `refs/heads/main`. OS and architecture identifiers are not restricted to the current runners. @@ -62,7 +62,7 @@ too (including a single oversized snapshot), rather than silently exceeding the Exact immutable cache IDs are deleted, and API errors fail the maintenance job. This is a **soft cap**, not an atomic GitHub storage quota: cleanup jobs serialize with each other, -not with cache uploads. Each OS's storage can exceed 30 GB between maintenance runs or during concurrent +not with cache uploads. Each OS's storage can exceed 150 GB between maintenance runs or during concurrent publication. Cleanup rechecks every OS afterward and fails visibly if any is still over budget; the next run retries. Cache eviction may cause a cold build but does not change build/test coverage. diff --git a/.github/scripts/prune_sccache.py b/.github/scripts/prune_sccache.py index f8ae3a5741399..36538ed48de5a 100644 --- a/.github/scripts/prune_sccache.py +++ b/.github/scripts/prune_sccache.py @@ -13,7 +13,7 @@ from datetime import datetime from typing import TypedDict -BUDGET_BYTES = 30_000_000_000 +BUDGET_BYTES = 150_000_000_000 MAIN_REF = "refs/heads/main" KEY_PREFIX = "sccache-v1-" MAIN_KEY = re.compile(r"(sccache-v1-[A-Za-z0-9_]+-[A-Za-z0-9_]+-[a-f0-9]{64}-)[a-f0-9]{40}") diff --git a/.github/scripts/test_prune_sccache.py b/.github/scripts/test_prune_sccache.py index 3056c9365fc51..6dc88f6d74b5f 100644 --- a/.github/scripts/test_prune_sccache.py +++ b/.github/scripts/test_prune_sccache.py @@ -46,6 +46,7 @@ def test_scope_excludes_other_namespaces_and_refs(self): self.assertEqual(ids(select_evictions(entries, 0)), [9]) def test_empty_exact_and_under_budget_need_no_eviction(self): + self.assertEqual(BUDGET_BYTES, 150_000_000_000) self.assertEqual(ids(select_evictions([], 0)), []) self.assertEqual(ids(select_evictions([cache(1, BUDGET_BYTES)])), []) self.assertEqual(ids(select_evictions([cache(1, 1)])), []) @@ -98,7 +99,7 @@ def test_aggregate_budget_preserves_newest_per_prefix(self): self.assertEqual(plan.evicted_baselines, []) def test_multiple_generations_and_prefixes_are_independent(self): - entries = [cache(i + 1, 1_000_000_000, f"{i % 10:x}") for i in range(170)] + entries = [cache(i + 1, BUDGET_BYTES // 30, f"{i % 10:x}") for i in range(170)] plan = select_evictions(entries) self.assertEqual(len(plan.deletions), 140) self.assertEqual(plan.remaining_bytes, BUDGET_BYTES) From c7f6cbc946e9350b577b976e331f40bbc7ff64f2 Mon Sep 17 00:00:00 2001 From: Erik Scorelle Date: Wed, 4 Nov 2026 10:30:56 -0500 Subject: [PATCH 6/7] Explain sccache compatibility baseline selection Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/scripts/prune_sccache.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/prune_sccache.py b/.github/scripts/prune_sccache.py index 36538ed48de5a..3662f789b1296 100644 --- a/.github/scripts/prune_sccache.py +++ b/.github/scripts/prune_sccache.py @@ -72,6 +72,8 @@ def select_evictions(inventory: Sequence[Cache], budget: int = BUDGET_BYTES) -> caches.sort(key=lambda cache: (creation_time(cache), cache["id"]), reverse=True) baselines = set() prefixes = set() + # Newest-first ordering makes the first snapshot per OS/arch/build prefix its preferred baseline. + # Baselines are evicted only if older generations cannot free enough space within that OS's budget. for cache in caches: prefix = compatibility(cache) if prefix and prefix not in prefixes: From bbf37057931e9fc43dba57c249448321e15ef794 Mon Sep 17 00:00:00 2001 From: eserscor Date: Thu, 5 Nov 2026 13:33:24 -0500 Subject: [PATCH 7/7] Improve cache listing in prune_sccache.py Refactor cache listing to handle pagination and sorting. Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/scripts/prune_sccache.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/scripts/prune_sccache.py b/.github/scripts/prune_sccache.py index 3662f789b1296..2395da6f97ea0 100644 --- a/.github/scripts/prune_sccache.py +++ b/.github/scripts/prune_sccache.py @@ -111,7 +111,13 @@ def gh_api(args: list[str]) -> str: def list_caches(repository: str, api: Callable[[list[str]], str]) -> list[Cache]: caches = {} pages = json.loads( - api(["--paginate", "--slurp", f"repos/{repository}/actions/caches?per_page=100&key={KEY_PREFIX}"]) + api( + [ + "--paginate", + "--slurp", + f"repos/{repository}/actions/caches?per_page=100&key={KEY_PREFIX}&sort=created_at&direction=asc", + ] + ) ) if not isinstance(pages, list) or not pages: raise ValueError("Invalid cache pagination response")