From a6a2b0ff7cb97c6a46954370cd611d0dc5b25104 Mon Sep 17 00:00:00 2001 From: zozo123 Date: Sat, 3 Oct 2026 20:37:35 +0300 Subject: [PATCH 01/13] Restore the CI image from a snapshot of Docker's data directory Every job that prepares the CI image runs `docker image load` on the stash, which unpacks and checksums every layer of an 8 GB image again, about two minutes per job. Extracting a copy of the image store into a stopped daemon gives the same image in about half a minute on the same runners. The image stash stays as it is, for other consumers and as the fallback when a snapshot does not fit the daemon a job runs on. --- .../prepare_breeze_and_image/action.yml | 26 +++ .github/workflows/ci-image-build.yml | 73 ++++++--- scripts/ci/docker_data_root_snapshot.sh | 116 +++++++++++++ .../ci/test_docker_data_root_snapshot.py | 154 ++++++++++++++++++ 4 files changed, 345 insertions(+), 24 deletions(-) create mode 100755 scripts/ci/docker_data_root_snapshot.sh create mode 100644 scripts/tests/ci/test_docker_data_root_snapshot.py diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 6c91e28bff84d..2fb9518d81823 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -79,6 +79,30 @@ runs: run: | echo "Checking free space!" df -H + - name: "Restore CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" + # yamllint disable-line rule:line-length + uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0 + with: + key: "ci-image-snapshot-v1-${{ inputs.platform }}-${{ inputs.python }}" + path: "/mnt/" + only-current-branch: 'true' + id: restore-snapshot + if: inputs.image-type == 'ci' && inputs.image-stash-ref == '' + # Falls back to the image stash below whenever the snapshot does not fit this runner's daemon. + - name: "Unpack CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" + id: snapshot + env: + PLATFORM: ${{ inputs.platform }} + PYTHON: ${{ inputs.python }} + run: | + if ./scripts/ci/docker_data_root_snapshot.sh restore \ + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON}.tar.zst"; then + echo "restored=true" >> "${GITHUB_OUTPUT}" + else + echo "Falling back to loading the image stash" + fi + shell: bash + if: steps.restore-snapshot.outputs.stash-hit == 'true' - name: > Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }} ${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }} @@ -90,6 +114,7 @@ runs: path: "/mnt/" only-current-branch: 'true' fail-on-download: 'true' + if: steps.snapshot.outputs.restored != 'true' - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}" env: PLATFORM: ${{ inputs.platform }} @@ -98,3 +123,4 @@ runs: run: > breeze ${IMAGE_TYPE}-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt" shell: bash + if: steps.snapshot.outputs.restored != 'true' diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 1541337bc5c73..ad822692de14b 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -321,6 +321,55 @@ jobs: docker rmi "${CACHE_FROM_IMAGE}" shell: bash if: always() && env.CACHE_FROM_IMAGE != '' + - name: "Export mount cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} + run: > + breeze ci-image export-mount-cache + --cache-file /tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz + if: > + inputs.upload-mount-cache-artifact == 'true' && + steps.stashed-image.outputs.reusable != 'true' + - name: > + Stash cache mount ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} + ${{ inputs.image-stash-ref != '' && format('for ref {0}', inputs.image-stash-ref) || '' }} + uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 + with: + key: "ci-cache-mount-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ + ${{ inputs.image-stash-ref != '' && format('-{0}', inputs.image-stash-ref) || '' }}" + path: "/tmp/ci-cache-mount-save-v3-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz" + if-no-files-found: 'error' + # A ref's cache is read by the next publish of that same ref, days rather than hours + # later, so it gets the retention the ref's image gets rather than the branch's. + retention-days: ${{ inputs.image-stash-ref != '' && '6' || '2' }} + if: > + inputs.upload-mount-cache-artifact == 'true' && + steps.stashed-image.outputs.reusable != 'true' + # Every job that prepares the CI image would otherwise `docker image load` the stash below, + # unpacking and checksumming each layer again; a copy of the image store restores with one + # extraction. Taken while the freshly built layers are still in the page cache, and after the + # mount cache export, as it drops the build cache that the export reads. + - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + env: + PLATFORM: ${{ inputs.platform }} + run: > + ./scripts/ci/docker_data_root_snapshot.sh create + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" + shell: bash + if: > + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && + steps.stashed-image.outputs.reusable != 'true' + - name: "Stash CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 + with: + key: ci-image-snapshot-v1-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} + path: "/mnt/ci-image-snapshot-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.zst*" + if-no-files-found: 'error' + compression-level: '0' + retention-days: '2' + if: > + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && + steps.stashed-image.outputs.reusable != 'true' # zstd stands in for the stash's zlib (compression-level 0 below), which every consumer inflates. # The .tar name, which every Breeze expects, stays: docker image load detects zstd by content. - name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" @@ -375,29 +424,5 @@ jobs: if: > inputs.upload-image-artifact == 'true' && inputs.image-stash-ref != '' && steps.stashed-image.outputs.reusable != 'true' - - name: "Export mount cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - env: - PYTHON_MAJOR_MINOR_VERSION: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} - run: > - breeze ci-image export-mount-cache - --cache-file /tmp/ci-cache-mount-save-v3-${PYTHON_MAJOR_MINOR_VERSION}.tar.gz - if: > - inputs.upload-mount-cache-artifact == 'true' && - steps.stashed-image.outputs.reusable != 'true' - - name: > - Stash cache mount ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }} - ${{ inputs.image-stash-ref != '' && format('for ref {0}', inputs.image-stash-ref) || '' }} - uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 - with: - key: "ci-cache-mount-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ - ${{ inputs.image-stash-ref != '' && format('-{0}', inputs.image-stash-ref) || '' }}" - path: "/tmp/ci-cache-mount-save-v3-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.gz" - if-no-files-found: 'error' - # A ref's cache is read by the next publish of that same ref, days rather than hours - # later, so it gets the retention the ref's image gets rather than the branch's. - retention-days: ${{ inputs.image-stash-ref != '' && '6' || '2' }} - if: > - inputs.upload-mount-cache-artifact == 'true' && - steps.stashed-image.outputs.reusable != 'true' - name: "Check disk space after build" run: df -H diff --git a/scripts/ci/docker_data_root_snapshot.sh b/scripts/ci/docker_data_root_snapshot.sh new file mode 100755 index 0000000000000..c23699f1d3b83 --- /dev/null +++ b/scripts/ci/docker_data_root_snapshot.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Moves the CI image between jobs as a copy of the Docker data directory that holds it, so a job +# restores it with one extraction instead of `docker image load` unpacking and checksumming every +# layer again. +# +# docker_data_root_snapshot.sh create SNAPSHOT_FILE +# Keeps only the CI image in the daemon and writes its image store to SNAPSHOT_FILE, plus +# SNAPSHOT_FILE.meta naming the daemon that wrote it and the image. +# docker_data_root_snapshot.sh restore SNAPSHOT_FILE +# Restores the image. Exits non-zero, leaving the daemon running and without images, when +# the snapshot is missing or was written by a different daemon, or the daemon already holds +# images, so the caller can fall back to `docker image load`. +set -euo pipefail + +DATA_ROOT="/var/lib/docker" + +# A data directory is only readable by the daemon version and storage driver that wrote it. +function daemon_fingerprint() { + docker info --format '{{.ServerVersion}} {{.Driver}} {{.Architecture}} {{.DockerRootDir}}' +} + +function stop_daemon() { + sudo systemctl stop docker.socket docker +} + +function start_daemon() { + sudo systemctl start docker +} + +function remove_image_store() { + sudo rm -rf "${DATA_ROOT}/image" "${DATA_ROOT}/overlay2" +} + +function create_snapshot() { + local snapshot_file="${1}" + local image_id + image_id="$(docker images --quiet --filter "label=org.apache.airflow.image=airflow-ci" | sort -u)" + if [[ -z "${image_id}" || "${image_id}" == *$'\n'* ]]; then + echo "Expected exactly one CI image in the daemon, found: '${image_id}'" >&2 + exit 1 + fi + docker ps --all --quiet | xargs --no-run-if-empty docker rm --force >/dev/null + docker images --quiet | sort -u | { grep --invert-match --fixed-strings --line-regexp "${image_id}" || true; } \ + | xargs --no-run-if-empty docker rmi --force >/dev/null + docker builder prune --all --force >/dev/null + printf '%s %s\n' "$(daemon_fingerprint)" "${image_id}" > "${snapshot_file}.meta" + stop_daemon + sudo tar --directory "${DATA_ROOT}" --xattrs --acls --numeric-owner --create --file - image overlay2 \ + | zstd -3 -T0 --quiet --force -o "${snapshot_file}" + start_daemon +} + +function restore_snapshot() { + local snapshot_file="${1}" + local -a meta + local image_id snapshot_fingerprint daemon + if [[ ! -f "${snapshot_file}" || ! -f "${snapshot_file}.meta" ]]; then + echo "No snapshot at ${snapshot_file}" + exit 2 + fi + read -r -a meta < "${snapshot_file}.meta" + image_id="${meta[${#meta[@]}-1]}" + snapshot_fingerprint="${meta[*]:0:${#meta[@]}-1}" + daemon="$(daemon_fingerprint)" + if [[ "${snapshot_fingerprint}" != "${daemon}" ]]; then + echo "The snapshot was written by '${snapshot_fingerprint}', this daemon is '${daemon}'" + exit 3 + fi + if [[ -n "$(docker images --all --quiet)" ]]; then + echo "The daemon already holds images, which restoring the snapshot would drop" + exit 3 + fi + stop_daemon + remove_image_store + zstd -d -T0 --quiet --stdout "${snapshot_file}" \ + | sudo tar --directory "${DATA_ROOT}" --xattrs --acls --numeric-owner --extract --file - + start_daemon + if ! docker run --rm --entrypoint /bin/bash "${image_id}" -c true; then + echo "The restored image ${image_id} does not run" + stop_daemon + remove_image_store + start_daemon + exit 4 + fi + rm -f "${snapshot_file}" "${snapshot_file}.meta" +} + +case "${1:-}" in + create) + create_snapshot "${2}" + ;; + restore) + restore_snapshot "${2}" + ;; + *) + echo "Usage: ${0} create|restore SNAPSHOT_FILE" >&2 + exit 1 + ;; +esac diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py b/scripts/tests/ci/test_docker_data_root_snapshot.py new file mode 100644 index 0000000000000..3b114aa6a5993 --- /dev/null +++ b/scripts/tests/ci/test_docker_data_root_snapshot.py @@ -0,0 +1,154 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import shlex +import subprocess +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[2] / "ci" / "docker_data_root_snapshot.sh" +FINGERPRINT = "28.5.2 overlay2 x86_64 /var/lib/docker" + + +@pytest.fixture +def fake_tools(tmp_path): + """Shadow ``docker``, ``sudo`` and ``zstd`` with a logger answering the queries the script makes.""" + tools = tmp_path / "bin" + tools.mkdir() + command = tools / "command" + command.write_text( + r"""#!/usr/bin/env bash +name="${0##*/}" +printf '%s\n' "${name} $*" >> "${COMMAND_LOG}" +if [[ "${name}" == "docker" ]]; then + case "$*" in + "info --format "*) echo "${DAEMON_FINGERPRINT}" ;; + "images --quiet --filter label=org.apache.airflow.image=airflow-ci") printf '%b' "${CI_IMAGES:-}" ;; + "images --quiet") printf '%b' "${ALL_IMAGES:-}" ;; + "images --all --quiet") printf '%b' "${ALL_IMAGES:-}" ;; + "run "*) exit "${RUN_EXIT:-0}" ;; + esac +fi +""" + ) + command.chmod(0o755) + for name in ("docker", "sudo", "zstd"): + (tools / name).symlink_to(command) + return { + "PATH": f"{tools}:{os.environ['PATH']}", + "COMMAND_LOG": str(tmp_path / "commands.log"), + "DAEMON_FINGERPRINT": FINGERPRINT, + } + + +@pytest.fixture +def snapshot(tmp_path): + snapshot_file = tmp_path / "ci-image-snapshot-linux_amd64-3.10.tar.zst" + snapshot_file.write_bytes(b"") + Path(f"{snapshot_file}.meta").write_text(f"{FINGERPRINT} abc123\n") + return snapshot_file + + +def run_script(env, *args): + return subprocess.run( + ["bash", "--noprofile", "--norc", str(SCRIPT), *args], + env={**os.environ, **env}, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + + +def read_commands(env): + log = Path(env["COMMAND_LOG"]) + return [shlex.split(line) for line in log.read_text().splitlines()] if log.exists() else [] + + +def test_restore_unpacks_snapshot_into_stopped_daemon(fake_tools, snapshot): + result = run_script(fake_tools, "restore", str(snapshot)) + + assert result.returncode == 0, result.stdout + result.stderr + commands = read_commands(fake_tools) + stop = commands.index(["sudo", "systemctl", "stop", "docker.socket", "docker"]) + remove = commands.index(["sudo", "rm", "-rf", "/var/lib/docker/image", "/var/lib/docker/overlay2"]) + extract = next(i for i, c in enumerate(commands) if c[:2] == ["sudo", "tar"] and "--extract" in c) + start = commands.index(["sudo", "systemctl", "start", "docker"]) + run = commands.index(["docker", "run", "--rm", "--entrypoint", "/bin/bash", "abc123", "-c", "true"]) + assert stop < remove < extract < start < run + assert not snapshot.exists() + + +@pytest.mark.parametrize( + ("env", "remove_meta", "expected_exit"), + [ + pytest.param({}, True, 2, id="no-snapshot"), + pytest.param( + {"DAEMON_FINGERPRINT": "29.0.0 overlay2 x86_64 /var/lib/docker"}, False, 3, id="other-daemon" + ), + pytest.param({"ALL_IMAGES": "def456\\n"}, False, 3, id="daemon-holds-images"), + ], +) +def test_restore_leaves_daemon_alone_when_snapshot_does_not_fit( + fake_tools, snapshot, env, remove_meta, expected_exit +): + if remove_meta: + Path(f"{snapshot}.meta").unlink() + + result = run_script({**fake_tools, **env}, "restore", str(snapshot)) + + assert result.returncode == expected_exit + assert not [c for c in read_commands(fake_tools) if c[:2] == ["sudo", "systemctl"]] + + +def test_restore_empties_store_again_when_image_does_not_run(fake_tools, snapshot): + result = run_script({**fake_tools, "RUN_EXIT": "1"}, "restore", str(snapshot)) + + assert result.returncode == 4 + commands = read_commands(fake_tools) + run = next(i for i, c in enumerate(commands) if c[:2] == ["docker", "run"]) + after_run = commands[run + 1 :] + assert ["sudo", "rm", "-rf", "/var/lib/docker/image", "/var/lib/docker/overlay2"] in after_run + assert after_run[-1] == ["sudo", "systemctl", "start", "docker"] + + +def test_create_keeps_only_the_ci_image(fake_tools, tmp_path): + snapshot_file = tmp_path / "out.tar.zst" + env = {**fake_tools, "CI_IMAGES": "abc123\\nabc123\\n", "ALL_IMAGES": "abc123\\nother1\\nabc123\\n"} + + result = run_script(env, "create", str(snapshot_file)) + + assert result.returncode == 0, result.stdout + result.stderr + commands = read_commands(env) + assert ["docker", "rmi", "--force", "other1"] in commands + assert not [c for c in commands if c[:2] == ["docker", "rmi"] and "abc123" in c] + assert ["docker", "builder", "prune", "--all", "--force"] in commands + assert Path(f"{snapshot_file}.meta").read_text() == f"{FINGERPRINT} abc123\n" + archive = next(c for c in commands if c[:2] == ["sudo", "tar"]) + assert archive[-2:] == ["image", "overlay2"] + + +def test_create_refuses_ambiguous_ci_image(fake_tools, tmp_path): + env = {**fake_tools, "CI_IMAGES": "abc123\\ndef456\\n"} + + result = run_script(env, "create", str(tmp_path / "out.tar.zst")) + + assert result.returncode == 1 + assert not [c for c in read_commands(env) if c[:2] == ["sudo", "systemctl"]] From 6be595dd236daa09c2ced695f2f028795461c83a Mon Sep 17 00:00:00 2001 From: Joe El Date: Sat, 3 Oct 2026 22:39:37 +0000 Subject: [PATCH 02/13] Recover Docker safely when CI snapshot materialization fails Snapshot failures must not prevent the authoritative stash from loading, and an older branch snapshot must not substitute an image for the current checkout. --- .../prepare_breeze_and_image/action.yml | 1 + .github/workflows/ci-image-build.yml | 7 ++ scripts/ci/docker_data_root_snapshot.sh | 51 ++++++++++++-- .../ci/test_docker_data_root_snapshot.py | 69 ++++++++++++++++++- 4 files changed, 118 insertions(+), 10 deletions(-) diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 2fb9518d81823..a4f9182d1592c 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -80,6 +80,7 @@ runs: echo "Checking free space!" df -H - name: "Restore CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" + continue-on-error: true # yamllint disable-line rule:line-length uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0 with: diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index ad822692de14b..06e4fbd496da2 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -349,7 +349,11 @@ jobs: # unpacking and checksumming each layer again; a copy of the image store restores with one # extraction. Taken while the freshly built layers are still in the page cache, and after the # mount cache export, as it drops the build cache that the export reads. + # Only PR-scoped snapshots: arbitrary checkout refs in privileged dispatch/scheduled + # workflows must never publish a snapshot into the default branch's cache scope. - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + id: snapshot-export + continue-on-error: true env: PLATFORM: ${{ inputs.platform }} run: > @@ -357,9 +361,11 @@ jobs: "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" shell: bash if: > + github.event_name == 'pull_request' && inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && steps.stashed-image.outputs.reusable != 'true' - name: "Stash CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + continue-on-error: true uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 with: key: ci-image-snapshot-v1-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} @@ -368,6 +374,7 @@ jobs: compression-level: '0' retention-days: '2' if: > + github.event_name == 'pull_request' && steps.snapshot-export.outcome == 'success' && inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && steps.stashed-image.outputs.reusable != 'true' # zstd stands in for the stash's zlib (compression-level 0 below), which every consumer inflates. diff --git a/scripts/ci/docker_data_root_snapshot.sh b/scripts/ci/docker_data_root_snapshot.sh index c23699f1d3b83..4b039515ce4d8 100755 --- a/scripts/ci/docker_data_root_snapshot.sh +++ b/scripts/ci/docker_data_root_snapshot.sh @@ -30,6 +30,32 @@ set -euo pipefail DATA_ROOT="/var/lib/docker" +DAEMON_STOPPED=false +RESTORE_IN_PROGRESS=false + +function cleanup_daemon() { + local result=$? + trap - EXIT + if [[ "${RESTORE_IN_PROGRESS}" == true ]]; then + stop_daemon + remove_image_store + start_daemon + elif [[ "${DAEMON_STOPPED}" == true ]]; then + start_daemon + fi + exit "${result}" +} +trap cleanup_daemon EXIT + +function check_supported_daemon() { + local daemon=$1 + local _ driver root + read -r _ driver _ root <<< "${daemon}" + if [[ "${driver}" != overlay2 || "${root}" != "${DATA_ROOT}" ]]; then + echo "Unsupported Docker image store: ${daemon}" + exit 3 + fi +} # A data directory is only readable by the daemon version and storage driver that wrote it. function daemon_fingerprint() { @@ -38,10 +64,12 @@ function daemon_fingerprint() { function stop_daemon() { sudo systemctl stop docker.socket docker + DAEMON_STOPPED=true } function start_daemon() { sudo systemctl start docker + DAEMON_STOPPED=false } function remove_image_store() { @@ -51,6 +79,7 @@ function remove_image_store() { function create_snapshot() { local snapshot_file="${1}" local image_id + check_supported_daemon "$(daemon_fingerprint)" image_id="$(docker images --quiet --filter "label=org.apache.airflow.image=airflow-ci" | sort -u)" if [[ -z "${image_id}" || "${image_id}" == *$'\n'* ]]; then echo "Expected exactly one CI image in the daemon, found: '${image_id}'" >&2 @@ -60,7 +89,7 @@ function create_snapshot() { docker images --quiet | sort -u | { grep --invert-match --fixed-strings --line-regexp "${image_id}" || true; } \ | xargs --no-run-if-empty docker rmi --force >/dev/null docker builder prune --all --force >/dev/null - printf '%s %s\n' "$(daemon_fingerprint)" "${image_id}" > "${snapshot_file}.meta" + printf '%s %s %s\n' "$(daemon_fingerprint)" "${image_id}" "$(git rev-parse HEAD)" > "${snapshot_file}.meta" stop_daemon sudo tar --directory "${DATA_ROOT}" --xattrs --acls --numeric-owner --create --file - image overlay2 \ | zstd -3 -T0 --quiet --force -o "${snapshot_file}" @@ -76,29 +105,37 @@ function restore_snapshot() { exit 2 fi read -r -a meta < "${snapshot_file}.meta" - image_id="${meta[${#meta[@]}-1]}" - snapshot_fingerprint="${meta[*]:0:${#meta[@]}-1}" + if [[ ${#meta[@]} != 6 ]]; then + echo "Invalid snapshot metadata" + exit 3 + fi + if [[ "${meta[5]}" != "$(git rev-parse HEAD)" ]]; then + echo "Snapshot belongs to a different checkout revision" + exit 3 + fi + image_id="${meta[4]}" + snapshot_fingerprint="${meta[*]:0:4}" daemon="$(daemon_fingerprint)" + check_supported_daemon "${daemon}" if [[ "${snapshot_fingerprint}" != "${daemon}" ]]; then echo "The snapshot was written by '${snapshot_fingerprint}', this daemon is '${daemon}'" exit 3 fi - if [[ -n "$(docker images --all --quiet)" ]]; then + if [[ -n "$(docker images --all --quiet)" || -n "$(docker ps --all --quiet)" ]]; then echo "The daemon already holds images, which restoring the snapshot would drop" exit 3 fi stop_daemon + RESTORE_IN_PROGRESS=true remove_image_store zstd -d -T0 --quiet --stdout "${snapshot_file}" \ | sudo tar --directory "${DATA_ROOT}" --xattrs --acls --numeric-owner --extract --file - start_daemon if ! docker run --rm --entrypoint /bin/bash "${image_id}" -c true; then echo "The restored image ${image_id} does not run" - stop_daemon - remove_image_store - start_daemon exit 4 fi + RESTORE_IN_PROGRESS=false rm -f "${snapshot_file}" "${snapshot_file}.meta" } diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py b/scripts/tests/ci/test_docker_data_root_snapshot.py index 3b114aa6a5993..af5f3f3b6b7c6 100644 --- a/scripts/tests/ci/test_docker_data_root_snapshot.py +++ b/scripts/tests/ci/test_docker_data_root_snapshot.py @@ -37,19 +37,34 @@ def fake_tools(tmp_path): r"""#!/usr/bin/env bash name="${0##*/}" printf '%s\n' "${name} $*" >> "${COMMAND_LOG}" +if [[ "${name}" == "git" && "$*" == "rev-parse HEAD" ]]; then + echo "${CHECKOUT_SHA:-revision1}" +fi +if [[ "${name}" == "zstd" ]]; then + exit "${ZSTD_EXIT:-0}" +fi +if [[ "${name}" == "sudo" && "$*" == "tar "* ]]; then + exit "${TAR_EXIT:-0}" +fi +if [[ "${name}" == "sudo" && "$*" == "systemctl start docker" && "${START_FAIL_ONCE:-0}" == 1 ]]; then + if [[ $(grep -c 'sudo systemctl start docker' "${COMMAND_LOG}") == 1 ]]; then + exit 1 + fi +fi if [[ "${name}" == "docker" ]]; then case "$*" in "info --format "*) echo "${DAEMON_FINGERPRINT}" ;; "images --quiet --filter label=org.apache.airflow.image=airflow-ci") printf '%b' "${CI_IMAGES:-}" ;; "images --quiet") printf '%b' "${ALL_IMAGES:-}" ;; "images --all --quiet") printf '%b' "${ALL_IMAGES:-}" ;; + "ps --all --quiet") printf '%b' "${CONTAINERS:-}" ;; "run "*) exit "${RUN_EXIT:-0}" ;; esac fi """ ) command.chmod(0o755) - for name in ("docker", "sudo", "zstd"): + for name in ("docker", "sudo", "zstd", "git"): (tools / name).symlink_to(command) return { "PATH": f"{tools}:{os.environ['PATH']}", @@ -62,7 +77,7 @@ def fake_tools(tmp_path): def snapshot(tmp_path): snapshot_file = tmp_path / "ci-image-snapshot-linux_amd64-3.10.tar.zst" snapshot_file.write_bytes(b"") - Path(f"{snapshot_file}.meta").write_text(f"{FINGERPRINT} abc123\n") + Path(f"{snapshot_file}.meta").write_text(f"{FINGERPRINT} abc123 revision1\n") return snapshot_file @@ -140,7 +155,7 @@ def test_create_keeps_only_the_ci_image(fake_tools, tmp_path): assert ["docker", "rmi", "--force", "other1"] in commands assert not [c for c in commands if c[:2] == ["docker", "rmi"] and "abc123" in c] assert ["docker", "builder", "prune", "--all", "--force"] in commands - assert Path(f"{snapshot_file}.meta").read_text() == f"{FINGERPRINT} abc123\n" + assert Path(f"{snapshot_file}.meta").read_text() == f"{FINGERPRINT} abc123 revision1\n" archive = next(c for c in commands if c[:2] == ["sudo", "tar"]) assert archive[-2:] == ["image", "overlay2"] @@ -152,3 +167,51 @@ def test_create_refuses_ambiguous_ci_image(fake_tools, tmp_path): assert result.returncode == 1 assert not [c for c in read_commands(env) if c[:2] == ["sudo", "systemctl"]] + + +@pytest.mark.parametrize("failure", [{"ZSTD_EXIT": "1"}, {"TAR_EXIT": "1"}, {"START_FAIL_ONCE": "1"}]) +def test_restore_recovers_empty_running_daemon_after_materialization_failure(fake_tools, snapshot, failure): + result = run_script({**fake_tools, **failure}, "restore", str(snapshot)) + assert result.returncode != 0 + commands = read_commands(fake_tools) + assert commands[-2:] == [ + ["sudo", "rm", "-rf", "/var/lib/docker/image", "/var/lib/docker/overlay2"], + ["sudo", "systemctl", "start", "docker"], + ] + assert not any(command[:2] == ["docker", "run"] for command in commands) + + +@pytest.mark.parametrize("failure", [{"ZSTD_EXIT": "1"}, {"TAR_EXIT": "1"}]) +def test_create_restarts_daemon_after_archive_failure(fake_tools, tmp_path, failure): + result = run_script( + {**fake_tools, **failure, "CI_IMAGES": "abc123"}, "create", str(tmp_path / "snapshot.tar.zst") + ) + assert result.returncode != 0 + assert read_commands(fake_tools)[-1] == ["sudo", "systemctl", "start", "docker"] + + +@pytest.mark.parametrize("mode", ["create", "restore"]) +@pytest.mark.parametrize("daemon", ["28.5.2 btrfs x86_64 /var/lib/docker", "28.5.2 overlay2 x86_64 /other"]) +def test_unsupported_storage_does_not_modify_daemon(fake_tools, snapshot, mode, daemon): + result = run_script({**fake_tools, "DAEMON_FINGERPRINT": daemon}, mode, str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +def test_existing_container_prevents_restore(fake_tools, snapshot): + result = run_script({**fake_tools, "CONTAINERS": "container1"}, "restore", str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +def test_malformed_metadata_prevents_restore(fake_tools, snapshot): + Path(f"{snapshot}.meta").write_text("invalid\n") + result = run_script(fake_tools, "restore", str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +def test_stale_snapshot_does_not_modify_daemon(fake_tools, snapshot): + result = run_script({**fake_tools, "CHECKOUT_SHA": "revision2"}, "restore", str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) From 9b5b2127a0c1e68911333ae795a4b5eb6329ee95 Mon Sep 17 00:00:00 2001 From: Joe El Date: Sat, 3 Oct 2026 22:43:10 +0000 Subject: [PATCH 03/13] Scope CI snapshots to the current workflow run Raw daemon snapshots are job handoffs, so publishing them into branch caches unnecessarily exposes later runs to arbitrary checkout refs. --- .github/actions/prepare_breeze_and_image/action.yml | 10 +++++----- .github/workflows/ci-image-build.yml | 12 +++++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index a4f9182d1592c..e49246b3f6766 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -81,12 +81,12 @@ runs: df -H - name: "Restore CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" continue-on-error: true - # yamllint disable-line rule:line-length - uses: apache/infrastructure-actions/stash/restore@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # restore/v1.0.0 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - key: "ci-image-snapshot-v1-${{ inputs.platform }}-${{ inputs.python }}" + name: >- + ci-image-snapshot-v2-${{ inputs.python }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} path: "/mnt/" - only-current-branch: 'true' id: restore-snapshot if: inputs.image-type == 'ci' && inputs.image-stash-ref == '' # Falls back to the image stash below whenever the snapshot does not fit this runner's daemon. @@ -103,7 +103,7 @@ runs: echo "Falling back to loading the image stash" fi shell: bash - if: steps.restore-snapshot.outputs.stash-hit == 'true' + if: steps.restore-snapshot.outcome == 'success' - name: > Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }} ${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }} diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 06e4fbd496da2..4af130bcbca7e 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -349,8 +349,8 @@ jobs: # unpacking and checksumming each layer again; a copy of the image store restores with one # extraction. Taken while the freshly built layers are still in the page cache, and after the # mount cache export, as it drops the build cache that the export reads. - # Only PR-scoped snapshots: arbitrary checkout refs in privileged dispatch/scheduled - # workflows must never publish a snapshot into the default branch's cache scope. + # Snapshots are handoffs within this run, not branch caches. An arbitrary checkout ref + # must not publish raw daemon state for a later run on the default branch. - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" id: snapshot-export continue-on-error: true @@ -364,11 +364,13 @@ jobs: github.event_name == 'pull_request' && inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && steps.stashed-image.outputs.reusable != 'true' - - name: "Stash CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + - name: "Upload CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" continue-on-error: true - uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - key: ci-image-snapshot-v1-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} + name: >- + ci-image-snapshot-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} path: "/mnt/ci-image-snapshot-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.zst*" if-no-files-found: 'error' compression-level: '0' From 3b588c2e052bf0fc651cbd70d9945147f0fbee51 Mon Sep 17 00:00:00 2001 From: Joe El Date: Sat, 3 Oct 2026 22:47:16 +0000 Subject: [PATCH 04/13] Publish authoritative caches before optional CI snapshot execution The snapshot script runs checkout code and prunes Docker state. Existing cache publications must finish before that optional execution. --- .github/workflows/ci-image-build.yml | 114 +++++++++++++-------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 4af130bcbca7e..8f1950dde9782 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -321,6 +321,60 @@ jobs: docker rmi "${CACHE_FROM_IMAGE}" shell: bash if: always() && env.CACHE_FROM_IMAGE != '' + # zstd stands in for the stash's zlib (compression-level 0 below), which every consumer inflates. + # The .tar name, which every Breeze expects, stays: docker image load detects zstd by content. + - name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + env: + PLATFORM: ${{ inputs.platform }} + COMMIT_FILE: "/mnt/ci-image-commit-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.txt" + run: | + breeze ci-image save --platform "${PLATFORM}" --image-file-dir "/mnt" + for image_file in /mnt/ci-image-save-*-"${PYTHON_MAJOR_MINOR_VERSION}".tar; do + zstd -6 -T0 --rm -o "${image_file}.zst" "${image_file}" + mv "${image_file}.zst" "${image_file}" + done + git rev-parse HEAD > "${COMMIT_FILE}" + shell: bash + if: inputs.upload-image-artifact == 'true' && steps.stashed-image.outputs.reusable != 'true' + - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 + with: + key: ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} + path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" + if-no-files-found: 'error' + compression-level: '0' + retention-days: '2' + if: inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' + # The per-branch stash above is written by every build on this branch, so an image built + # from a ref other than the branch tip cannot live there: whoever restores it next would + # get sources they never asked for. The ref keeps its own stash instead, which is also + # what a later publish of that same ref reads - hence a retention that outlives the shared + # one, the same ref being rebuilt three days apart at the very least. + - name: "Stash CI docker image built for ref ${{ inputs.image-stash-ref }}" + uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 + with: + key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ + -${{ inputs.image-stash-ref }}" + path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" + if-no-files-found: 'error' + compression-level: '0' + retention-days: '6' + if: > + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref != '' && + steps.stashed-image.outputs.reusable != 'true' + # Saved last and with the image's retention, so that finding this commit is enough to + # know the image it describes is there to be restored. + - name: "Stash the commit the CI image for ref ${{ inputs.image-stash-ref }} was built from" + uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 + with: + key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ + -${{ inputs.image-stash-ref }}" + path: "/mnt/ci-image-commit-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.txt" + if-no-files-found: 'error' + retention-days: '6' + if: > + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref != '' && + steps.stashed-image.outputs.reusable != 'true' - name: "Export mount cache ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" env: PYTHON_MAJOR_MINOR_VERSION: ${{ env.PYTHON_MAJOR_MINOR_VERSION }} @@ -345,10 +399,10 @@ jobs: if: > inputs.upload-mount-cache-artifact == 'true' && steps.stashed-image.outputs.reusable != 'true' - # Every job that prepares the CI image would otherwise `docker image load` the stash below, + # Every job that prepares the CI image would otherwise `docker image load` the image stash, # unpacking and checksumming each layer again; a copy of the image store restores with one - # extraction. Taken while the freshly built layers are still in the page cache, and after the - # mount cache export, as it drops the build cache that the export reads. + # extraction. Run after the authoritative image and mount-cache publications: snapshot + # creation prunes the build cache and must not affect the artifacts those steps publish. # Snapshots are handoffs within this run, not branch caches. An arbitrary checkout ref # must not publish raw daemon state for a later run on the default branch. - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" @@ -379,59 +433,5 @@ jobs: github.event_name == 'pull_request' && steps.snapshot-export.outcome == 'success' && inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && steps.stashed-image.outputs.reusable != 'true' - # zstd stands in for the stash's zlib (compression-level 0 below), which every consumer inflates. - # The .tar name, which every Breeze expects, stays: docker image load detects zstd by content. - - name: "Export CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - env: - PLATFORM: ${{ inputs.platform }} - COMMIT_FILE: "/mnt/ci-image-commit-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.txt" - run: | - breeze ci-image save --platform "${PLATFORM}" --image-file-dir "/mnt" - for image_file in /mnt/ci-image-save-*-"${PYTHON_MAJOR_MINOR_VERSION}".tar; do - zstd -6 -T0 --rm -o "${image_file}.zst" "${image_file}" - mv "${image_file}.zst" "${image_file}" - done - git rev-parse HEAD > "${COMMIT_FILE}" - shell: bash - if: inputs.upload-image-artifact == 'true' && steps.stashed-image.outputs.reusable != 'true' - - name: "Stash CI docker image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" - uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 - with: - key: ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }} - path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" - if-no-files-found: 'error' - compression-level: '0' - retention-days: '2' - if: inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' - # The per-branch stash above is written by every build on this branch, so an image built - # from a ref other than the branch tip cannot live there: whoever restores it next would - # get sources they never asked for. The ref keeps its own stash instead, which is also - # what a later publish of that same ref reads - hence a retention that outlives the shared - # one, the same ref being rebuilt three days apart at the very least. - - name: "Stash CI docker image built for ref ${{ inputs.image-stash-ref }}" - uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 - with: - key: "ci-image-save-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ - -${{ inputs.image-stash-ref }}" - path: "/mnt/ci-image-save-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar" - if-no-files-found: 'error' - compression-level: '0' - retention-days: '6' - if: > - inputs.upload-image-artifact == 'true' && inputs.image-stash-ref != '' && - steps.stashed-image.outputs.reusable != 'true' - # Saved last and with the image's retention, so that finding this commit is enough to - # know the image it describes is there to be restored. - - name: "Stash the commit the CI image for ref ${{ inputs.image-stash-ref }} was built from" - uses: apache/infrastructure-actions/stash/save@61dcea11f19e2bbe1263f14d72235e8da17d3ad0 # save/v1.0.0 - with: - key: "ci-image-commit-v3-${{ inputs.platform }}-${{ env.PYTHON_MAJOR_MINOR_VERSION }}\ - -${{ inputs.image-stash-ref }}" - path: "/mnt/ci-image-commit-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.txt" - if-no-files-found: 'error' - retention-days: '6' - if: > - inputs.upload-image-artifact == 'true' && inputs.image-stash-ref != '' && - steps.stashed-image.outputs.reusable != 'true' - name: "Check disk space after build" run: df -H From bf067aba024daf913d83ec5a80e4ebe0b080d8a5 Mon Sep 17 00:00:00 2001 From: Joe El Date: Sun, 4 Oct 2026 04:49:10 +0000 Subject: [PATCH 05/13] Isolate CI snapshot creation from branch cache publishers The optional snapshot producer executes checkout code without publishing branch caches; its artifacts are available only to consumers in the same workflow run. --- .github/workflows/ci-image-build.yml | 53 +++++++++++++++++++--------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 8f1950dde9782..07159462d16cc 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -399,12 +399,42 @@ jobs: if: > inputs.upload-mount-cache-artifact == 'true' && steps.stashed-image.outputs.reusable != 'true' - # Every job that prepares the CI image would otherwise `docker image load` the image stash, - # unpacking and checksumming each layer again; a copy of the image store restores with one - # extraction. Run after the authoritative image and mount-cache publications: snapshot - # creation prunes the build cache and must not affect the artifacts those steps publish. - # Snapshots are handoffs within this run, not branch caches. An arbitrary checkout ref - # must not publish raw daemon state for a later run on the default branch. + - name: "Check disk space after build" + run: df -H + + # Run checkout code in a separate job with no branch-cache publications. This optional + # producer hands the snapshot only to consumers of the same workflow run. + snapshot-ci-images: + name: "Snapshot CI ${{ inputs.platform }} image ${{ matrix.python-version }}" + needs: build-ci-images + if: > + github.event_name == 'pull_request' && + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' + continue-on-error: true + timeout-minutes: 20 + runs-on: ${{ fromJSON(inputs.runners) }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(inputs.python-versions) }} + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: "Checkout sources" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-ref }} + persist-credentials: false + - name: "Prepare authoritative CI image" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ inputs.platform }} + python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} + make-mnt-writeable-and-cleanup: 'true' - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" id: snapshot-export continue-on-error: true @@ -414,10 +444,6 @@ jobs: ./scripts/ci/docker_data_root_snapshot.sh create "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" shell: bash - if: > - github.event_name == 'pull_request' && - inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && - steps.stashed-image.outputs.reusable != 'true' - name: "Upload CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -429,9 +455,4 @@ jobs: if-no-files-found: 'error' compression-level: '0' retention-days: '2' - if: > - github.event_name == 'pull_request' && steps.snapshot-export.outcome == 'success' && - inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && - steps.stashed-image.outputs.reusable != 'true' - - name: "Check disk space after build" - run: df -H + if: steps.snapshot-export.outcome == 'success' From 89948f0f326a97fb682b6add78e3a05abbab79c4 Mon Sep 17 00:00:00 2001 From: Joe El Date: Sun, 4 Oct 2026 04:55:24 +0000 Subject: [PATCH 06/13] Bind optional snapshot jobs to the workflow commit Snapshot producers must not accept arbitrary checkout refs in privileged workflow contexts, and a partially failed Docker stop must still trigger restart cleanup. --- .github/actions/prepare_breeze_and_image/action.yml | 8 +++++++- .github/workflows/ci-image-build.yml | 6 ++++-- scripts/ci/docker_data_root_snapshot.sh | 7 +++---- scripts/tests/ci/test_docker_data_root_snapshot.py | 11 +++++++++++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index e49246b3f6766..6da0e7371604f 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -49,6 +49,10 @@ inputs: other than the branch tip is stashed under a key of its own by ci-image-build.yml. required: false default: "" + restore-snapshot: + description: 'Whether to attempt the optional snapshot from this workflow run' + required: false + default: 'true' outputs: host-python-version: description: Python version used in host @@ -88,7 +92,9 @@ runs: inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} path: "/mnt/" id: restore-snapshot - if: inputs.image-type == 'ci' && inputs.image-stash-ref == '' + if: > + github.event_name == 'pull_request' && inputs.restore-snapshot == 'true' && + inputs.image-type == 'ci' && inputs.image-stash-ref == '' # Falls back to the image stash below whenever the snapshot does not fit this runner's daemon. - name: "Unpack CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" id: snapshot diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 07159462d16cc..7c3ac4634c2e9 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -409,7 +409,8 @@ jobs: needs: build-ci-images if: > github.event_name == 'pull_request' && - inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && + (inputs.checkout-ref == '' || inputs.checkout-ref == github.sha) continue-on-error: true timeout-minutes: 20 runs-on: ${{ fromJSON(inputs.runners) }} @@ -426,7 +427,7 @@ jobs: - name: "Checkout sources" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ inputs.checkout-ref }} + ref: ${{ github.sha }} persist-credentials: false - name: "Prepare authoritative CI image" uses: ./.github/actions/prepare_breeze_and_image @@ -435,6 +436,7 @@ jobs: python: ${{ matrix.python-version }} use-uv: ${{ inputs.use-uv }} make-mnt-writeable-and-cleanup: 'true' + restore-snapshot: 'false' - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" id: snapshot-export continue-on-error: true diff --git a/scripts/ci/docker_data_root_snapshot.sh b/scripts/ci/docker_data_root_snapshot.sh index 4b039515ce4d8..478fd0ce28a76 100755 --- a/scripts/ci/docker_data_root_snapshot.sh +++ b/scripts/ci/docker_data_root_snapshot.sh @@ -24,9 +24,8 @@ # Keeps only the CI image in the daemon and writes its image store to SNAPSHOT_FILE, plus # SNAPSHOT_FILE.meta naming the daemon that wrote it and the image. # docker_data_root_snapshot.sh restore SNAPSHOT_FILE -# Restores the image. Exits non-zero, leaving the daemon running and without images, when -# the snapshot is missing or was written by a different daemon, or the daemon already holds -# images, so the caller can fall back to `docker image load`. +# Restores the image. Incompatible snapshots leave the daemon untouched. Failed +# materialization cleans partial image state and restarts Docker for the normal stash path. set -euo pipefail DATA_ROOT="/var/lib/docker" @@ -63,8 +62,8 @@ function daemon_fingerprint() { } function stop_daemon() { - sudo systemctl stop docker.socket docker DAEMON_STOPPED=true + sudo systemctl stop docker.socket docker } function start_daemon() { diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py b/scripts/tests/ci/test_docker_data_root_snapshot.py index af5f3f3b6b7c6..cd31502b70b87 100644 --- a/scripts/tests/ci/test_docker_data_root_snapshot.py +++ b/scripts/tests/ci/test_docker_data_root_snapshot.py @@ -46,6 +46,9 @@ def fake_tools(tmp_path): if [[ "${name}" == "sudo" && "$*" == "tar "* ]]; then exit "${TAR_EXIT:-0}" fi +if [[ "${name}" == "sudo" && "$*" == "systemctl stop docker.socket docker" && "${STOP_EXIT:-0}" != 0 ]]; then + exit "${STOP_EXIT}" +fi if [[ "${name}" == "sudo" && "$*" == "systemctl start docker" && "${START_FAIL_ONCE:-0}" == 1 ]]; then if [[ $(grep -c 'sudo systemctl start docker' "${COMMAND_LOG}") == 1 ]]; then exit 1 @@ -215,3 +218,11 @@ def test_stale_snapshot_does_not_modify_daemon(fake_tools, snapshot): result = run_script({**fake_tools, "CHECKOUT_SHA": "revision2"}, "restore", str(snapshot)) assert result.returncode == 3 assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +@pytest.mark.parametrize("mode", ["create", "restore"]) +def test_partial_stop_failure_restarts_daemon(fake_tools, snapshot, mode): + result = run_script({**fake_tools, "STOP_EXIT": "1", "CI_IMAGES": "abc123"}, mode, str(snapshot)) + assert result.returncode != 0 + assert read_commands(fake_tools)[-1] == ["sudo", "systemctl", "start", "docker"] + assert not any(command[:2] == ["sudo", "rm"] for command in read_commands(fake_tools)) From 7b629d14c00e7f6ea628cdc61101da48a61632b4 Mon Sep 17 00:00:00 2001 From: Joe El Date: Sun, 4 Oct 2026 05:18:47 +0000 Subject: [PATCH 07/13] Avoid reloading the CI image before snapshot creation Preparing the authoritative image in a second runner adds nearly three minutes before consumers can start. All branch publications must finish before the optional snapshot code prunes Docker state. --- .../prepare_breeze_and_image/action.yml | 6 +-- .github/workflows/ci-image-build.yml | 40 +++++-------------- 2 files changed, 10 insertions(+), 36 deletions(-) diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 6da0e7371604f..39e248b46ac35 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -49,10 +49,6 @@ inputs: other than the branch tip is stashed under a key of its own by ci-image-build.yml. required: false default: "" - restore-snapshot: - description: 'Whether to attempt the optional snapshot from this workflow run' - required: false - default: 'true' outputs: host-python-version: description: Python version used in host @@ -93,7 +89,7 @@ runs: path: "/mnt/" id: restore-snapshot if: > - github.event_name == 'pull_request' && inputs.restore-snapshot == 'true' && + github.event_name == 'pull_request' && inputs.image-type == 'ci' && inputs.image-stash-ref == '' # Falls back to the image stash below whenever the snapshot does not fit this runner's daemon. - name: "Unpack CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 7c3ac4634c2e9..483a7fead2d28 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -402,41 +402,18 @@ jobs: - name: "Check disk space after build" run: df -H - # Run checkout code in a separate job with no branch-cache publications. This optional - # producer hands the snapshot only to consumers of the same workflow run. - snapshot-ci-images: - name: "Snapshot CI ${{ inputs.platform }} image ${{ matrix.python-version }}" - needs: build-ci-images - if: > - github.event_name == 'pull_request' && - inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && - (inputs.checkout-ref == '' || inputs.checkout-ref == github.sha) - continue-on-error: true - timeout-minutes: 20 - runs-on: ${{ fromJSON(inputs.runners) }} - permissions: - contents: read - strategy: - fail-fast: false - matrix: - python-version: ${{ fromJSON(inputs.python-versions) }} - env: - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - steps: - - name: "Checkout sources" + # All branch caches are published before checkout code snapshots and prunes Docker. + - name: "Checkout workflow commit for optional snapshot" + id: snapshot-checkout + continue-on-error: true uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} persist-credentials: false - - name: "Prepare authoritative CI image" - uses: ./.github/actions/prepare_breeze_and_image - with: - platform: ${{ inputs.platform }} - python: ${{ matrix.python-version }} - use-uv: ${{ inputs.use-uv }} - make-mnt-writeable-and-cleanup: 'true' - restore-snapshot: 'false' + if: > + github.event_name == 'pull_request' && + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && + (inputs.checkout-ref == '' || inputs.checkout-ref == github.sha) - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" id: snapshot-export continue-on-error: true @@ -446,6 +423,7 @@ jobs: ./scripts/ci/docker_data_root_snapshot.sh create "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" shell: bash + if: steps.snapshot-checkout.outcome == 'success' - name: "Upload CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 From ec34fff2793b8bedca8fa2862a81e51ebb7f26a1 Mon Sep 17 00:00:00 2001 From: Joe El Date: Sun, 4 Oct 2026 05:30:59 +0000 Subject: [PATCH 08/13] Execute the snapshot script from the immutable workflow commit The builder can check out a configurable ref before publishing caches. Snapshot execution must read the workflow commit itself rather than trust files left in that working tree. --- .github/workflows/ci-image-build.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 483a7fead2d28..885e0d4443e88 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -419,9 +419,10 @@ jobs: continue-on-error: true env: PLATFORM: ${{ inputs.platform }} - run: > - ./scripts/ci/docker_data_root_snapshot.sh create - "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" + # Execute the immutable workflow blob, rather than files a configurable checkout leaves. + run: | + git show "${GITHUB_SHA}:scripts/ci/docker_data_root_snapshot.sh" | bash -s -- create \ + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" shell: bash if: steps.snapshot-checkout.outcome == 'success' - name: "Upload CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" From dd1b1fde1aee50ffca9ce6f29e9e33167802d7ea Mon Sep 17 00:00:00 2001 From: Joe El Date: Sun, 4 Oct 2026 05:38:47 +0000 Subject: [PATCH 09/13] Keep optional snapshots in an isolated read-only job Executing snapshot code inside the configurable-ref builder introduces a cache-poisoning finding. A separate job preserves the security boundary; its latency cost must remain explicit when measuring the runner-time benefit. --- .../prepare_breeze_and_image/action.yml | 6 ++- .github/workflows/ci-image-build.yml | 47 ++++++++++++++----- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 39e248b46ac35..6da0e7371604f 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -49,6 +49,10 @@ inputs: other than the branch tip is stashed under a key of its own by ci-image-build.yml. required: false default: "" + restore-snapshot: + description: 'Whether to attempt the optional snapshot from this workflow run' + required: false + default: 'true' outputs: host-python-version: description: Python version used in host @@ -89,7 +93,7 @@ runs: path: "/mnt/" id: restore-snapshot if: > - github.event_name == 'pull_request' && + github.event_name == 'pull_request' && inputs.restore-snapshot == 'true' && inputs.image-type == 'ci' && inputs.image-stash-ref == '' # Falls back to the image stash below whenever the snapshot does not fit this runner's daemon. - name: "Unpack CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 885e0d4443e88..7c3ac4634c2e9 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -402,29 +402,50 @@ jobs: - name: "Check disk space after build" run: df -H - # All branch caches are published before checkout code snapshots and prunes Docker. - - name: "Checkout workflow commit for optional snapshot" - id: snapshot-checkout - continue-on-error: true + # Run checkout code in a separate job with no branch-cache publications. This optional + # producer hands the snapshot only to consumers of the same workflow run. + snapshot-ci-images: + name: "Snapshot CI ${{ inputs.platform }} image ${{ matrix.python-version }}" + needs: build-ci-images + if: > + github.event_name == 'pull_request' && + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && + (inputs.checkout-ref == '' || inputs.checkout-ref == github.sha) + continue-on-error: true + timeout-minutes: 20 + runs-on: ${{ fromJSON(inputs.runners) }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(inputs.python-versions) }} + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: "Checkout sources" uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.sha }} persist-credentials: false - if: > - github.event_name == 'pull_request' && - inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && - (inputs.checkout-ref == '' || inputs.checkout-ref == github.sha) + - name: "Prepare authoritative CI image" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ inputs.platform }} + python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} + make-mnt-writeable-and-cleanup: 'true' + restore-snapshot: 'false' - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" id: snapshot-export continue-on-error: true env: PLATFORM: ${{ inputs.platform }} - # Execute the immutable workflow blob, rather than files a configurable checkout leaves. - run: | - git show "${GITHUB_SHA}:scripts/ci/docker_data_root_snapshot.sh" | bash -s -- create \ - "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" + run: > + ./scripts/ci/docker_data_root_snapshot.sh create + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" shell: bash - if: steps.snapshot-checkout.outcome == 'success' - name: "Upload CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" continue-on-error: true uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 From 428190b1f67d8b810112bf9fe41ae22b2507ba16 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Sun, 4 Oct 2026 08:57:20 +0300 Subject: [PATCH 10/13] Add CI image snapshot metadata preflight Avoid downloading incompatible Docker snapshots by checking their small metadata artifact first. Preserve the existing stash fallback for every miss or validation failure.\n\nCo-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../prepare_breeze_and_image/action.yml | 26 +++++++++++-- .github/workflows/ci-image-build.yml | 12 ++++++ scripts/ci/docker_data_root_snapshot.sh | 37 +++++++++++++++---- .../ci/test_docker_data_root_snapshot.py | 26 +++++++++++++ 4 files changed, 90 insertions(+), 11 deletions(-) diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 6da0e7371604f..cbb29c3a8c564 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -83,18 +83,38 @@ runs: run: | echo "Checking free space!" df -H - - name: "Restore CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" + - name: "Preflight CI image snapshot metadata ${{ inputs.platform }}:${{ inputs.python }}" continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: name: >- - ci-image-snapshot-v2-${{ inputs.python }}-${{ + ci-image-snapshot-meta-v2-${{ inputs.python }}-${{ inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} path: "/mnt/" - id: restore-snapshot + id: snapshot-metadata if: > github.event_name == 'pull_request' && inputs.restore-snapshot == 'true' && inputs.image-type == 'ci' && inputs.image-stash-ref == '' + - name: "Check CI image snapshot compatibility ${{ inputs.platform }}:${{ inputs.python }}" + id: snapshot-preflight + env: + PLATFORM: ${{ inputs.platform }} + PYTHON: ${{ inputs.python }} + run: | + ./scripts/ci/docker_data_root_snapshot.sh preflight \ + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON}.tar.zst.meta" + shell: bash + if: steps.snapshot-metadata.outcome == 'success' + - name: "Download CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: >- + ci-image-snapshot-v2-${{ inputs.python }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: "/mnt/" + id: restore-snapshot + if: steps.snapshot-preflight.outcome == 'success' # Falls back to the image stash below whenever the snapshot does not fit this runner's daemon. - name: "Unpack CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" id: snapshot diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 7c3ac4634c2e9..2636147c11188 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -458,3 +458,15 @@ jobs: compression-level: '0' retention-days: '2' if: steps.snapshot-export.outcome == 'success' + - name: "Upload CI image snapshot metadata ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: >- + ci-image-snapshot-meta-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: "/mnt/ci-image-snapshot-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.zst.meta" + if-no-files-found: 'error' + compression-level: '9' + retention-days: '2' + if: steps.snapshot-export.outcome == 'success' diff --git a/scripts/ci/docker_data_root_snapshot.sh b/scripts/ci/docker_data_root_snapshot.sh index 478fd0ce28a76..95295f6515f3b 100755 --- a/scripts/ci/docker_data_root_snapshot.sh +++ b/scripts/ci/docker_data_root_snapshot.sh @@ -26,6 +26,8 @@ # docker_data_root_snapshot.sh restore SNAPSHOT_FILE # Restores the image. Incompatible snapshots leave the daemon untouched. Failed # materialization cleans partial image state and restarts Docker for the normal stash path. +# docker_data_root_snapshot.sh preflight METADATA_FILE +# Checks compatibility without requiring the multi-gigabyte snapshot archive. set -euo pipefail DATA_ROOT="/var/lib/docker" @@ -95,15 +97,15 @@ function create_snapshot() { start_daemon } -function restore_snapshot() { - local snapshot_file="${1}" +function validate_metadata() { + local metadata_file="${1}" local -a meta - local image_id snapshot_fingerprint daemon - if [[ ! -f "${snapshot_file}" || ! -f "${snapshot_file}.meta" ]]; then - echo "No snapshot at ${snapshot_file}" + local snapshot_fingerprint daemon + if [[ ! -f "${metadata_file}" ]]; then + echo "No snapshot metadata at ${metadata_file}" exit 2 fi - read -r -a meta < "${snapshot_file}.meta" + read -r -a meta < "${metadata_file}" if [[ ${#meta[@]} != 6 ]]; then echo "Invalid snapshot metadata" exit 3 @@ -112,7 +114,6 @@ function restore_snapshot() { echo "Snapshot belongs to a different checkout revision" exit 3 fi - image_id="${meta[4]}" snapshot_fingerprint="${meta[*]:0:4}" daemon="$(daemon_fingerprint)" check_supported_daemon "${daemon}" @@ -124,6 +125,23 @@ function restore_snapshot() { echo "The daemon already holds images, which restoring the snapshot would drop" exit 3 fi +} + +function preflight_snapshot() { + validate_metadata "${1}" +} + +function restore_snapshot() { + local snapshot_file="${1}" + local -a meta + local image_id + if [[ ! -f "${snapshot_file}" || ! -f "${snapshot_file}.meta" ]]; then + echo "No snapshot at ${snapshot_file}" + exit 2 + fi + validate_metadata "${snapshot_file}.meta" + read -r -a meta < "${snapshot_file}.meta" + image_id="${meta[4]}" stop_daemon RESTORE_IN_PROGRESS=true remove_image_store @@ -145,8 +163,11 @@ case "${1:-}" in restore) restore_snapshot "${2}" ;; + preflight) + preflight_snapshot "${2}" + ;; *) - echo "Usage: ${0} create|restore SNAPSHOT_FILE" >&2 + echo "Usage: ${0} create|preflight|restore FILE" >&2 exit 1 ;; esac diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py b/scripts/tests/ci/test_docker_data_root_snapshot.py index cd31502b70b87..b668753131555 100644 --- a/scripts/tests/ci/test_docker_data_root_snapshot.py +++ b/scripts/tests/ci/test_docker_data_root_snapshot.py @@ -114,6 +114,32 @@ def test_restore_unpacks_snapshot_into_stopped_daemon(fake_tools, snapshot): assert not snapshot.exists() +def test_preflight_checks_metadata_without_touching_snapshot_or_daemon(fake_tools, snapshot): + result = run_script(fake_tools, "preflight", f"{snapshot}.meta") + + assert result.returncode == 0, result.stdout + result.stderr + assert snapshot.exists() + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +@pytest.mark.parametrize( + ("env", "metadata", "expected_exit"), + [ + pytest.param({}, "invalid\n", 3, id="malformed"), + pytest.param({"CHECKOUT_SHA": "revision2"}, None, 3, id="stale-checkout"), + pytest.param({"DAEMON_FINGERPRINT": "29.0.0 overlay2 x86_64 /var/lib/docker"}, None, 3, id="other-daemon"), + ], +) +def test_preflight_rejects_untrusted_metadata(fake_tools, snapshot, env, metadata, expected_exit): + if metadata is not None: + Path(f"{snapshot}.meta").write_text(metadata) + + result = run_script({**fake_tools, **env}, "preflight", f"{snapshot}.meta") + + assert result.returncode == expected_exit + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + @pytest.mark.parametrize( ("env", "remove_meta", "expected_exit"), [ From 5f19b9bef0c82d62f1d6e6c01ae2793f43e7a168 Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Sun, 4 Oct 2026 13:49:27 +0300 Subject: [PATCH 11/13] Preserve Docker state when snapshot inventory checks fail An unavailable daemon must not be mistaken for an empty image store before destructive snapshot materialization. --- scripts/ci/docker_data_root_snapshot.sh | 6 ++++-- scripts/tests/ci/test_docker_data_root_snapshot.py | 12 ++++++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/ci/docker_data_root_snapshot.sh b/scripts/ci/docker_data_root_snapshot.sh index 478fd0ce28a76..054d2f0d27b5c 100755 --- a/scripts/ci/docker_data_root_snapshot.sh +++ b/scripts/ci/docker_data_root_snapshot.sh @@ -98,7 +98,7 @@ function create_snapshot() { function restore_snapshot() { local snapshot_file="${1}" local -a meta - local image_id snapshot_fingerprint daemon + local image_id snapshot_fingerprint daemon existing_images existing_containers if [[ ! -f "${snapshot_file}" || ! -f "${snapshot_file}.meta" ]]; then echo "No snapshot at ${snapshot_file}" exit 2 @@ -120,7 +120,9 @@ function restore_snapshot() { echo "The snapshot was written by '${snapshot_fingerprint}', this daemon is '${daemon}'" exit 3 fi - if [[ -n "$(docker images --all --quiet)" || -n "$(docker ps --all --quiet)" ]]; then + existing_images="$(docker images --all --quiet)" + existing_containers="$(docker ps --all --quiet)" + if [[ -n "${existing_images}" || -n "${existing_containers}" ]]; then echo "The daemon already holds images, which restoring the snapshot would drop" exit 3 fi diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py b/scripts/tests/ci/test_docker_data_root_snapshot.py index cd31502b70b87..f4f55f69cb10f 100644 --- a/scripts/tests/ci/test_docker_data_root_snapshot.py +++ b/scripts/tests/ci/test_docker_data_root_snapshot.py @@ -59,8 +59,8 @@ def fake_tools(tmp_path): "info --format "*) echo "${DAEMON_FINGERPRINT}" ;; "images --quiet --filter label=org.apache.airflow.image=airflow-ci") printf '%b' "${CI_IMAGES:-}" ;; "images --quiet") printf '%b' "${ALL_IMAGES:-}" ;; - "images --all --quiet") printf '%b' "${ALL_IMAGES:-}" ;; - "ps --all --quiet") printf '%b' "${CONTAINERS:-}" ;; + "images --all --quiet") printf '%b' "${ALL_IMAGES:-}"; exit "${IMAGES_EXIT:-0}" ;; + "ps --all --quiet") printf '%b' "${CONTAINERS:-}"; exit "${CONTAINERS_EXIT:-0}" ;; "run "*) exit "${RUN_EXIT:-0}" ;; esac fi @@ -207,6 +207,14 @@ def test_existing_container_prevents_restore(fake_tools, snapshot): assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) +@pytest.mark.parametrize("failure", [{"IMAGES_EXIT": "1"}, {"CONTAINERS_EXIT": "1"}]) +def test_failed_daemon_inventory_does_not_modify_store(fake_tools, snapshot, failure): + result = run_script({**fake_tools, **failure}, "restore", str(snapshot)) + + assert result.returncode != 0 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + def test_malformed_metadata_prevents_restore(fake_tools, snapshot): Path(f"{snapshot}.meta").write_text("invalid\n") result = run_script(fake_tools, "restore", str(snapshot)) From 3a5022b98d1b5bb896fca6db77a7acf20da5584b Mon Sep 17 00:00:00 2001 From: Yossi Eliaz Date: Sun, 4 Oct 2026 16:46:45 +0300 Subject: [PATCH 12/13] Retrigger CI after provider compatibility timeout From c624a3e71c28f7deec6e4e4a434e97b13736c8f5 Mon Sep 17 00:00:00 2001 From: zozo123 Date: Mon, 5 Oct 2026 17:20:04 -0700 Subject: [PATCH 13/13] Rerun CI after the hosted runner shutdown The OpenLineage compatibility job was interrupted by a runner shutdown before reporting a test result.