diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 6c91e28bff84d..23cef185e9670 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -49,6 +49,10 @@ inputs: other than the branch tip is stashed under a key of its own by ci-image-build.yml. required: false default: "" + restore-snapshot: + description: 'Whether to attempt the optional snapshot from this workflow run' + required: false + default: 'true' outputs: host-python-version: description: Python version used in host @@ -79,6 +83,54 @@ runs: run: | echo "Checking free space!" df -H + - name: "Preflight CI image snapshot metadata ${{ inputs.platform }}:${{ inputs.python }}" + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: >- + ci-image-snapshot-meta-v2-${{ inputs.python }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: "/mnt/" + id: snapshot-metadata + if: > + github.event_name == 'pull_request' && inputs.restore-snapshot == 'true' && + inputs.image-type == 'ci' && inputs.image-stash-ref == '' + - name: "Check CI image snapshot compatibility ${{ inputs.platform }}:${{ inputs.python }}" + id: snapshot-preflight + continue-on-error: true + env: + PLATFORM: ${{ inputs.platform }} + PYTHON: ${{ inputs.python }} + run: | + ./scripts/ci/docker_data_root_snapshot.sh preflight \ + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON}.tar.zst.meta" + shell: bash + if: steps.snapshot-metadata.outcome == 'success' + - name: "Download CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: >- + ci-image-snapshot-v2-${{ inputs.python }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: "/mnt/" + id: restore-snapshot + if: steps.snapshot-preflight.outcome == 'success' + # Falls back to the image stash below whenever the snapshot does not fit this runner's daemon. + - name: "Unpack CI image snapshot ${{ inputs.platform }}:${{ inputs.python }}" + id: snapshot + env: + PLATFORM: ${{ inputs.platform }} + PYTHON: ${{ inputs.python }} + run: | + if ./scripts/ci/docker_data_root_snapshot.sh restore \ + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON}.tar.zst"; then + echo "restored=true" >> "${GITHUB_OUTPUT}" + else + echo "Falling back to loading the image stash" + fi + shell: bash + if: steps.restore-snapshot.outcome == 'success' - name: > Restore ${{ inputs.image-type }} docker image ${{ inputs.platform }}:${{ inputs.python }} ${{ inputs.image-stash-ref != '' && format('built for ref {0}', inputs.image-stash-ref) || '' }} @@ -90,6 +142,7 @@ runs: path: "/mnt/" only-current-branch: 'true' fail-on-download: 'true' + if: steps.snapshot.outputs.restored != 'true' - name: "Load ${{ inputs.image-type }} image ${{ inputs.platform }}:${{ inputs.python }}" env: PLATFORM: ${{ inputs.platform }} @@ -98,3 +151,4 @@ runs: run: > breeze ${IMAGE_TYPE}-image load --platform "${PLATFORM}" --python "${PYTHON}" --image-file-dir "/mnt" shell: bash + if: steps.snapshot.outputs.restored != 'true' diff --git a/.github/workflows/ci-image-build.yml b/.github/workflows/ci-image-build.yml index 1541337bc5c73..2636147c11188 100644 --- a/.github/workflows/ci-image-build.yml +++ b/.github/workflows/ci-image-build.yml @@ -401,3 +401,72 @@ jobs: steps.stashed-image.outputs.reusable != 'true' - name: "Check disk space after build" run: df -H + + # Run checkout code in a separate job with no branch-cache publications. This optional + # producer hands the snapshot only to consumers of the same workflow run. + snapshot-ci-images: + name: "Snapshot CI ${{ inputs.platform }} image ${{ matrix.python-version }}" + needs: build-ci-images + if: > + github.event_name == 'pull_request' && + inputs.upload-image-artifact == 'true' && inputs.image-stash-ref == '' && + (inputs.checkout-ref == '' || inputs.checkout-ref == github.sha) + continue-on-error: true + timeout-minutes: 20 + runs-on: ${{ fromJSON(inputs.runners) }} + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-version: ${{ fromJSON(inputs.python-versions) }} + env: + PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: "Checkout sources" + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: "Prepare authoritative CI image" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ inputs.platform }} + python: ${{ matrix.python-version }} + use-uv: ${{ inputs.use-uv }} + make-mnt-writeable-and-cleanup: 'true' + restore-snapshot: 'false' + - name: "Snapshot CI image ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + id: snapshot-export + continue-on-error: true + env: + PLATFORM: ${{ inputs.platform }} + run: > + ./scripts/ci/docker_data_root_snapshot.sh create + "/mnt/ci-image-snapshot-${PLATFORM//\//_}-${PYTHON_MAJOR_MINOR_VERSION}.tar.zst" + shell: bash + - name: "Upload CI image snapshot ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: >- + ci-image-snapshot-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: "/mnt/ci-image-snapshot-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.zst*" + if-no-files-found: 'error' + compression-level: '0' + retention-days: '2' + if: steps.snapshot-export.outcome == 'success' + - name: "Upload CI image snapshot metadata ${{ inputs.platform }}:${{ env.PYTHON_MAJOR_MINOR_VERSION }}" + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: >- + ci-image-snapshot-meta-v2-${{ env.PYTHON_MAJOR_MINOR_VERSION }}-${{ + inputs.platform == 'linux/amd64' && 'amd64' || 'arm64' }} + path: "/mnt/ci-image-snapshot-*-${{ env.PYTHON_MAJOR_MINOR_VERSION }}.tar.zst.meta" + if-no-files-found: 'error' + compression-level: '9' + retention-days: '2' + if: steps.snapshot-export.outcome == 'success' diff --git a/scripts/ci/docker_data_root_snapshot.sh b/scripts/ci/docker_data_root_snapshot.sh new file mode 100755 index 0000000000000..1cf5b489a4d1a --- /dev/null +++ b/scripts/ci/docker_data_root_snapshot.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Moves the CI image between jobs as a copy of the Docker data directory that holds it, so a job +# restores it with one extraction instead of `docker image load` unpacking and checksumming every +# layer again. +# +# docker_data_root_snapshot.sh create SNAPSHOT_FILE +# Keeps only the CI image in the daemon and writes its image store to SNAPSHOT_FILE, plus +# SNAPSHOT_FILE.meta naming the daemon that wrote it and the image. +# docker_data_root_snapshot.sh restore SNAPSHOT_FILE +# Restores the image. Incompatible snapshots leave the daemon untouched. Failed +# materialization cleans partial image state and restarts Docker for the normal stash path. +# docker_data_root_snapshot.sh preflight METADATA_FILE +# Checks compatibility without requiring the multi-gigabyte snapshot archive. +set -euo pipefail + +DATA_ROOT="/var/lib/docker" +DAEMON_STOPPED=false +RESTORE_IN_PROGRESS=false + +function cleanup_daemon() { + local result=$? + trap - EXIT + if [[ "${RESTORE_IN_PROGRESS}" == true ]]; then + stop_daemon + remove_image_store + start_daemon + elif [[ "${DAEMON_STOPPED}" == true ]]; then + start_daemon + fi + exit "${result}" +} +trap cleanup_daemon EXIT + +function check_supported_daemon() { + local daemon=$1 + local _ driver root + read -r _ driver _ root <<< "${daemon}" + if [[ "${driver}" != overlay2 || "${root}" != "${DATA_ROOT}" ]]; then + echo "Unsupported Docker image store: ${daemon}" + exit 3 + fi +} + +# A data directory is only readable by the daemon version and storage driver that wrote it. +function daemon_fingerprint() { + docker info --format '{{.ServerVersion}} {{.Driver}} {{.Architecture}} {{.DockerRootDir}}' +} + +function stop_daemon() { + DAEMON_STOPPED=true + sudo systemctl stop docker.socket docker +} + +function start_daemon() { + sudo systemctl start docker + DAEMON_STOPPED=false +} + +function remove_image_store() { + sudo rm -rf "${DATA_ROOT}/image" "${DATA_ROOT}/overlay2" +} + +function create_snapshot() { + local snapshot_file="${1}" + local image_id + check_supported_daemon "$(daemon_fingerprint)" + image_id="$(docker images --quiet --filter "label=org.apache.airflow.image=airflow-ci" | sort -u)" + if [[ -z "${image_id}" || "${image_id}" == *$'\n'* ]]; then + echo "Expected exactly one CI image in the daemon, found: '${image_id}'" >&2 + exit 1 + fi + docker ps --all --quiet | xargs --no-run-if-empty docker rm --force >/dev/null + docker images --quiet | sort -u | { grep --invert-match --fixed-strings --line-regexp "${image_id}" || true; } \ + | xargs --no-run-if-empty docker rmi --force >/dev/null + docker builder prune --all --force >/dev/null + printf '%s %s %s\n' "$(daemon_fingerprint)" "${image_id}" "$(git rev-parse HEAD)" > "${snapshot_file}.meta" + stop_daemon + sudo tar --directory "${DATA_ROOT}" --xattrs --acls --numeric-owner --create --file - image overlay2 \ + | zstd -3 -T0 --quiet --force -o "${snapshot_file}" + start_daemon +} + +function validate_metadata() { + local metadata_file="${1}" + local -a meta + local snapshot_fingerprint daemon existing_images existing_containers + if [[ ! -f "${metadata_file}" ]]; then + echo "No snapshot metadata at ${metadata_file}" + exit 2 + fi + read -r -a meta < "${metadata_file}" + if [[ ${#meta[@]} != 6 ]]; then + echo "Invalid snapshot metadata" + exit 3 + fi + if [[ "${meta[5]}" != "$(git rev-parse HEAD)" ]]; then + echo "Snapshot belongs to a different checkout revision" + exit 3 + fi + snapshot_fingerprint="${meta[*]:0:4}" + daemon="$(daemon_fingerprint)" + check_supported_daemon "${daemon}" + if [[ "${snapshot_fingerprint}" != "${daemon}" ]]; then + echo "The snapshot was written by '${snapshot_fingerprint}', this daemon is '${daemon}'" + exit 3 + fi + existing_images="$(docker images --all --quiet)" + existing_containers="$(docker ps --all --quiet)" + if [[ -n "${existing_images}" || -n "${existing_containers}" ]]; then + echo "The daemon already holds images, which restoring the snapshot would drop" + exit 3 + fi +} + +function preflight_snapshot() { + validate_metadata "${1}" +} + +function restore_snapshot() { + local snapshot_file="${1}" + local -a meta + local image_id + if [[ ! -f "${snapshot_file}" || ! -f "${snapshot_file}.meta" ]]; then + echo "No snapshot at ${snapshot_file}" + exit 2 + fi + validate_metadata "${snapshot_file}.meta" + read -r -a meta < "${snapshot_file}.meta" + image_id="${meta[4]}" + stop_daemon + RESTORE_IN_PROGRESS=true + remove_image_store + zstd -d -T0 --quiet --stdout "${snapshot_file}" \ + | sudo tar --directory "${DATA_ROOT}" --xattrs --acls --numeric-owner --extract --file - + start_daemon + if ! docker run --rm --entrypoint /bin/bash "${image_id}" -c true; then + echo "The restored image ${image_id} does not run" + exit 4 + fi + RESTORE_IN_PROGRESS=false + rm -f "${snapshot_file}" "${snapshot_file}.meta" +} + +case "${1:-}" in + create) + create_snapshot "${2}" + ;; + restore) + restore_snapshot "${2}" + ;; + preflight) + preflight_snapshot "${2}" + ;; + *) + echo "Usage: ${0} create|preflight|restore FILE" >&2 + exit 1 + ;; +esac diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py b/scripts/tests/ci/test_docker_data_root_snapshot.py new file mode 100644 index 0000000000000..c542c346d997e --- /dev/null +++ b/scripts/tests/ci/test_docker_data_root_snapshot.py @@ -0,0 +1,289 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import os +import shlex +import subprocess +from pathlib import Path + +import pytest +import yaml + +SCRIPT = Path(__file__).resolve().parents[2] / "ci" / "docker_data_root_snapshot.sh" +FINGERPRINT = "28.5.2 overlay2 x86_64 /var/lib/docker" + + +@pytest.fixture +def fake_tools(tmp_path): + """Shadow ``docker``, ``sudo`` and ``zstd`` with a logger answering the queries the script makes.""" + tools = tmp_path / "bin" + tools.mkdir() + command = tools / "command" + command.write_text( + r"""#!/usr/bin/env bash +name="${0##*/}" +printf '%s\n' "${name} $*" >> "${COMMAND_LOG}" +if [[ "${name}" == "git" && "$*" == "rev-parse HEAD" ]]; then + echo "${CHECKOUT_SHA:-revision1}" +fi +if [[ "${name}" == "zstd" ]]; then + exit "${ZSTD_EXIT:-0}" +fi +if [[ "${name}" == "sudo" && "$*" == "tar "* ]]; then + exit "${TAR_EXIT:-0}" +fi +if [[ "${name}" == "sudo" && "$*" == "systemctl stop docker.socket docker" && "${STOP_EXIT:-0}" != 0 ]]; then + exit "${STOP_EXIT}" +fi +if [[ "${name}" == "sudo" && "$*" == "systemctl start docker" && "${START_FAIL_ONCE:-0}" == 1 ]]; then + if [[ $(grep -c 'sudo systemctl start docker' "${COMMAND_LOG}") == 1 ]]; then + exit 1 + fi +fi +if [[ "${name}" == "docker" ]]; then + case "$*" in + "info --format "*) echo "${DAEMON_FINGERPRINT}" ;; + "images --quiet --filter label=org.apache.airflow.image=airflow-ci") printf '%b' "${CI_IMAGES:-}" ;; + "images --quiet") printf '%b' "${ALL_IMAGES:-}" ;; + "images --all --quiet") printf '%b' "${ALL_IMAGES:-}"; exit "${IMAGES_EXIT:-0}" ;; + "ps --all --quiet") printf '%b' "${CONTAINERS:-}"; exit "${CONTAINERS_EXIT:-0}" ;; + "run "*) exit "${RUN_EXIT:-0}" ;; + esac +fi +""" + ) + command.chmod(0o755) + for name in ("docker", "sudo", "zstd", "git"): + (tools / name).symlink_to(command) + return { + "PATH": f"{tools}:{os.environ['PATH']}", + "COMMAND_LOG": str(tmp_path / "commands.log"), + "DAEMON_FINGERPRINT": FINGERPRINT, + } + + +@pytest.fixture +def snapshot(tmp_path): + snapshot_file = tmp_path / "ci-image-snapshot-linux_amd64-3.10.tar.zst" + snapshot_file.write_bytes(b"") + Path(f"{snapshot_file}.meta").write_text(f"{FINGERPRINT} abc123 revision1\n") + return snapshot_file + + +def run_script(env, *args): + return subprocess.run( + ["bash", "--noprofile", "--norc", str(SCRIPT), *args], + env={**os.environ, **env}, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + + +def read_commands(env): + log = Path(env["COMMAND_LOG"]) + return [shlex.split(line) for line in log.read_text().splitlines()] if log.exists() else [] + + +def test_restore_unpacks_snapshot_into_stopped_daemon(fake_tools, snapshot): + result = run_script(fake_tools, "restore", str(snapshot)) + + assert result.returncode == 0, result.stdout + result.stderr + commands = read_commands(fake_tools) + stop = commands.index(["sudo", "systemctl", "stop", "docker.socket", "docker"]) + remove = commands.index(["sudo", "rm", "-rf", "/var/lib/docker/image", "/var/lib/docker/overlay2"]) + extract = next(i for i, c in enumerate(commands) if c[:2] == ["sudo", "tar"] and "--extract" in c) + start = commands.index(["sudo", "systemctl", "start", "docker"]) + run = commands.index(["docker", "run", "--rm", "--entrypoint", "/bin/bash", "abc123", "-c", "true"]) + assert stop < remove < extract < start < run + assert not snapshot.exists() + + +def test_preflight_checks_metadata_without_touching_snapshot_or_daemon(fake_tools, snapshot): + result = run_script(fake_tools, "preflight", f"{snapshot}.meta") + + assert result.returncode == 0, result.stdout + result.stderr + assert snapshot.exists() + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +def test_incompatible_preflight_preserves_action_fallback(fake_tools, snapshot): + action_file = SCRIPT.parents[2] / ".github/actions/prepare_breeze_and_image/action.yml" + steps = yaml.safe_load(action_file.read_text())["runs"]["steps"] + preflight = next(step for step in steps if step.get("id") == "snapshot-preflight") + download = next(step for step in steps if step.get("id") == "restore-snapshot") + stash = next(step for step in steps if "stash/restore@" in step.get("uses", "")) + load = next(step for step in steps if step["name"].startswith("Load ")) + + result = run_script({**fake_tools, "CHECKOUT_SHA": "revision2"}, "preflight", f"{snapshot}.meta") + + assert result.returncode == 3 + # A failed optional check must not trigger GitHub's implicit success() gate for later steps. + assert preflight["continue-on-error"] is True + assert download["if"] == "steps.snapshot-preflight.outcome == 'success'" + assert stash["if"] == load["if"] == "steps.snapshot.outputs.restored != 'true'" + + +@pytest.mark.parametrize( + ("env", "metadata", "expected_exit"), + [ + pytest.param({}, "invalid\n", 3, id="malformed"), + pytest.param({"CHECKOUT_SHA": "revision2"}, None, 3, id="stale-checkout"), + pytest.param( + {"DAEMON_FINGERPRINT": "29.0.0 overlay2 x86_64 /var/lib/docker"}, None, 3, id="other-daemon" + ), + pytest.param({"ALL_IMAGES": "def456"}, None, 3, id="daemon-holds-images"), + pytest.param({"CONTAINERS": "container1"}, None, 3, id="daemon-holds-containers"), + pytest.param( + {"DAEMON_FINGERPRINT": "28.5.2 btrfs x86_64 /var/lib/docker"}, None, 3, id="unsupported-driver" + ), + ], +) +def test_preflight_rejects_untrusted_metadata(fake_tools, snapshot, env, metadata, expected_exit): + if metadata is not None: + Path(f"{snapshot}.meta").write_text(metadata) + + result = run_script({**fake_tools, **env}, "preflight", f"{snapshot}.meta") + + assert result.returncode == expected_exit + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +@pytest.mark.parametrize( + ("env", "remove_meta", "expected_exit"), + [ + pytest.param({}, True, 2, id="no-snapshot"), + pytest.param( + {"DAEMON_FINGERPRINT": "29.0.0 overlay2 x86_64 /var/lib/docker"}, False, 3, id="other-daemon" + ), + pytest.param({"ALL_IMAGES": "def456\\n"}, False, 3, id="daemon-holds-images"), + ], +) +def test_restore_leaves_daemon_alone_when_snapshot_does_not_fit( + fake_tools, snapshot, env, remove_meta, expected_exit +): + if remove_meta: + Path(f"{snapshot}.meta").unlink() + + result = run_script({**fake_tools, **env}, "restore", str(snapshot)) + + assert result.returncode == expected_exit + assert not [c for c in read_commands(fake_tools) if c[:2] == ["sudo", "systemctl"]] + + +def test_restore_empties_store_again_when_image_does_not_run(fake_tools, snapshot): + result = run_script({**fake_tools, "RUN_EXIT": "1"}, "restore", str(snapshot)) + + assert result.returncode == 4 + commands = read_commands(fake_tools) + run = next(i for i, c in enumerate(commands) if c[:2] == ["docker", "run"]) + after_run = commands[run + 1 :] + assert ["sudo", "rm", "-rf", "/var/lib/docker/image", "/var/lib/docker/overlay2"] in after_run + assert after_run[-1] == ["sudo", "systemctl", "start", "docker"] + + +def test_create_keeps_only_the_ci_image(fake_tools, tmp_path): + snapshot_file = tmp_path / "out.tar.zst" + env = {**fake_tools, "CI_IMAGES": "abc123\\nabc123\\n", "ALL_IMAGES": "abc123\\nother1\\nabc123\\n"} + + result = run_script(env, "create", str(snapshot_file)) + + assert result.returncode == 0, result.stdout + result.stderr + commands = read_commands(env) + assert ["docker", "rmi", "--force", "other1"] in commands + assert not [c for c in commands if c[:2] == ["docker", "rmi"] and "abc123" in c] + assert ["docker", "builder", "prune", "--all", "--force"] in commands + assert Path(f"{snapshot_file}.meta").read_text() == f"{FINGERPRINT} abc123 revision1\n" + archive = next(c for c in commands if c[:2] == ["sudo", "tar"]) + assert archive[-2:] == ["image", "overlay2"] + + +def test_create_refuses_ambiguous_ci_image(fake_tools, tmp_path): + env = {**fake_tools, "CI_IMAGES": "abc123\\ndef456\\n"} + + result = run_script(env, "create", str(tmp_path / "out.tar.zst")) + + assert result.returncode == 1 + assert not [c for c in read_commands(env) if c[:2] == ["sudo", "systemctl"]] + + +@pytest.mark.parametrize("failure", [{"ZSTD_EXIT": "1"}, {"TAR_EXIT": "1"}, {"START_FAIL_ONCE": "1"}]) +def test_restore_recovers_empty_running_daemon_after_materialization_failure(fake_tools, snapshot, failure): + result = run_script({**fake_tools, **failure}, "restore", str(snapshot)) + assert result.returncode != 0 + commands = read_commands(fake_tools) + assert commands[-2:] == [ + ["sudo", "rm", "-rf", "/var/lib/docker/image", "/var/lib/docker/overlay2"], + ["sudo", "systemctl", "start", "docker"], + ] + assert not any(command[:2] == ["docker", "run"] for command in commands) + + +@pytest.mark.parametrize("failure", [{"ZSTD_EXIT": "1"}, {"TAR_EXIT": "1"}]) +def test_create_restarts_daemon_after_archive_failure(fake_tools, tmp_path, failure): + result = run_script( + {**fake_tools, **failure, "CI_IMAGES": "abc123"}, "create", str(tmp_path / "snapshot.tar.zst") + ) + assert result.returncode != 0 + assert read_commands(fake_tools)[-1] == ["sudo", "systemctl", "start", "docker"] + + +@pytest.mark.parametrize("mode", ["create", "restore"]) +@pytest.mark.parametrize("daemon", ["28.5.2 btrfs x86_64 /var/lib/docker", "28.5.2 overlay2 x86_64 /other"]) +def test_unsupported_storage_does_not_modify_daemon(fake_tools, snapshot, mode, daemon): + result = run_script({**fake_tools, "DAEMON_FINGERPRINT": daemon}, mode, str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +def test_existing_container_prevents_restore(fake_tools, snapshot): + result = run_script({**fake_tools, "CONTAINERS": "container1"}, "restore", str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +@pytest.mark.parametrize("mode", ["restore", "preflight"]) +@pytest.mark.parametrize("failure", [{"IMAGES_EXIT": "1"}, {"CONTAINERS_EXIT": "1"}]) +def test_failed_daemon_inventory_does_not_modify_store(fake_tools, snapshot, failure, mode): + path = f"{snapshot}.meta" if mode == "preflight" else str(snapshot) + result = run_script({**fake_tools, **failure}, mode, path) + + assert result.returncode != 0 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +def test_malformed_metadata_prevents_restore(fake_tools, snapshot): + Path(f"{snapshot}.meta").write_text("invalid\n") + result = run_script(fake_tools, "restore", str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +def test_stale_snapshot_does_not_modify_daemon(fake_tools, snapshot): + result = run_script({**fake_tools, "CHECKOUT_SHA": "revision2"}, "restore", str(snapshot)) + assert result.returncode == 3 + assert not any(command[0] == "sudo" for command in read_commands(fake_tools)) + + +@pytest.mark.parametrize("mode", ["create", "restore"]) +def test_partial_stop_failure_restarts_daemon(fake_tools, snapshot, mode): + result = run_script({**fake_tools, "STOP_EXIT": "1", "CI_IMAGES": "abc123"}, mode, str(snapshot)) + assert result.returncode != 0 + assert read_commands(fake_tools)[-1] == ["sudo", "systemctl", "start", "docker"] + assert not any(command[:2] == ["sudo", "rm"] for command in read_commands(fake_tools))