diff --git a/README.md b/README.md index fee69f77..878c12af 100644 --- a/README.md +++ b/README.md @@ -73,48 +73,38 @@ or nix build .#custom-bundle-squashfs ``` -To build the full bundle as a gzip-compressed tar OCI image: +To build the full bundle as an OCI image: ``` nix build .#bundle-oci ``` -The result is an OCI image layout directory with the reference `bundle`, suitable -for copying with an OCI-layout-aware tool such as -`skopeo copy oci:./result:bundle docker://REGISTRY/REPOSITORY:TAG`. -The output also provides a copy helper that keeps the exact image in its Nix -closure and preserves the manifest digest: +The result is an OCI image layout directory with the reference `bundle`. Copy +it with the helper, which keeps the image in its Nix closure and preserves its +manifest digest: ``` nix run .#bundle-oci.copyTo -- docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest ``` -The destination can use any Skopeo-supported transport, including `oci:`. -Additional arguments are passed to `skopeo copy`. The helper trusts the locally -built source (`--insecure-policy`); registry TLS verification remains enabled. -For a clean Git source, the manifest records the full flake revision, source URL +The destination can use any Skopeo-supported transport. Additional arguments +are passed to `skopeo copy`. The helper trusts the locally built source +(`--insecure-policy`); registry TLS verification remains enabled. For a clean +Git source, the manifest records the full flake revision, source URL `https://github.com/replit/nixmodules`, and output name `bundle-oci` in OCI -annotations. The manifest creation annotation and config `created` field use -the flake's last-modified timestamp in UTC, never the build's wall clock. -Local sources without a clean revision omit the revision annotation; sources -without a last-modified timestamp omit creation metadata. Publish from a -revision-pinned Git flake when complete provenance is required. - -It contains one `application/vnd.oci.image.layer.v1.tar+gzip` layer. Gzip is -transport compression: the consumer decompresses it when unpacking the image. -The manifest describes the compressed blob's digest and size; the config's -diffID is the digest of the uncompressed tar stream. Tar entry order, -timestamps, ownership, extended headers and gzip headers are deterministic. - -The filesystem is generated directly from the same pinned full bundle used by -`disk-script`: its complete store closure and `/etc/nixmodules` metadata, with -unchanged store paths, file contents, modes and symlink targets. All guest -UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and -per-module OCI builders are unchanged. There is no disk conversion or dependency -update. The tar archive preserves hardlinks and extended attributes. A consumer -that converts the tar to EROFS must leave filesystem compression disabled for -files to remain DAX-eligible. This uses more unpacked disk space than compressed -EROFS; it does not automatically share all guest memory. +annotations. Creation metadata uses the flake's last-modified timestamp in UTC. +Local sources without revision or timestamp metadata omit those annotations. + +The image contains one `application/vnd.oci.image.layer.v1.erofs+zstd` layer. +The layer descriptor's digest and size cover the zstd-compressed data; its +`dev.replit.erofs.uncompressed-size` annotation records the raw EROFS image +size. The config diffID is the SHA-256 digest of that raw image. The EROFS +filesystem has internal compression and inline data disabled, with the complete +bundle closure and `/etc/nixmodules` metadata. + +This changes the existing `bundle-oci` format and is not backward-compatible +with consumers expecting a tar+gzip layer. Consumers must support this OCI +layer media type and EROFS; deploy that support before publishing images. A small fixture exercises the producer without building the full bundle: diff --git a/pkgs/bundle-oci/build.sh b/pkgs/bundle-oci/build.sh new file mode 100644 index 00000000..984b2eac --- /dev/null +++ b/pkgs/bundle-oci/build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +bundle="$1" +store_paths="$2" +output="$3" +architecture="$4" +metadata="$5" +layout_script="$6" + +mkdir -p root/nix/store root/etc/nixmodules "$output/blobs/sha256" +cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ +while IFS= read -r path; do + cp -a --reflink=auto "$path" root/nix/store/ +done < "$store_paths" +chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules + +env -u SOURCE_DATE_EPOCH mkfs.erofs -U 00000000-0000-0000-0000-000000000001 -T 1 --all-time \ + --force-uid=11000 --force-gid=11000 -E noinline_data raw.erofs root +zstd -15 --long=27 -T8 --check --no-progress --quiet raw.erofs -o layer.erofs.zst +python3 "$layout_script" raw.erofs layer.erofs.zst "$output" "$architecture" "$metadata" diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix index f9720788..2bcaa674 100644 --- a/pkgs/bundle-oci/check.nix +++ b/pkgs/bundle-oci/check.nix @@ -3,16 +3,15 @@ let dependency = pkgs.runCommand "bundle-oci-dependency" { } '' mkdir -p "$out" - printf 'dependency\n' > "$out/data" - head -c 131072 /dev/zero >> "$out/data" + printf 'fixture dependency\n' > "$out/data" ln "$out/data" "$out/hardlink" ''; module = pkgs.runCommand "bundle-oci-module" { } '' mkdir -p "$out/bin" printf '%s\n' '${dependency}' > "$out/module.json" printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool" - head -c 131072 /dev/zero >> "$out/bin/tool" - chmod 755 "$out/bin/tool" + chmod 751 "$out/bin/tool" + ln "$out/module.json" "$out/module-hardlink" ln -s ../module.json "$out/bin/relative" ln -s '${dependency}/data' "$out/absolute" ''; @@ -28,10 +27,15 @@ let in pkgs.runCommand "bundle-oci-check" { - nativeBuildInputs = [ pkgs.gnutar pkgs.gzip pkgs.python3 ]; + nativeBuildInputs = [ + pkgs.erofs-utils + pkgs.python3 + pkgs.skopeo + pkgs.zstd + ]; } '' - python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh} + python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${module} ${./layout.py} ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \ --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \ --digestfile "$PWD/copied.digest" @@ -44,7 +48,15 @@ pkgs.runCommand "bundle-oci-check" assert digest_file.read_text().strip() == descriptor["digest"] for blob in (original / "blobs/sha256").iterdir(): assert blob.read_bytes() == (copied / "blobs/sha256" / blob.name).read_bytes() - print("copyTo preserved manifest digest and every OCI blob") + copied_manifest = json.loads( + (copied / "blobs/sha256" / copied_descriptor["digest"].split(":")[1]).read_text() + ) + assert copied_manifest["layers"][0]["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" + original_manifest = json.loads( + (original / "blobs/sha256" / descriptor["digest"].split(":")[1]).read_text() + ) + assert copied_manifest["layers"][0]["annotations"] == original_manifest["layers"][0]["annotations"] + print("copyTo preserved the custom layer media type, manifest digest and every OCI blob") PY touch "$out" '' diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index f101a060..261e9445 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -1,17 +1,17 @@ import hashlib -import gzip -import errno import importlib.util +import io import json import os import pathlib +import re import stat -import shutil import subprocess import sys import tarfile -image, bundle, closure, dependency, layout_script, layer_script = map(pathlib.Path, sys.argv[1:]) + +image, bundle, closure, dependency, module, layout_script = map(pathlib.Path, sys.argv[1:]) assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"} @@ -31,7 +31,8 @@ def blob(descriptor): config = json.loads(blob(manifest["config"]).read_text()) assert config["architecture"] == "amd64" and config["os"] == "linux" assert config["created"] == "2023-11-14T22:13:20Z" -assert manifest["annotations"] == { +annotations = manifest["annotations"] +assert annotations == { "org.opencontainers.image.source": "https://github.com/replit/nixmodules", "org.opencontainers.image.revision": "0123456789abcdef0123456789abcdef01234567", "org.opencontainers.image.created": config["created"], @@ -39,61 +40,52 @@ def blob(descriptor): } assert len(manifest["layers"]) == 1 layer = manifest["layers"][0] -assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip" -archive = blob(layer) -tar_data = gzip.decompress(archive.read_bytes()) -diff_id = "sha256:" + hashlib.sha256(tar_data).hexdigest() -assert config["rootfs"] == {"type": "layers", "diff_ids": [diff_id]} -assert diff_id != layer["digest"] -assert archive.read_bytes()[3:8] == b"\x00" * 5 -spec = importlib.util.spec_from_file_location("layout", layout_script) -layout = importlib.util.module_from_spec(spec) -spec.loader.exec_module(layout) -repeated = pathlib.Path("repeated") -(repeated / "blobs/sha256").mkdir(parents=True) -shutil.copyfile(archive, "repeated.tar.gz") -layout.write_layout(pathlib.Path("repeated.tar.gz"), repeated, "x86_64", { - "revision": "0123456789abcdef0123456789abcdef01234567", - "sourceTimestamp": 1700000000, -}) -for path in image.rglob("*"): - if path.is_file(): - assert path.read_bytes() == (repeated / path.relative_to(image)).read_bytes() -local = pathlib.Path("local") -(local / "blobs/sha256").mkdir(parents=True) -shutil.copyfile(archive, "local.tar.gz") -layout.write_layout(pathlib.Path("local.tar.gz"), local, "x86_64", { - "revision": None, "sourceTimestamp": None, -}) -local_descriptor = json.loads((local / "index.json").read_text())["manifests"][0] -local_manifest = json.loads( - (local / "blobs/sha256" / local_descriptor["digest"].split(":")[1]).read_text() +assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"].isdecimal() +assert set(layer["annotations"]) == {"dev.replit.erofs.uncompressed-size"} +compressed = blob(layer) +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str( + int(layer["annotations"]["dev.replit.erofs.uncompressed-size"]) +) +raw = pathlib.Path("raw.erofs") +subprocess.run(["zstd", "-d", "-f", str(compressed), "-o", str(raw)], check=True) +raw_bytes = raw.read_bytes() +raw_digest = "sha256:" + hashlib.sha256(raw_bytes).hexdigest() +assert config["rootfs"] == {"type": "layers", "diff_ids": [raw_digest]} +assert layer["digest"] != raw_digest +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str(len(raw_bytes)) +dump_stats = subprocess.run( + ["dump.erofs", "-S", str(raw)], capture_output=True, text=True, check=True +).stdout +assert re.search(r"Filesystem compressed files:\s+0\b", dump_stats) +dump_superblock = subprocess.run( + ["dump.erofs", "-s", str(raw)], capture_output=True, text=True, check=True +).stdout +assert ( + "Filesystem UUID: 00000000-0000-0000-0000-000000000001" + in dump_superblock ) -assert "org.opencontainers.image.revision" not in local_manifest["annotations"] -assert "org.opencontainers.image.created" not in local_manifest["annotations"] -local_config = json.loads( - (local / "blobs/sha256" / local_manifest["config"]["digest"].split(":")[1]).read_text() +features = next(line for line in dump_superblock.splitlines() if "Filesystem features:" in line) +assert "compr_cfgs" not in features +tool_inode = subprocess.run( + ["dump.erofs", f"--path={module}/bin/tool", str(raw)], + capture_output=True, + text=True, + check=True, +).stdout +assert "Layout: 0" in tool_inode +assert "Uid: 11000 Gid: 11000" in tool_inode +subprocess.run( + [ + "fsck.erofs", + "--preserve-perms", + *(["--preserve-owner"] if os.geteuid() == 0 else []), + f"--extract={pathlib.Path('extracted')}", + str(raw), + ], + check=True, ) -assert "created" not in local_config root = pathlib.Path("extracted") -root.mkdir() -subprocess.run([ - "tar", "--extract", "--gzip", "--file", str(archive), - "--directory", str(root), "--no-same-owner", "--same-permissions", - "--xattrs", "--xattrs-include=*", -], check=True) -with tarfile.open(archive, "r:gz") as source: - members = source.getmembers() - assert all(m.uid == 11000 and m.gid == 11000 and m.mtime == 1 for m in members) - assert all(not m.uname and not m.gname for m in members) - assert all("atime" not in m.pax_headers and "ctime" not in m.pax_headers for m in members) - assert [m.name for m in members] == sorted(m.name for m in members) - -rebuilt = pathlib.Path("rebuilt.tar.gz") -subprocess.run(["bash", str(layer_script), str(root), str(rebuilt)], check=True) -assert rebuilt.read_bytes() == archive.read_bytes() -print("Tar metadata and reproducible gzip layer verified") - paths = closure.read_text().splitlines() assert str(dependency) in paths assert {p.name for p in (root / "nix/store").iterdir()} == { @@ -123,37 +115,174 @@ def compare(source, target): assert (packed_dependency / "data").stat().st_ino == ( packed_dependency / "hardlink" ).stat().st_ino -print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified") - -xattr_root = pathlib.Path("xattrs") -xattr_root.mkdir() -xattr_file = xattr_root / ("long-name-" * 20) -xattr_file.write_bytes(b"xattrs\n") -try: - os.setxattr(xattr_file, "user.fixture", b"preserved") - xattrs_supported = True -except OSError as error: - if error.errno != errno.ENOTSUP: - raise - xattrs_supported = False - print("Skipping xattr round-trip: build filesystem does not support xattrs") -subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs.tar.gz"], check=True) -with tarfile.open("xattrs.tar.gz", "r:gz") as source: - member = source.getmember("./" + xattr_file.name) - assert member.isfile() and source.extractfile(member).read() == b"xattrs\n" - if xattrs_supported: - assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved" -if xattrs_supported: - restored = pathlib.Path("xattrs-restored") - restored.mkdir() - subprocess.run([ - "tar", "--extract", "--gzip", "--file=xattrs.tar.gz", - "--directory", str(restored), "--no-same-owner", - "--xattrs", "--xattrs-include=*", - ], check=True) - assert os.getxattr(restored / xattr_file.name, "user.fixture") == b"preserved" - print("Extended attributes preserved in archive and extraction") -os.utime(xattr_file, (12345, 67890)) -subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs-again.tar.gz"], check=True) -assert pathlib.Path("xattrs.tar.gz").read_bytes() == pathlib.Path("xattrs-again.tar.gz").read_bytes() -print("Long paths preserved; source timestamps do not change the layer") +fixture_module = root / str(module).lstrip("/") +assert (fixture_module / "module.json").stat().st_ino == ( + fixture_module / "module-hardlink" +).stat().st_ino +tool_mode = stat.S_IMODE((module / "bin/tool").stat().st_mode) +assert tool_mode & 0o111 +assert stat.S_IMODE((fixture_module / "bin/tool").stat().st_mode) == tool_mode +assert os.readlink(fixture_module / "bin/relative") == "../module.json" +assert os.readlink(fixture_module / "absolute") == f"{dependency}/data" +if os.geteuid() == 0: + assert (fixture_module / "bin/tool").stat().st_uid == 11000 + assert (fixture_module / "bin/tool").stat().st_gid == 11000 +print("EROFS payload, closure, plain layout, modes, symlinks, hardlinks and ownership verified") + +layout_spec = importlib.util.spec_from_file_location("layout", layout_script) +layout = importlib.util.module_from_spec(layout_spec) +layout_spec.loader.exec_module(layout) +layout.verify_layer(raw, compressed) +metadata = { + "revision": "0123456789abcdef0123456789abcdef01234567", + "sourceTimestamp": 1700000000, +} + + +def expect_error(raw_path, compressed_path): + try: + layout.verify_layer(raw_path, compressed_path) + except (OSError, ValueError, subprocess.SubprocessError): + return + raise AssertionError("invalid layout input was accepted") + + +truncated = pathlib.Path("truncated.zst") +truncated.write_bytes(compressed.read_bytes()[:-4]) +expect_error(raw, truncated) +corrupt = pathlib.Path("corrupt.zst") +corrupt.write_bytes(compressed.read_bytes()[:-1] + bytes([compressed.read_bytes()[-1] ^ 0xFF])) +expect_error(raw, corrupt) +mismatch = pathlib.Path("mismatch.erofs") +mismatch.write_bytes(raw_bytes + b"mismatch") +expect_error(mismatch, compressed) +expect_error(pathlib.Path("missing.erofs"), compressed) +for architecture, invalid_metadata in ( + ("unsupported", metadata), + ("x86_64", {"revision": "bad", "sourceTimestamp": None}), +): + invalid_output = pathlib.Path("invalid-layout") + (invalid_output / "blobs/sha256").mkdir(parents=True, exist_ok=True) + try: + layout.write_layout(raw, compressed, invalid_output, architecture, invalid_metadata) + except ValueError: + continue + raise AssertionError("invalid layout metadata was accepted") + +repeat = pathlib.Path("repeat.zst") +subprocess.run( + [ + "zstd", + "-15", + "--long=27", + "-T8", + "--check", + "--no-progress", + "--quiet", + str(raw), + "-o", + str(repeat), + ], + check=True, +) +assert repeat.read_bytes() == compressed.read_bytes() +repeat_layout = pathlib.Path("repeat-layout") +(repeat_layout / "blobs/sha256").mkdir(parents=True) +layout.write_layout(raw, repeat, repeat_layout, "x86_64", metadata) +for path in image.rglob("*"): + if path.is_file(): + assert path.read_bytes() == (repeat_layout / path.relative_to(image)).read_bytes() + +timestamp_root = pathlib.Path("timestamp-root") +timestamp_root.mkdir() +timestamp_file = timestamp_root / "data" +timestamp_file.write_bytes(b"deterministic EROFS fixture\n") +os.chmod(timestamp_file, 0o751) +os.link(timestamp_file, timestamp_root / "hardlink") +(timestamp_root / "relative").symlink_to("data") +timestamp_mkfs_args = [ + "env", + "-u", + "SOURCE_DATE_EPOCH", + "mkfs.erofs", + "-U", "00000000-0000-0000-0000-000000000001", + "-T", "1", + "--all-time", + "--force-uid=11000", + "--force-gid=11000", + "-E", "noinline_data", +] +timestamp_raw = pathlib.Path("timestamp.erofs") +subprocess.run( + timestamp_mkfs_args + [str(timestamp_raw), str(timestamp_root)], + check=True, +) +os.utime(timestamp_file, (23456, 23456)) +os.utime(timestamp_root, (34567, 34567)) +os.utime(timestamp_root / "relative", (45678, 45678), follow_symlinks=False) +timestamp_repeat = pathlib.Path("timestamp-repeat.erofs") +subprocess.run( + timestamp_mkfs_args + [str(timestamp_repeat), str(timestamp_root)], + check=True, +) +assert timestamp_raw.read_bytes() == timestamp_repeat.read_bytes() + +xattr_tar = pathlib.Path("xattr.tar") +xattr_value = b"nixmodules-fixture-xattr-value" +xattr_content = b"extended attribute fixture\n" + + +def write_xattr_tar(): + with tarfile.open(xattr_tar, "w", format=tarfile.PAX_FORMAT) as archive: + entry = tarfile.TarInfo("xattr-data") + entry.size = len(xattr_content) + entry.mode = 0o751 + entry.mtime = 1 + entry.pax_headers = {"SCHILY.xattr.user.fixture": xattr_value.decode()} + archive.addfile(entry, io.BytesIO(xattr_content)) + + +write_xattr_tar() +xattr_raw = pathlib.Path("xattr.erofs") +mkfs_args = [ + "env", + "-u", + "SOURCE_DATE_EPOCH", + "mkfs.erofs", + "-U", "00000000-0000-0000-0000-000000000001", + "-T", "1", + "--all-time", + "--force-uid=11000", + "--force-gid=11000", + "-E", "noinline_data", + "--tar=f", + str(xattr_raw), + str(xattr_tar), +] +subprocess.run(mkfs_args, check=True) +xattr_dump = subprocess.run( + ["dump.erofs", "--path=/xattr-data", str(xattr_raw)], + capture_output=True, + text=True, + check=True, +).stdout +assert "Layout: 0" in xattr_dump +assert "Uid: 11000 Gid: 11000" in xattr_dump +assert re.search(r"Xattr size: [1-9]\d*", xattr_dump) +assert xattr_value in xattr_raw.read_bytes() +xattr_extracted = pathlib.Path("xattr-extracted") +subprocess.run( + [ + "fsck.erofs", + "--preserve-perms", + *(["--preserve-owner"] if os.geteuid() == 0 else []), + f"--extract={xattr_extracted}", + str(xattr_raw), + ], + check=True, +) +assert (xattr_extracted / "xattr-data").read_bytes() == xattr_content +if os.geteuid() == 0: + assert (xattr_extracted / "xattr-data").stat().st_uid == 11000 + assert (xattr_extracted / "xattr-data").stat().st_gid == 11000 +print("Deterministic raw and compressed outputs, xattrs, forced ownership and invalid-input checks verified") diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix index ae357cff..51066c1f 100644 --- a/pkgs/bundle-oci/default.nix +++ b/pkgs/bundle-oci/default.nix @@ -3,8 +3,8 @@ , stdenv , bundle , closureInfo -, gnutar -, gzip +, erofs-utils +, zstd , python3 , writeShellApplication , skopeo @@ -17,7 +17,7 @@ let metadata = builtins.toJSON { inherit revision sourceTimestamp; }; image = runCommand "nixmodules-bundle-oci" { - nativeBuildInputs = [ gnutar gzip python3 ]; + nativeBuildInputs = [ erofs-utils zstd python3 ]; inherit bundle bundleClosure; architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; __structuredAttrs = true; @@ -39,15 +39,7 @@ let }; } '' - mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" - cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ - while IFS= read -r path; do - cp -a --reflink=auto "$path" root/nix/store/ - done < "$bundleClosure/store-paths" - chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules - - bash ${./layer.sh} root layer.tar.gz - python3 ${./layout.py} layer.tar.gz "$out" "$architecture" ${lib.escapeShellArg metadata} + bash ${./build.sh} "$bundle" "$bundleClosure/store-paths" "$out" "$architecture" ${lib.escapeShellArg metadata} ${./layout.py} ''; in image diff --git a/pkgs/bundle-oci/layer.sh b/pkgs/bundle-oci/layer.sh deleted file mode 100644 index c215cd72..00000000 --- a/pkgs/bundle-oci/layer.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -euo pipefail - -export LC_ALL=C -tar --create --format=pax --sort=name --mtime=@1 \ - --owner=11000 --group=11000 --numeric-owner \ - --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ - --xattrs --xattrs-include='*' \ - --directory="$1" . | gzip --no-name -9 > "$2" diff --git a/pkgs/bundle-oci/layout.py b/pkgs/bundle-oci/layout.py index 02f6b228..5b199c29 100644 --- a/pkgs/bundle-oci/layout.py +++ b/pkgs/bundle-oci/layout.py @@ -1,15 +1,47 @@ import hashlib import datetime -import gzip import json import pathlib import shutil +import subprocess import sys -def write_layout(layer, output, architecture, metadata): +def verify_layer(raw_image, compressed_layer): + raw_image = pathlib.Path(raw_image) + compressed_layer = pathlib.Path(compressed_layer) + raw_digest = hashlib.sha256() + raw_size = 0 + with raw_image.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + raw_digest.update(chunk) + raw_size += len(chunk) + + process = subprocess.Popen( + ["zstd", "-d", "-c", "--", str(compressed_layer)], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + decompressed_digest = hashlib.sha256() + decompressed_size = 0 + assert process.stdout is not None + for chunk in iter(lambda: process.stdout.read(1024 * 1024), b""): + decompressed_digest.update(chunk) + decompressed_size += len(chunk) + if process.wait() != 0: + raise ValueError("compressed EROFS layer failed zstd verification") + if decompressed_size != raw_size or decompressed_digest.digest() != raw_digest.digest(): + raise ValueError("compressed layer does not match the raw EROFS image") + + +def write_layout(raw_image, compressed_layer, output, architecture, metadata): architectures = {"x86_64": "amd64", "aarch64": "arm64"} + if architecture not in architectures: + raise ValueError(f"unsupported architecture: {architecture}") architecture = architectures[architecture] + raw_image = pathlib.Path(raw_image) + compressed_layer = pathlib.Path(compressed_layer) + output = pathlib.Path(output) blobs = output / "blobs" / "sha256" annotations = { "org.opencontainers.image.source": "https://github.com/replit/nixmodules", @@ -28,14 +60,22 @@ def write_layout(layer, output, architecture, metadata): annotations["org.opencontainers.image.created"] = timestamp created["created"] = timestamp - def descriptor(path, media_type): - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) + raw_digest = hashlib.sha256() + raw_size = 0 + with raw_image.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + raw_digest.update(chunk) + raw_size += len(chunk) + + compressed_digest = hashlib.sha256() + with compressed_layer.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + compressed_digest.update(chunk) + + def descriptor(path, media_type, digest=None): return { "mediaType": media_type, - "digest": "sha256:" + digest.hexdigest(), + "digest": "sha256:" + (digest or hashlib.sha256(path.read_bytes()).hexdigest()), "size": path.stat().st_size, } @@ -45,19 +85,22 @@ def store_json(value, media_type): path.write_bytes(data) return descriptor(path, media_type) - diff_digest = hashlib.sha256() - with gzip.open(layer, "rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - diff_digest.update(chunk) - layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.tar+gzip") - shutil.move(layer, blobs / layer_descriptor["digest"].split(":")[1]) + layer_descriptor = { + **descriptor( + compressed_layer, + "application/vnd.oci.image.layer.v1.erofs+zstd", + compressed_digest.hexdigest(), + ), + "annotations": {"dev.replit.erofs.uncompressed-size": str(raw_size)}, + } + shutil.move(compressed_layer, blobs / compressed_digest.hexdigest()) config = store_json( { **created, "architecture": architecture, "os": "linux", "config": {}, - "rootfs": {"type": "layers", "diff_ids": ["sha256:" + diff_digest.hexdigest()]}, + "rootfs": {"type": "layers", "diff_ids": ["sha256:" + raw_digest.hexdigest()]}, }, "application/vnd.oci.image.config.v1+json", ) @@ -80,6 +123,9 @@ def store_json(value, media_type): if __name__ == "__main__": write_layout( - pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), - sys.argv[3], json.loads(sys.argv[4]), + pathlib.Path(sys.argv[1]), + pathlib.Path(sys.argv[2]), + pathlib.Path(sys.argv[3]), + sys.argv[4], + json.loads(sys.argv[5]), )