From ec0f4dfd40bc288f1d88c6c1ce931f59f714927b Mon Sep 17 00:00:00 2001 From: rahul Date: Thu, 15 Oct 2026 07:06:44 +0000 Subject: [PATCH 1/2] Add optional zstd-compressed EROFS bundle image Assisted-by: Replit --- README.md | 27 +++ pkgs/bundle-oci-erofs-zstd/build.sh | 21 +++ pkgs/bundle-oci-erofs-zstd/check.nix | 62 +++++++ pkgs/bundle-oci-erofs-zstd/check.py | 239 +++++++++++++++++++++++++ pkgs/bundle-oci-erofs-zstd/default.nix | 44 +++++ pkgs/bundle-oci-erofs-zstd/layout.py | 132 ++++++++++++++ pkgs/default.nix | 8 + scripts/ci_check.sh | 1 + 8 files changed, 534 insertions(+) create mode 100644 pkgs/bundle-oci-erofs-zstd/build.sh create mode 100644 pkgs/bundle-oci-erofs-zstd/check.nix create mode 100644 pkgs/bundle-oci-erofs-zstd/check.py create mode 100644 pkgs/bundle-oci-erofs-zstd/default.nix create mode 100644 pkgs/bundle-oci-erofs-zstd/layout.py diff --git a/README.md b/README.md index fee69f77..62761530 100644 --- a/README.md +++ b/README.md @@ -121,3 +121,30 @@ A small fixture exercises the producer without building the full bundle: ``` nix build .#bundle-oci-check ``` + +## Optional EROFS OCI image + +An opt-in image is available as `bundle-oci-erofs-zstd`; the existing +`bundle-oci` output remains the default and is unchanged. Build and copy it with: + +``` +nix build .#bundle-oci-erofs-zstd +nix run .#bundle-oci-erofs-zstd.copyTo -- docker://REGISTRY/REPOSITORY:TAG +``` + +This image has one `application/vnd.oci.image.layer.v1.erofs+zstd` layer. +Consumers must explicitly support this OCI layer media type and the EROFS +filesystem before using this output; generic OCI consumers may reject it. The +descriptor digest and size cover the zstd-compressed layer, while the config +diffID covers the raw EROFS image. The layer descriptor's +`dev.replit.erofs.uncompressed-size` annotation reports the raw image size in +bytes. +The filesystem is built without EROFS internal compression and with inline +data disabled. + +The small fixture checks the image format and copy helper without building the +full bundle: + +``` +nix build .#bundle-oci-erofs-zstd-check +``` diff --git a/pkgs/bundle-oci-erofs-zstd/build.sh b/pkgs/bundle-oci-erofs-zstd/build.sh new file mode 100644 index 00000000..013a40ff --- /dev/null +++ b/pkgs/bundle-oci-erofs-zstd/build.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail + +bundle="$1" +store_paths="$2" +output="$3" +architecture="$4" +metadata="$5" +layout_script="$6" + +mkdir -p root/nix/store root/etc/nixmodules "$output/blobs/sha256" +cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ +while IFS= read -r path; do + cp -a --reflink=auto "$path" root/nix/store/ +done < "$store_paths" +chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules + +mkfs.erofs -U 00000000-0000-0000-0000-000000000001 -T 1 \ + --force-uid=11000 --force-gid=11000 -E noinline_data raw.erofs root +zstd -15 --long=27 -T8 --check --no-progress --quiet raw.erofs -o layer.erofs.zst +python3 "$layout_script" raw.erofs layer.erofs.zst "$output" "$architecture" "$metadata" diff --git a/pkgs/bundle-oci-erofs-zstd/check.nix b/pkgs/bundle-oci-erofs-zstd/check.nix new file mode 100644 index 00000000..0ff90e0b --- /dev/null +++ b/pkgs/bundle-oci-erofs-zstd/check.nix @@ -0,0 +1,62 @@ +{ pkgs }: + +let + dependency = pkgs.runCommand "bundle-oci-erofs-zstd-dependency" { } '' + mkdir -p "$out" + printf 'fixture dependency\n' > "$out/data" + ln "$out/data" "$out/hardlink" + ''; + module = pkgs.runCommand "bundle-oci-erofs-zstd-module" { } '' + mkdir -p "$out/bin" + printf '%s\n' '${dependency}' > "$out/module.json" + printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool" + chmod 751 "$out/bin/tool" + ln "$out/module.json" "$out/module-hardlink" + ln -s ../module.json "$out/bin/relative" + ln -s '${dependency}/data' "$out/absolute" + ''; + bundle = (pkgs.callPackage ../bundle { + self = { modules.fixture = module; }; + }) { }; + image = pkgs.callPackage ./. { + inherit bundle; + revision = "0123456789abcdef0123456789abcdef01234567"; + sourceTimestamp = 1700000000; + }; + closure = pkgs.closureInfo { rootPaths = [ bundle ]; }; +in +pkgs.runCommand "bundle-oci-erofs-zstd-check" +{ + nativeBuildInputs = [ + pkgs.erofs-utils + pkgs.python3 + pkgs.skopeo + pkgs.zstd + ]; +} + '' + python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${module} ${./layout.py} + ${image.copyTo}/bin/copy-nixmodules-bundle-oci-erofs-zstd "oci:$PWD/copied:bundle" \ + --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \ + --digestfile "$PWD/copied.digest" + python3 - ${image} "$PWD/copied" "$PWD/copied.digest" <<'PY' + import json, pathlib, sys + original, copied, digest_file = map(pathlib.Path, sys.argv[1:]) + descriptor = json.loads((original / "index.json").read_text())["manifests"][0] + copied_descriptor = json.loads((copied / "index.json").read_text())["manifests"][0] + assert copied_descriptor["digest"] == descriptor["digest"] + assert digest_file.read_text().strip() == descriptor["digest"] + for blob in (original / "blobs/sha256").iterdir(): + assert blob.read_bytes() == (copied / "blobs/sha256" / blob.name).read_bytes() + copied_manifest = json.loads( + (copied / "blobs/sha256" / copied_descriptor["digest"].split(":")[1]).read_text() + ) + assert copied_manifest["layers"][0]["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" + original_manifest = json.loads( + (original / "blobs/sha256" / descriptor["digest"].split(":")[1]).read_text() + ) + assert copied_manifest["layers"][0]["annotations"] == original_manifest["layers"][0]["annotations"] + print("copyTo preserved the custom layer media type, manifest digest and every OCI blob") + PY + touch "$out" + '' diff --git a/pkgs/bundle-oci-erofs-zstd/check.py b/pkgs/bundle-oci-erofs-zstd/check.py new file mode 100644 index 00000000..8ff7451b --- /dev/null +++ b/pkgs/bundle-oci-erofs-zstd/check.py @@ -0,0 +1,239 @@ +import hashlib +import importlib.util +import json +import os +import pathlib +import re +import stat +import subprocess +import sys + + +image, bundle, closure, dependency, module, layout_script = map(pathlib.Path, sys.argv[1:]) +assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"} + + +def blob(descriptor): + path = image / "blobs" / "sha256" / descriptor["digest"].removeprefix("sha256:") + assert path.stat().st_size == descriptor["size"] + assert "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() == descriptor["digest"] + return path + + +index = json.loads((image / "index.json").read_text()) +assert index["schemaVersion"] == 2 and len(index["manifests"]) == 1 +manifest = json.loads(blob(index["manifests"][0]).read_text()) +assert manifest["schemaVersion"] == 2 +assert manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json" +assert manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json" +config = json.loads(blob(manifest["config"]).read_text()) +assert config["architecture"] == "amd64" and config["os"] == "linux" +assert config["created"] == "2023-11-14T22:13:20Z" +annotations = manifest["annotations"] +assert annotations == { + "org.opencontainers.image.source": "https://github.com/replit/nixmodules", + "org.opencontainers.image.revision": "0123456789abcdef0123456789abcdef01234567", + "org.opencontainers.image.created": config["created"], + "dev.replit.nixmodules.flake-output": "bundle-oci-erofs-zstd", +} +assert len(manifest["layers"]) == 1 +layer = manifest["layers"][0] +assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"].isdecimal() +assert set(layer["annotations"]) == {"dev.replit.erofs.uncompressed-size"} +compressed = blob(layer) +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str( + int(layer["annotations"]["dev.replit.erofs.uncompressed-size"]) +) +raw = pathlib.Path("raw.erofs") +subprocess.run(["zstd", "-d", "-f", str(compressed), "-o", str(raw)], check=True) +raw_bytes = raw.read_bytes() +raw_digest = "sha256:" + hashlib.sha256(raw_bytes).hexdigest() +assert config["rootfs"] == {"type": "layers", "diff_ids": [raw_digest]} +assert layer["digest"] != raw_digest +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str(len(raw_bytes)) +dump_stats = subprocess.run( + ["dump.erofs", "-S", str(raw)], capture_output=True, text=True, check=True +).stdout +assert re.search(r"Filesystem compressed files:\s+0\b", dump_stats) +dump_superblock = subprocess.run( + ["dump.erofs", "-s", str(raw)], capture_output=True, text=True, check=True +).stdout +assert ( + "Filesystem UUID: 00000000-0000-0000-0000-000000000001" + in dump_superblock +) +features = next(line for line in dump_superblock.splitlines() if "Filesystem features:" in line) +assert "compr_cfgs" not in features +tool_inode = subprocess.run( + ["dump.erofs", f"--path={module}/bin/tool", str(raw)], + capture_output=True, + text=True, + check=True, +).stdout +assert "Layout: 0" in tool_inode +assert "Uid: 11000 Gid: 11000" in tool_inode +subprocess.run( + [ + "fsck.erofs", + "--preserve-perms", + "--xattrs", + *(["--preserve-owner"] if os.geteuid() == 0 else []), + f"--extract={pathlib.Path('extracted')}", + str(raw), + ], + check=True, +) +root = pathlib.Path("extracted") +paths = closure.read_text().splitlines() +assert str(dependency) in paths +assert {p.name for p in (root / "nix/store").iterdir()} == { + pathlib.Path(p).name for p in paths +} + + +def compare(source, target): + before, after = source.lstat(), target.lstat() + assert stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode), target + assert stat.S_IMODE(before.st_mode) == stat.S_IMODE(after.st_mode), target + if source.is_symlink(): + assert os.readlink(source) == os.readlink(target), target + elif source.is_dir(): + assert {p.name for p in source.iterdir()} == {p.name for p in target.iterdir()} + for child in source.iterdir(): + compare(child, target / child.name) + else: + assert source.read_bytes() == target.read_bytes(), target + + +for path in paths: + compare(pathlib.Path(path), root / path.lstrip("/")) +for metadata in (bundle / "etc/nixmodules").iterdir(): + compare(metadata, root / "etc/nixmodules" / metadata.name) +packed_dependency = root / str(dependency).lstrip("/") +assert (packed_dependency / "data").stat().st_ino == ( + packed_dependency / "hardlink" +).stat().st_ino +fixture_module = root / str(module).lstrip("/") +assert (fixture_module / "module.json").stat().st_ino == ( + fixture_module / "module-hardlink" +).stat().st_ino +assert stat.S_IMODE((fixture_module / "bin/tool").stat().st_mode) == 0o751 +assert os.readlink(fixture_module / "bin/relative") == "../module.json" +assert os.readlink(fixture_module / "absolute") == f"{dependency}/data" +if os.geteuid() == 0: + assert (fixture_module / "bin/tool").stat().st_uid == 11000 + assert (fixture_module / "bin/tool").stat().st_gid == 11000 +print("EROFS payload, closure, plain layout, modes, symlinks, hardlinks and ownership verified") + +layout_spec = importlib.util.spec_from_file_location("layout", layout_script) +layout = importlib.util.module_from_spec(layout_spec) +layout_spec.loader.exec_module(layout) +layout.verify_layer(raw, compressed) +metadata = { + "revision": "0123456789abcdef0123456789abcdef01234567", + "sourceTimestamp": 1700000000, +} + + +def expect_error(raw_path, compressed_path): + try: + layout.verify_layer(raw_path, compressed_path) + except (OSError, ValueError, subprocess.SubprocessError): + return + raise AssertionError("invalid layout input was accepted") + + +truncated = pathlib.Path("truncated.zst") +truncated.write_bytes(compressed.read_bytes()[:-4]) +expect_error(raw, truncated) +corrupt = pathlib.Path("corrupt.zst") +corrupt.write_bytes(compressed.read_bytes()[:-1] + bytes([compressed.read_bytes()[-1] ^ 0xFF])) +expect_error(raw, corrupt) +mismatch = pathlib.Path("mismatch.erofs") +mismatch.write_bytes(raw_bytes + b"mismatch") +expect_error(mismatch, compressed) +expect_error(pathlib.Path("missing.erofs"), compressed) +for architecture, invalid_metadata in ( + ("unsupported", metadata), + ("x86_64", {"revision": "bad", "sourceTimestamp": None}), +): + invalid_output = pathlib.Path("invalid-layout") + (invalid_output / "blobs/sha256").mkdir(parents=True, exist_ok=True) + try: + layout.write_layout(raw, compressed, invalid_output, architecture, invalid_metadata) + except ValueError: + continue + raise AssertionError("invalid layout metadata was accepted") + +repeat = pathlib.Path("repeat.zst") +subprocess.run( + [ + "zstd", + "-15", + "--long=27", + "-T8", + "--check", + "--no-progress", + "--quiet", + str(raw), + "-o", + str(repeat), + ], + check=True, +) +assert repeat.read_bytes() == compressed.read_bytes() +repeat_layout = pathlib.Path("repeat-layout") +(repeat_layout / "blobs/sha256").mkdir(parents=True) +layout.write_layout(raw, repeat, repeat_layout, "x86_64", metadata) +for path in image.rglob("*"): + if path.is_file(): + assert path.read_bytes() == (repeat_layout / path.relative_to(image)).read_bytes() + +xattr_root = pathlib.Path("xattr-root") +xattr_root.mkdir() +xattr_file = xattr_root / "xattr-data" +xattr_file.write_bytes(b"extended attribute fixture\n") +os.setxattr(xattr_file, "user.fixture", b"preserved") +xattr_raw = pathlib.Path("xattr.erofs") +mkfs_args = [ + "mkfs.erofs", + "-U", "00000000-0000-0000-0000-000000000001", + "-T", "1", + "--force-uid=11000", + "--force-gid=11000", + "-E", "noinline_data", + str(xattr_raw), + str(xattr_root), +] +subprocess.run(mkfs_args, check=True) +xattr_dump = subprocess.run( + ["dump.erofs", f"--path=/{xattr_file.name}", str(xattr_raw)], + capture_output=True, + text=True, + check=True, +).stdout +assert "Layout: 0" in xattr_dump +assert "Uid: 11000 Gid: 11000" in xattr_dump +os.utime(xattr_file, (12345, 12345)) +os.utime(xattr_root, (23456, 23456)) +xattr_repeat = pathlib.Path("xattr-repeat.erofs") +subprocess.run(mkfs_args[:-2] + [str(xattr_repeat), str(xattr_root)], check=True) +assert xattr_raw.read_bytes() == xattr_repeat.read_bytes() +xattr_extracted = pathlib.Path("xattr-extracted") +subprocess.run( + [ + "fsck.erofs", + "--preserve-perms", + "--xattrs", + *(["--preserve-owner"] if os.geteuid() == 0 else []), + f"--extract={xattr_extracted}", + str(xattr_raw), + ], + check=True, +) +assert os.getxattr(xattr_extracted / "xattr-data", "user.fixture") == b"preserved" +if os.geteuid() == 0: + assert (xattr_extracted / "xattr-data").stat().st_uid == 11000 + assert (xattr_extracted / "xattr-data").stat().st_gid == 11000 +print("Deterministic compression, xattrs, forced ownership and invalid-input checks verified") diff --git a/pkgs/bundle-oci-erofs-zstd/default.nix b/pkgs/bundle-oci-erofs-zstd/default.nix new file mode 100644 index 00000000..05552114 --- /dev/null +++ b/pkgs/bundle-oci-erofs-zstd/default.nix @@ -0,0 +1,44 @@ +{ runCommand +, lib +, stdenv +, bundle +, closureInfo +, erofs-utils +, zstd +, python3 +, writeShellApplication +, skopeo +, revision ? null +, sourceTimestamp ? null +}: + +let + bundleClosure = closureInfo { rootPaths = [ bundle ]; }; + metadata = builtins.toJSON { inherit revision sourceTimestamp; }; + image = runCommand "nixmodules-bundle-oci-erofs-zstd" + { + nativeBuildInputs = [ erofs-utils zstd python3 ]; + inherit bundle bundleClosure; + architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; + __structuredAttrs = true; + unsafeDiscardReferences.out = true; + passthru.copyTo = writeShellApplication { + name = "copy-nixmodules-bundle-oci-erofs-zstd"; + runtimeInputs = [ skopeo ]; + text = '' + if [ "$#" -lt 1 ]; then + echo "usage: copy-nixmodules-bundle-oci-erofs-zstd [skopeo copy options]" >&2 + exit 1 + fi + destination="$1" + shift + exec skopeo --insecure-policy copy --preserve-digests \ + "oci:${image}:bundle" "$destination" "$@" + ''; + }; + } + '' + bash ${./build.sh} "$bundle" "$bundleClosure/store-paths" "$out" "$architecture" ${lib.escapeShellArg metadata} ${./layout.py} + ''; +in +image diff --git a/pkgs/bundle-oci-erofs-zstd/layout.py b/pkgs/bundle-oci-erofs-zstd/layout.py new file mode 100644 index 00000000..66bbf4f4 --- /dev/null +++ b/pkgs/bundle-oci-erofs-zstd/layout.py @@ -0,0 +1,132 @@ +import datetime +import hashlib +import json +import pathlib +import shutil +import subprocess +import sys + + +def verify_layer(raw_image, compressed_layer): + raw_image = pathlib.Path(raw_image) + compressed_layer = pathlib.Path(compressed_layer) + raw_digest = hashlib.sha256() + raw_size = 0 + with raw_image.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + raw_digest.update(chunk) + raw_size += len(chunk) + + process = subprocess.Popen( + ["zstd", "-d", "-c", "--", str(compressed_layer)], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + decompressed_digest = hashlib.sha256() + decompressed_size = 0 + assert process.stdout is not None + for chunk in iter(lambda: process.stdout.read(1024 * 1024), b""): + decompressed_digest.update(chunk) + decompressed_size += len(chunk) + if process.wait() != 0: + raise ValueError("compressed EROFS layer failed zstd verification") + if decompressed_size != raw_size or decompressed_digest.digest() != raw_digest.digest(): + raise ValueError("compressed layer does not match the raw EROFS image") + + +def write_layout(raw_image, compressed_layer, output, architecture, metadata): + architectures = {"x86_64": "amd64", "aarch64": "arm64"} + if architecture not in architectures: + raise ValueError(f"unsupported architecture: {architecture}") + architecture = architectures[architecture] + raw_image = pathlib.Path(raw_image) + compressed_layer = pathlib.Path(compressed_layer) + output = pathlib.Path(output) + blobs = output / "blobs" / "sha256" + + raw_digest = hashlib.sha256() + raw_size = 0 + with raw_image.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + raw_digest.update(chunk) + raw_size += len(chunk) + + compressed_digest = hashlib.sha256() + with compressed_layer.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + compressed_digest.update(chunk) + + annotations = { + "org.opencontainers.image.source": "https://github.com/replit/nixmodules", + "dev.replit.nixmodules.flake-output": "bundle-oci-erofs-zstd", + } + revision = metadata["revision"] + if revision is not None: + if len(revision) != 40 or any(c not in "0123456789abcdef" for c in revision): + raise ValueError("source revision must be a full lowercase Git SHA") + annotations["org.opencontainers.image.revision"] = revision + created = {} + if metadata["sourceTimestamp"] is not None: + timestamp = datetime.datetime.fromtimestamp( + metadata["sourceTimestamp"], datetime.timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%SZ") + annotations["org.opencontainers.image.created"] = timestamp + created["created"] = timestamp + + def descriptor(path, media_type, digest=None): + return { + "mediaType": media_type, + "digest": "sha256:" + (digest or hashlib.sha256(path.read_bytes()).hexdigest()), + "size": path.stat().st_size, + } + + def store_json(value, media_type): + data = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + path = blobs / hashlib.sha256(data).hexdigest() + path.write_bytes(data) + return descriptor(path, media_type) + + layer_descriptor = { + **descriptor( + compressed_layer, + "application/vnd.oci.image.layer.v1.erofs+zstd", + compressed_digest.hexdigest(), + ), + "annotations": {"dev.replit.erofs.uncompressed-size": str(raw_size)}, + } + shutil.move(compressed_layer, blobs / compressed_digest.hexdigest()) + config = store_json( + { + **created, + "architecture": architecture, + "os": "linux", + "config": {}, + "rootfs": {"type": "layers", "diff_ids": ["sha256:" + raw_digest.hexdigest()]}, + }, + "application/vnd.oci.image.config.v1+json", + ) + manifest = store_json( + { + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": config, + "layers": [layer_descriptor], + "annotations": annotations, + }, + "application/vnd.oci.image.manifest.v1+json", + ) + manifest["annotations"] = {"org.opencontainers.image.ref.name": "bundle"} + (output / "index.json").write_text( + json.dumps({"schemaVersion": 2, "manifests": [manifest]}) + "\n" + ) + (output / "oci-layout").write_text('{"imageLayoutVersion":"1.0.0"}\n') + + +if __name__ == "__main__": + write_layout( + pathlib.Path(sys.argv[1]), + pathlib.Path(sys.argv[2]), + pathlib.Path(sys.argv[3]), + sys.argv[4], + json.loads(sys.argv[5]), + ) diff --git a/pkgs/default.nix b/pkgs/default.nix index 95f7adc6..d75f53a9 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -52,6 +52,14 @@ rec { bundle-oci-check = pkgs.callPackage ./bundle-oci/check.nix { }; + bundle-oci-erofs-zstd = pkgs.callPackage ./bundle-oci-erofs-zstd { + inherit bundle; + revision = self.rev or null; + sourceTimestamp = self.lastModified or null; + }; + + bundle-oci-erofs-zstd-check = pkgs.callPackage ./bundle-oci-erofs-zstd/check.nix { }; + custom-bundle = bundle-fn { moduleIds = dev-module-ids; }; diff --git a/scripts/ci_check.sh b/scripts/ci_check.sh index b964c939..cf0f78bb 100755 --- a/scripts/ci_check.sh +++ b/scripts/ci_check.sh @@ -20,3 +20,4 @@ nix develop "${NIX_FLAGS[@]}" --command echo Hello, world nix eval .#bundle nix build .#bundle-oci-check --no-link +nix build .#bundle-oci-erofs-zstd-check --no-link From 28a1adba65bc4973994d670b4c4f2b0794e04f50 Mon Sep 17 00:00:00 2001 From: rahul Date: Thu, 15 Oct 2026 07:23:05 +0000 Subject: [PATCH 2/2] Replace bundle OCI layer with prebuilt EROFS Keep the existing output name and provenance. Fix fixture mode and timestamp checks. Assisted-by: Replit --- README.md | 79 ++--- pkgs/bundle-oci-erofs-zstd/check.nix | 62 ---- pkgs/bundle-oci-erofs-zstd/check.py | 239 -------------- pkgs/bundle-oci-erofs-zstd/default.nix | 44 --- pkgs/bundle-oci-erofs-zstd/layout.py | 132 -------- .../build.sh | 2 +- pkgs/bundle-oci/check.nix | 26 +- pkgs/bundle-oci/check.py | 311 +++++++++++++----- pkgs/bundle-oci/default.nix | 16 +- pkgs/bundle-oci/layer.sh | 8 - pkgs/bundle-oci/layout.py | 80 ++++- pkgs/default.nix | 8 - scripts/ci_check.sh | 1 - 13 files changed, 328 insertions(+), 680 deletions(-) delete mode 100644 pkgs/bundle-oci-erofs-zstd/check.nix delete mode 100644 pkgs/bundle-oci-erofs-zstd/check.py delete mode 100644 pkgs/bundle-oci-erofs-zstd/default.nix delete mode 100644 pkgs/bundle-oci-erofs-zstd/layout.py rename pkgs/{bundle-oci-erofs-zstd => bundle-oci}/build.sh (87%) delete mode 100644 pkgs/bundle-oci/layer.sh diff --git a/README.md b/README.md index 62761530..878c12af 100644 --- a/README.md +++ b/README.md @@ -73,78 +73,41 @@ or nix build .#custom-bundle-squashfs ``` -To build the full bundle as a gzip-compressed tar OCI image: +To build the full bundle as an OCI image: ``` nix build .#bundle-oci ``` -The result is an OCI image layout directory with the reference `bundle`, suitable -for copying with an OCI-layout-aware tool such as -`skopeo copy oci:./result:bundle docker://REGISTRY/REPOSITORY:TAG`. -The output also provides a copy helper that keeps the exact image in its Nix -closure and preserves the manifest digest: +The result is an OCI image layout directory with the reference `bundle`. Copy +it with the helper, which keeps the image in its Nix closure and preserves its +manifest digest: ``` nix run .#bundle-oci.copyTo -- docker://REGISTRY/REPOSITORY:TAG --authfile ./auth.json --digestfile ./digest ``` -The destination can use any Skopeo-supported transport, including `oci:`. -Additional arguments are passed to `skopeo copy`. The helper trusts the locally -built source (`--insecure-policy`); registry TLS verification remains enabled. -For a clean Git source, the manifest records the full flake revision, source URL +The destination can use any Skopeo-supported transport. Additional arguments +are passed to `skopeo copy`. The helper trusts the locally built source +(`--insecure-policy`); registry TLS verification remains enabled. For a clean +Git source, the manifest records the full flake revision, source URL `https://github.com/replit/nixmodules`, and output name `bundle-oci` in OCI -annotations. The manifest creation annotation and config `created` field use -the flake's last-modified timestamp in UTC, never the build's wall clock. -Local sources without a clean revision omit the revision annotation; sources -without a last-modified timestamp omit creation metadata. Publish from a -revision-pinned Git flake when complete provenance is required. - -It contains one `application/vnd.oci.image.layer.v1.tar+gzip` layer. Gzip is -transport compression: the consumer decompresses it when unpacking the image. -The manifest describes the compressed blob's digest and size; the config's -diffID is the digest of the uncompressed tar stream. Tar entry order, -timestamps, ownership, extended headers and gzip headers are deterministic. - -The filesystem is generated directly from the same pinned full bundle used by -`disk-script`: its complete store closure and `/etc/nixmodules` metadata, with -unchanged store paths, file contents, modes and symlink targets. All guest -UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and -per-module OCI builders are unchanged. There is no disk conversion or dependency -update. The tar archive preserves hardlinks and extended attributes. A consumer -that converts the tar to EROFS must leave filesystem compression disabled for -files to remain DAX-eligible. This uses more unpacked disk space than compressed -EROFS; it does not automatically share all guest memory. +annotations. Creation metadata uses the flake's last-modified timestamp in UTC. +Local sources without revision or timestamp metadata omit those annotations. -A small fixture exercises the producer without building the full bundle: - -``` -nix build .#bundle-oci-check -``` - -## Optional EROFS OCI image +The image contains one `application/vnd.oci.image.layer.v1.erofs+zstd` layer. +The layer descriptor's digest and size cover the zstd-compressed data; its +`dev.replit.erofs.uncompressed-size` annotation records the raw EROFS image +size. The config diffID is the SHA-256 digest of that raw image. The EROFS +filesystem has internal compression and inline data disabled, with the complete +bundle closure and `/etc/nixmodules` metadata. -An opt-in image is available as `bundle-oci-erofs-zstd`; the existing -`bundle-oci` output remains the default and is unchanged. Build and copy it with: +This changes the existing `bundle-oci` format and is not backward-compatible +with consumers expecting a tar+gzip layer. Consumers must support this OCI +layer media type and EROFS; deploy that support before publishing images. -``` -nix build .#bundle-oci-erofs-zstd -nix run .#bundle-oci-erofs-zstd.copyTo -- docker://REGISTRY/REPOSITORY:TAG -``` - -This image has one `application/vnd.oci.image.layer.v1.erofs+zstd` layer. -Consumers must explicitly support this OCI layer media type and the EROFS -filesystem before using this output; generic OCI consumers may reject it. The -descriptor digest and size cover the zstd-compressed layer, while the config -diffID covers the raw EROFS image. The layer descriptor's -`dev.replit.erofs.uncompressed-size` annotation reports the raw image size in -bytes. -The filesystem is built without EROFS internal compression and with inline -data disabled. - -The small fixture checks the image format and copy helper without building the -full bundle: +A small fixture exercises the producer without building the full bundle: ``` -nix build .#bundle-oci-erofs-zstd-check +nix build .#bundle-oci-check ``` diff --git a/pkgs/bundle-oci-erofs-zstd/check.nix b/pkgs/bundle-oci-erofs-zstd/check.nix deleted file mode 100644 index 0ff90e0b..00000000 --- a/pkgs/bundle-oci-erofs-zstd/check.nix +++ /dev/null @@ -1,62 +0,0 @@ -{ pkgs }: - -let - dependency = pkgs.runCommand "bundle-oci-erofs-zstd-dependency" { } '' - mkdir -p "$out" - printf 'fixture dependency\n' > "$out/data" - ln "$out/data" "$out/hardlink" - ''; - module = pkgs.runCommand "bundle-oci-erofs-zstd-module" { } '' - mkdir -p "$out/bin" - printf '%s\n' '${dependency}' > "$out/module.json" - printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool" - chmod 751 "$out/bin/tool" - ln "$out/module.json" "$out/module-hardlink" - ln -s ../module.json "$out/bin/relative" - ln -s '${dependency}/data' "$out/absolute" - ''; - bundle = (pkgs.callPackage ../bundle { - self = { modules.fixture = module; }; - }) { }; - image = pkgs.callPackage ./. { - inherit bundle; - revision = "0123456789abcdef0123456789abcdef01234567"; - sourceTimestamp = 1700000000; - }; - closure = pkgs.closureInfo { rootPaths = [ bundle ]; }; -in -pkgs.runCommand "bundle-oci-erofs-zstd-check" -{ - nativeBuildInputs = [ - pkgs.erofs-utils - pkgs.python3 - pkgs.skopeo - pkgs.zstd - ]; -} - '' - python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${module} ${./layout.py} - ${image.copyTo}/bin/copy-nixmodules-bundle-oci-erofs-zstd "oci:$PWD/copied:bundle" \ - --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \ - --digestfile "$PWD/copied.digest" - python3 - ${image} "$PWD/copied" "$PWD/copied.digest" <<'PY' - import json, pathlib, sys - original, copied, digest_file = map(pathlib.Path, sys.argv[1:]) - descriptor = json.loads((original / "index.json").read_text())["manifests"][0] - copied_descriptor = json.loads((copied / "index.json").read_text())["manifests"][0] - assert copied_descriptor["digest"] == descriptor["digest"] - assert digest_file.read_text().strip() == descriptor["digest"] - for blob in (original / "blobs/sha256").iterdir(): - assert blob.read_bytes() == (copied / "blobs/sha256" / blob.name).read_bytes() - copied_manifest = json.loads( - (copied / "blobs/sha256" / copied_descriptor["digest"].split(":")[1]).read_text() - ) - assert copied_manifest["layers"][0]["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" - original_manifest = json.loads( - (original / "blobs/sha256" / descriptor["digest"].split(":")[1]).read_text() - ) - assert copied_manifest["layers"][0]["annotations"] == original_manifest["layers"][0]["annotations"] - print("copyTo preserved the custom layer media type, manifest digest and every OCI blob") - PY - touch "$out" - '' diff --git a/pkgs/bundle-oci-erofs-zstd/check.py b/pkgs/bundle-oci-erofs-zstd/check.py deleted file mode 100644 index 8ff7451b..00000000 --- a/pkgs/bundle-oci-erofs-zstd/check.py +++ /dev/null @@ -1,239 +0,0 @@ -import hashlib -import importlib.util -import json -import os -import pathlib -import re -import stat -import subprocess -import sys - - -image, bundle, closure, dependency, module, layout_script = map(pathlib.Path, sys.argv[1:]) -assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"} - - -def blob(descriptor): - path = image / "blobs" / "sha256" / descriptor["digest"].removeprefix("sha256:") - assert path.stat().st_size == descriptor["size"] - assert "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest() == descriptor["digest"] - return path - - -index = json.loads((image / "index.json").read_text()) -assert index["schemaVersion"] == 2 and len(index["manifests"]) == 1 -manifest = json.loads(blob(index["manifests"][0]).read_text()) -assert manifest["schemaVersion"] == 2 -assert manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json" -assert manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json" -config = json.loads(blob(manifest["config"]).read_text()) -assert config["architecture"] == "amd64" and config["os"] == "linux" -assert config["created"] == "2023-11-14T22:13:20Z" -annotations = manifest["annotations"] -assert annotations == { - "org.opencontainers.image.source": "https://github.com/replit/nixmodules", - "org.opencontainers.image.revision": "0123456789abcdef0123456789abcdef01234567", - "org.opencontainers.image.created": config["created"], - "dev.replit.nixmodules.flake-output": "bundle-oci-erofs-zstd", -} -assert len(manifest["layers"]) == 1 -layer = manifest["layers"][0] -assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" -assert layer["annotations"]["dev.replit.erofs.uncompressed-size"].isdecimal() -assert set(layer["annotations"]) == {"dev.replit.erofs.uncompressed-size"} -compressed = blob(layer) -assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str( - int(layer["annotations"]["dev.replit.erofs.uncompressed-size"]) -) -raw = pathlib.Path("raw.erofs") -subprocess.run(["zstd", "-d", "-f", str(compressed), "-o", str(raw)], check=True) -raw_bytes = raw.read_bytes() -raw_digest = "sha256:" + hashlib.sha256(raw_bytes).hexdigest() -assert config["rootfs"] == {"type": "layers", "diff_ids": [raw_digest]} -assert layer["digest"] != raw_digest -assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str(len(raw_bytes)) -dump_stats = subprocess.run( - ["dump.erofs", "-S", str(raw)], capture_output=True, text=True, check=True -).stdout -assert re.search(r"Filesystem compressed files:\s+0\b", dump_stats) -dump_superblock = subprocess.run( - ["dump.erofs", "-s", str(raw)], capture_output=True, text=True, check=True -).stdout -assert ( - "Filesystem UUID: 00000000-0000-0000-0000-000000000001" - in dump_superblock -) -features = next(line for line in dump_superblock.splitlines() if "Filesystem features:" in line) -assert "compr_cfgs" not in features -tool_inode = subprocess.run( - ["dump.erofs", f"--path={module}/bin/tool", str(raw)], - capture_output=True, - text=True, - check=True, -).stdout -assert "Layout: 0" in tool_inode -assert "Uid: 11000 Gid: 11000" in tool_inode -subprocess.run( - [ - "fsck.erofs", - "--preserve-perms", - "--xattrs", - *(["--preserve-owner"] if os.geteuid() == 0 else []), - f"--extract={pathlib.Path('extracted')}", - str(raw), - ], - check=True, -) -root = pathlib.Path("extracted") -paths = closure.read_text().splitlines() -assert str(dependency) in paths -assert {p.name for p in (root / "nix/store").iterdir()} == { - pathlib.Path(p).name for p in paths -} - - -def compare(source, target): - before, after = source.lstat(), target.lstat() - assert stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode), target - assert stat.S_IMODE(before.st_mode) == stat.S_IMODE(after.st_mode), target - if source.is_symlink(): - assert os.readlink(source) == os.readlink(target), target - elif source.is_dir(): - assert {p.name for p in source.iterdir()} == {p.name for p in target.iterdir()} - for child in source.iterdir(): - compare(child, target / child.name) - else: - assert source.read_bytes() == target.read_bytes(), target - - -for path in paths: - compare(pathlib.Path(path), root / path.lstrip("/")) -for metadata in (bundle / "etc/nixmodules").iterdir(): - compare(metadata, root / "etc/nixmodules" / metadata.name) -packed_dependency = root / str(dependency).lstrip("/") -assert (packed_dependency / "data").stat().st_ino == ( - packed_dependency / "hardlink" -).stat().st_ino -fixture_module = root / str(module).lstrip("/") -assert (fixture_module / "module.json").stat().st_ino == ( - fixture_module / "module-hardlink" -).stat().st_ino -assert stat.S_IMODE((fixture_module / "bin/tool").stat().st_mode) == 0o751 -assert os.readlink(fixture_module / "bin/relative") == "../module.json" -assert os.readlink(fixture_module / "absolute") == f"{dependency}/data" -if os.geteuid() == 0: - assert (fixture_module / "bin/tool").stat().st_uid == 11000 - assert (fixture_module / "bin/tool").stat().st_gid == 11000 -print("EROFS payload, closure, plain layout, modes, symlinks, hardlinks and ownership verified") - -layout_spec = importlib.util.spec_from_file_location("layout", layout_script) -layout = importlib.util.module_from_spec(layout_spec) -layout_spec.loader.exec_module(layout) -layout.verify_layer(raw, compressed) -metadata = { - "revision": "0123456789abcdef0123456789abcdef01234567", - "sourceTimestamp": 1700000000, -} - - -def expect_error(raw_path, compressed_path): - try: - layout.verify_layer(raw_path, compressed_path) - except (OSError, ValueError, subprocess.SubprocessError): - return - raise AssertionError("invalid layout input was accepted") - - -truncated = pathlib.Path("truncated.zst") -truncated.write_bytes(compressed.read_bytes()[:-4]) -expect_error(raw, truncated) -corrupt = pathlib.Path("corrupt.zst") -corrupt.write_bytes(compressed.read_bytes()[:-1] + bytes([compressed.read_bytes()[-1] ^ 0xFF])) -expect_error(raw, corrupt) -mismatch = pathlib.Path("mismatch.erofs") -mismatch.write_bytes(raw_bytes + b"mismatch") -expect_error(mismatch, compressed) -expect_error(pathlib.Path("missing.erofs"), compressed) -for architecture, invalid_metadata in ( - ("unsupported", metadata), - ("x86_64", {"revision": "bad", "sourceTimestamp": None}), -): - invalid_output = pathlib.Path("invalid-layout") - (invalid_output / "blobs/sha256").mkdir(parents=True, exist_ok=True) - try: - layout.write_layout(raw, compressed, invalid_output, architecture, invalid_metadata) - except ValueError: - continue - raise AssertionError("invalid layout metadata was accepted") - -repeat = pathlib.Path("repeat.zst") -subprocess.run( - [ - "zstd", - "-15", - "--long=27", - "-T8", - "--check", - "--no-progress", - "--quiet", - str(raw), - "-o", - str(repeat), - ], - check=True, -) -assert repeat.read_bytes() == compressed.read_bytes() -repeat_layout = pathlib.Path("repeat-layout") -(repeat_layout / "blobs/sha256").mkdir(parents=True) -layout.write_layout(raw, repeat, repeat_layout, "x86_64", metadata) -for path in image.rglob("*"): - if path.is_file(): - assert path.read_bytes() == (repeat_layout / path.relative_to(image)).read_bytes() - -xattr_root = pathlib.Path("xattr-root") -xattr_root.mkdir() -xattr_file = xattr_root / "xattr-data" -xattr_file.write_bytes(b"extended attribute fixture\n") -os.setxattr(xattr_file, "user.fixture", b"preserved") -xattr_raw = pathlib.Path("xattr.erofs") -mkfs_args = [ - "mkfs.erofs", - "-U", "00000000-0000-0000-0000-000000000001", - "-T", "1", - "--force-uid=11000", - "--force-gid=11000", - "-E", "noinline_data", - str(xattr_raw), - str(xattr_root), -] -subprocess.run(mkfs_args, check=True) -xattr_dump = subprocess.run( - ["dump.erofs", f"--path=/{xattr_file.name}", str(xattr_raw)], - capture_output=True, - text=True, - check=True, -).stdout -assert "Layout: 0" in xattr_dump -assert "Uid: 11000 Gid: 11000" in xattr_dump -os.utime(xattr_file, (12345, 12345)) -os.utime(xattr_root, (23456, 23456)) -xattr_repeat = pathlib.Path("xattr-repeat.erofs") -subprocess.run(mkfs_args[:-2] + [str(xattr_repeat), str(xattr_root)], check=True) -assert xattr_raw.read_bytes() == xattr_repeat.read_bytes() -xattr_extracted = pathlib.Path("xattr-extracted") -subprocess.run( - [ - "fsck.erofs", - "--preserve-perms", - "--xattrs", - *(["--preserve-owner"] if os.geteuid() == 0 else []), - f"--extract={xattr_extracted}", - str(xattr_raw), - ], - check=True, -) -assert os.getxattr(xattr_extracted / "xattr-data", "user.fixture") == b"preserved" -if os.geteuid() == 0: - assert (xattr_extracted / "xattr-data").stat().st_uid == 11000 - assert (xattr_extracted / "xattr-data").stat().st_gid == 11000 -print("Deterministic compression, xattrs, forced ownership and invalid-input checks verified") diff --git a/pkgs/bundle-oci-erofs-zstd/default.nix b/pkgs/bundle-oci-erofs-zstd/default.nix deleted file mode 100644 index 05552114..00000000 --- a/pkgs/bundle-oci-erofs-zstd/default.nix +++ /dev/null @@ -1,44 +0,0 @@ -{ runCommand -, lib -, stdenv -, bundle -, closureInfo -, erofs-utils -, zstd -, python3 -, writeShellApplication -, skopeo -, revision ? null -, sourceTimestamp ? null -}: - -let - bundleClosure = closureInfo { rootPaths = [ bundle ]; }; - metadata = builtins.toJSON { inherit revision sourceTimestamp; }; - image = runCommand "nixmodules-bundle-oci-erofs-zstd" - { - nativeBuildInputs = [ erofs-utils zstd python3 ]; - inherit bundle bundleClosure; - architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; - __structuredAttrs = true; - unsafeDiscardReferences.out = true; - passthru.copyTo = writeShellApplication { - name = "copy-nixmodules-bundle-oci-erofs-zstd"; - runtimeInputs = [ skopeo ]; - text = '' - if [ "$#" -lt 1 ]; then - echo "usage: copy-nixmodules-bundle-oci-erofs-zstd [skopeo copy options]" >&2 - exit 1 - fi - destination="$1" - shift - exec skopeo --insecure-policy copy --preserve-digests \ - "oci:${image}:bundle" "$destination" "$@" - ''; - }; - } - '' - bash ${./build.sh} "$bundle" "$bundleClosure/store-paths" "$out" "$architecture" ${lib.escapeShellArg metadata} ${./layout.py} - ''; -in -image diff --git a/pkgs/bundle-oci-erofs-zstd/layout.py b/pkgs/bundle-oci-erofs-zstd/layout.py deleted file mode 100644 index 66bbf4f4..00000000 --- a/pkgs/bundle-oci-erofs-zstd/layout.py +++ /dev/null @@ -1,132 +0,0 @@ -import datetime -import hashlib -import json -import pathlib -import shutil -import subprocess -import sys - - -def verify_layer(raw_image, compressed_layer): - raw_image = pathlib.Path(raw_image) - compressed_layer = pathlib.Path(compressed_layer) - raw_digest = hashlib.sha256() - raw_size = 0 - with raw_image.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - raw_digest.update(chunk) - raw_size += len(chunk) - - process = subprocess.Popen( - ["zstd", "-d", "-c", "--", str(compressed_layer)], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ) - decompressed_digest = hashlib.sha256() - decompressed_size = 0 - assert process.stdout is not None - for chunk in iter(lambda: process.stdout.read(1024 * 1024), b""): - decompressed_digest.update(chunk) - decompressed_size += len(chunk) - if process.wait() != 0: - raise ValueError("compressed EROFS layer failed zstd verification") - if decompressed_size != raw_size or decompressed_digest.digest() != raw_digest.digest(): - raise ValueError("compressed layer does not match the raw EROFS image") - - -def write_layout(raw_image, compressed_layer, output, architecture, metadata): - architectures = {"x86_64": "amd64", "aarch64": "arm64"} - if architecture not in architectures: - raise ValueError(f"unsupported architecture: {architecture}") - architecture = architectures[architecture] - raw_image = pathlib.Path(raw_image) - compressed_layer = pathlib.Path(compressed_layer) - output = pathlib.Path(output) - blobs = output / "blobs" / "sha256" - - raw_digest = hashlib.sha256() - raw_size = 0 - with raw_image.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - raw_digest.update(chunk) - raw_size += len(chunk) - - compressed_digest = hashlib.sha256() - with compressed_layer.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - compressed_digest.update(chunk) - - annotations = { - "org.opencontainers.image.source": "https://github.com/replit/nixmodules", - "dev.replit.nixmodules.flake-output": "bundle-oci-erofs-zstd", - } - revision = metadata["revision"] - if revision is not None: - if len(revision) != 40 or any(c not in "0123456789abcdef" for c in revision): - raise ValueError("source revision must be a full lowercase Git SHA") - annotations["org.opencontainers.image.revision"] = revision - created = {} - if metadata["sourceTimestamp"] is not None: - timestamp = datetime.datetime.fromtimestamp( - metadata["sourceTimestamp"], datetime.timezone.utc - ).strftime("%Y-%m-%dT%H:%M:%SZ") - annotations["org.opencontainers.image.created"] = timestamp - created["created"] = timestamp - - def descriptor(path, media_type, digest=None): - return { - "mediaType": media_type, - "digest": "sha256:" + (digest or hashlib.sha256(path.read_bytes()).hexdigest()), - "size": path.stat().st_size, - } - - def store_json(value, media_type): - data = json.dumps(value, sort_keys=True, separators=(",", ":")).encode() - path = blobs / hashlib.sha256(data).hexdigest() - path.write_bytes(data) - return descriptor(path, media_type) - - layer_descriptor = { - **descriptor( - compressed_layer, - "application/vnd.oci.image.layer.v1.erofs+zstd", - compressed_digest.hexdigest(), - ), - "annotations": {"dev.replit.erofs.uncompressed-size": str(raw_size)}, - } - shutil.move(compressed_layer, blobs / compressed_digest.hexdigest()) - config = store_json( - { - **created, - "architecture": architecture, - "os": "linux", - "config": {}, - "rootfs": {"type": "layers", "diff_ids": ["sha256:" + raw_digest.hexdigest()]}, - }, - "application/vnd.oci.image.config.v1+json", - ) - manifest = store_json( - { - "schemaVersion": 2, - "mediaType": "application/vnd.oci.image.manifest.v1+json", - "config": config, - "layers": [layer_descriptor], - "annotations": annotations, - }, - "application/vnd.oci.image.manifest.v1+json", - ) - manifest["annotations"] = {"org.opencontainers.image.ref.name": "bundle"} - (output / "index.json").write_text( - json.dumps({"schemaVersion": 2, "manifests": [manifest]}) + "\n" - ) - (output / "oci-layout").write_text('{"imageLayoutVersion":"1.0.0"}\n') - - -if __name__ == "__main__": - write_layout( - pathlib.Path(sys.argv[1]), - pathlib.Path(sys.argv[2]), - pathlib.Path(sys.argv[3]), - sys.argv[4], - json.loads(sys.argv[5]), - ) diff --git a/pkgs/bundle-oci-erofs-zstd/build.sh b/pkgs/bundle-oci/build.sh similarity index 87% rename from pkgs/bundle-oci-erofs-zstd/build.sh rename to pkgs/bundle-oci/build.sh index 013a40ff..984b2eac 100644 --- a/pkgs/bundle-oci-erofs-zstd/build.sh +++ b/pkgs/bundle-oci/build.sh @@ -15,7 +15,7 @@ while IFS= read -r path; do done < "$store_paths" chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules -mkfs.erofs -U 00000000-0000-0000-0000-000000000001 -T 1 \ +env -u SOURCE_DATE_EPOCH mkfs.erofs -U 00000000-0000-0000-0000-000000000001 -T 1 --all-time \ --force-uid=11000 --force-gid=11000 -E noinline_data raw.erofs root zstd -15 --long=27 -T8 --check --no-progress --quiet raw.erofs -o layer.erofs.zst python3 "$layout_script" raw.erofs layer.erofs.zst "$output" "$architecture" "$metadata" diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix index f9720788..2bcaa674 100644 --- a/pkgs/bundle-oci/check.nix +++ b/pkgs/bundle-oci/check.nix @@ -3,16 +3,15 @@ let dependency = pkgs.runCommand "bundle-oci-dependency" { } '' mkdir -p "$out" - printf 'dependency\n' > "$out/data" - head -c 131072 /dev/zero >> "$out/data" + printf 'fixture dependency\n' > "$out/data" ln "$out/data" "$out/hardlink" ''; module = pkgs.runCommand "bundle-oci-module" { } '' mkdir -p "$out/bin" printf '%s\n' '${dependency}' > "$out/module.json" printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool" - head -c 131072 /dev/zero >> "$out/bin/tool" - chmod 755 "$out/bin/tool" + chmod 751 "$out/bin/tool" + ln "$out/module.json" "$out/module-hardlink" ln -s ../module.json "$out/bin/relative" ln -s '${dependency}/data' "$out/absolute" ''; @@ -28,10 +27,15 @@ let in pkgs.runCommand "bundle-oci-check" { - nativeBuildInputs = [ pkgs.gnutar pkgs.gzip pkgs.python3 ]; + nativeBuildInputs = [ + pkgs.erofs-utils + pkgs.python3 + pkgs.skopeo + pkgs.zstd + ]; } '' - python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh} + python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${module} ${./layout.py} ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \ --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \ --digestfile "$PWD/copied.digest" @@ -44,7 +48,15 @@ pkgs.runCommand "bundle-oci-check" assert digest_file.read_text().strip() == descriptor["digest"] for blob in (original / "blobs/sha256").iterdir(): assert blob.read_bytes() == (copied / "blobs/sha256" / blob.name).read_bytes() - print("copyTo preserved manifest digest and every OCI blob") + copied_manifest = json.loads( + (copied / "blobs/sha256" / copied_descriptor["digest"].split(":")[1]).read_text() + ) + assert copied_manifest["layers"][0]["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" + original_manifest = json.loads( + (original / "blobs/sha256" / descriptor["digest"].split(":")[1]).read_text() + ) + assert copied_manifest["layers"][0]["annotations"] == original_manifest["layers"][0]["annotations"] + print("copyTo preserved the custom layer media type, manifest digest and every OCI blob") PY touch "$out" '' diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index f101a060..261e9445 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -1,17 +1,17 @@ import hashlib -import gzip -import errno import importlib.util +import io import json import os import pathlib +import re import stat -import shutil import subprocess import sys import tarfile -image, bundle, closure, dependency, layout_script, layer_script = map(pathlib.Path, sys.argv[1:]) + +image, bundle, closure, dependency, module, layout_script = map(pathlib.Path, sys.argv[1:]) assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"} @@ -31,7 +31,8 @@ def blob(descriptor): config = json.loads(blob(manifest["config"]).read_text()) assert config["architecture"] == "amd64" and config["os"] == "linux" assert config["created"] == "2023-11-14T22:13:20Z" -assert manifest["annotations"] == { +annotations = manifest["annotations"] +assert annotations == { "org.opencontainers.image.source": "https://github.com/replit/nixmodules", "org.opencontainers.image.revision": "0123456789abcdef0123456789abcdef01234567", "org.opencontainers.image.created": config["created"], @@ -39,61 +40,52 @@ def blob(descriptor): } assert len(manifest["layers"]) == 1 layer = manifest["layers"][0] -assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip" -archive = blob(layer) -tar_data = gzip.decompress(archive.read_bytes()) -diff_id = "sha256:" + hashlib.sha256(tar_data).hexdigest() -assert config["rootfs"] == {"type": "layers", "diff_ids": [diff_id]} -assert diff_id != layer["digest"] -assert archive.read_bytes()[3:8] == b"\x00" * 5 -spec = importlib.util.spec_from_file_location("layout", layout_script) -layout = importlib.util.module_from_spec(spec) -spec.loader.exec_module(layout) -repeated = pathlib.Path("repeated") -(repeated / "blobs/sha256").mkdir(parents=True) -shutil.copyfile(archive, "repeated.tar.gz") -layout.write_layout(pathlib.Path("repeated.tar.gz"), repeated, "x86_64", { - "revision": "0123456789abcdef0123456789abcdef01234567", - "sourceTimestamp": 1700000000, -}) -for path in image.rglob("*"): - if path.is_file(): - assert path.read_bytes() == (repeated / path.relative_to(image)).read_bytes() -local = pathlib.Path("local") -(local / "blobs/sha256").mkdir(parents=True) -shutil.copyfile(archive, "local.tar.gz") -layout.write_layout(pathlib.Path("local.tar.gz"), local, "x86_64", { - "revision": None, "sourceTimestamp": None, -}) -local_descriptor = json.loads((local / "index.json").read_text())["manifests"][0] -local_manifest = json.loads( - (local / "blobs/sha256" / local_descriptor["digest"].split(":")[1]).read_text() +assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs+zstd" +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"].isdecimal() +assert set(layer["annotations"]) == {"dev.replit.erofs.uncompressed-size"} +compressed = blob(layer) +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str( + int(layer["annotations"]["dev.replit.erofs.uncompressed-size"]) +) +raw = pathlib.Path("raw.erofs") +subprocess.run(["zstd", "-d", "-f", str(compressed), "-o", str(raw)], check=True) +raw_bytes = raw.read_bytes() +raw_digest = "sha256:" + hashlib.sha256(raw_bytes).hexdigest() +assert config["rootfs"] == {"type": "layers", "diff_ids": [raw_digest]} +assert layer["digest"] != raw_digest +assert layer["annotations"]["dev.replit.erofs.uncompressed-size"] == str(len(raw_bytes)) +dump_stats = subprocess.run( + ["dump.erofs", "-S", str(raw)], capture_output=True, text=True, check=True +).stdout +assert re.search(r"Filesystem compressed files:\s+0\b", dump_stats) +dump_superblock = subprocess.run( + ["dump.erofs", "-s", str(raw)], capture_output=True, text=True, check=True +).stdout +assert ( + "Filesystem UUID: 00000000-0000-0000-0000-000000000001" + in dump_superblock ) -assert "org.opencontainers.image.revision" not in local_manifest["annotations"] -assert "org.opencontainers.image.created" not in local_manifest["annotations"] -local_config = json.loads( - (local / "blobs/sha256" / local_manifest["config"]["digest"].split(":")[1]).read_text() +features = next(line for line in dump_superblock.splitlines() if "Filesystem features:" in line) +assert "compr_cfgs" not in features +tool_inode = subprocess.run( + ["dump.erofs", f"--path={module}/bin/tool", str(raw)], + capture_output=True, + text=True, + check=True, +).stdout +assert "Layout: 0" in tool_inode +assert "Uid: 11000 Gid: 11000" in tool_inode +subprocess.run( + [ + "fsck.erofs", + "--preserve-perms", + *(["--preserve-owner"] if os.geteuid() == 0 else []), + f"--extract={pathlib.Path('extracted')}", + str(raw), + ], + check=True, ) -assert "created" not in local_config root = pathlib.Path("extracted") -root.mkdir() -subprocess.run([ - "tar", "--extract", "--gzip", "--file", str(archive), - "--directory", str(root), "--no-same-owner", "--same-permissions", - "--xattrs", "--xattrs-include=*", -], check=True) -with tarfile.open(archive, "r:gz") as source: - members = source.getmembers() - assert all(m.uid == 11000 and m.gid == 11000 and m.mtime == 1 for m in members) - assert all(not m.uname and not m.gname for m in members) - assert all("atime" not in m.pax_headers and "ctime" not in m.pax_headers for m in members) - assert [m.name for m in members] == sorted(m.name for m in members) - -rebuilt = pathlib.Path("rebuilt.tar.gz") -subprocess.run(["bash", str(layer_script), str(root), str(rebuilt)], check=True) -assert rebuilt.read_bytes() == archive.read_bytes() -print("Tar metadata and reproducible gzip layer verified") - paths = closure.read_text().splitlines() assert str(dependency) in paths assert {p.name for p in (root / "nix/store").iterdir()} == { @@ -123,37 +115,174 @@ def compare(source, target): assert (packed_dependency / "data").stat().st_ino == ( packed_dependency / "hardlink" ).stat().st_ino -print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified") - -xattr_root = pathlib.Path("xattrs") -xattr_root.mkdir() -xattr_file = xattr_root / ("long-name-" * 20) -xattr_file.write_bytes(b"xattrs\n") -try: - os.setxattr(xattr_file, "user.fixture", b"preserved") - xattrs_supported = True -except OSError as error: - if error.errno != errno.ENOTSUP: - raise - xattrs_supported = False - print("Skipping xattr round-trip: build filesystem does not support xattrs") -subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs.tar.gz"], check=True) -with tarfile.open("xattrs.tar.gz", "r:gz") as source: - member = source.getmember("./" + xattr_file.name) - assert member.isfile() and source.extractfile(member).read() == b"xattrs\n" - if xattrs_supported: - assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved" -if xattrs_supported: - restored = pathlib.Path("xattrs-restored") - restored.mkdir() - subprocess.run([ - "tar", "--extract", "--gzip", "--file=xattrs.tar.gz", - "--directory", str(restored), "--no-same-owner", - "--xattrs", "--xattrs-include=*", - ], check=True) - assert os.getxattr(restored / xattr_file.name, "user.fixture") == b"preserved" - print("Extended attributes preserved in archive and extraction") -os.utime(xattr_file, (12345, 67890)) -subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs-again.tar.gz"], check=True) -assert pathlib.Path("xattrs.tar.gz").read_bytes() == pathlib.Path("xattrs-again.tar.gz").read_bytes() -print("Long paths preserved; source timestamps do not change the layer") +fixture_module = root / str(module).lstrip("/") +assert (fixture_module / "module.json").stat().st_ino == ( + fixture_module / "module-hardlink" +).stat().st_ino +tool_mode = stat.S_IMODE((module / "bin/tool").stat().st_mode) +assert tool_mode & 0o111 +assert stat.S_IMODE((fixture_module / "bin/tool").stat().st_mode) == tool_mode +assert os.readlink(fixture_module / "bin/relative") == "../module.json" +assert os.readlink(fixture_module / "absolute") == f"{dependency}/data" +if os.geteuid() == 0: + assert (fixture_module / "bin/tool").stat().st_uid == 11000 + assert (fixture_module / "bin/tool").stat().st_gid == 11000 +print("EROFS payload, closure, plain layout, modes, symlinks, hardlinks and ownership verified") + +layout_spec = importlib.util.spec_from_file_location("layout", layout_script) +layout = importlib.util.module_from_spec(layout_spec) +layout_spec.loader.exec_module(layout) +layout.verify_layer(raw, compressed) +metadata = { + "revision": "0123456789abcdef0123456789abcdef01234567", + "sourceTimestamp": 1700000000, +} + + +def expect_error(raw_path, compressed_path): + try: + layout.verify_layer(raw_path, compressed_path) + except (OSError, ValueError, subprocess.SubprocessError): + return + raise AssertionError("invalid layout input was accepted") + + +truncated = pathlib.Path("truncated.zst") +truncated.write_bytes(compressed.read_bytes()[:-4]) +expect_error(raw, truncated) +corrupt = pathlib.Path("corrupt.zst") +corrupt.write_bytes(compressed.read_bytes()[:-1] + bytes([compressed.read_bytes()[-1] ^ 0xFF])) +expect_error(raw, corrupt) +mismatch = pathlib.Path("mismatch.erofs") +mismatch.write_bytes(raw_bytes + b"mismatch") +expect_error(mismatch, compressed) +expect_error(pathlib.Path("missing.erofs"), compressed) +for architecture, invalid_metadata in ( + ("unsupported", metadata), + ("x86_64", {"revision": "bad", "sourceTimestamp": None}), +): + invalid_output = pathlib.Path("invalid-layout") + (invalid_output / "blobs/sha256").mkdir(parents=True, exist_ok=True) + try: + layout.write_layout(raw, compressed, invalid_output, architecture, invalid_metadata) + except ValueError: + continue + raise AssertionError("invalid layout metadata was accepted") + +repeat = pathlib.Path("repeat.zst") +subprocess.run( + [ + "zstd", + "-15", + "--long=27", + "-T8", + "--check", + "--no-progress", + "--quiet", + str(raw), + "-o", + str(repeat), + ], + check=True, +) +assert repeat.read_bytes() == compressed.read_bytes() +repeat_layout = pathlib.Path("repeat-layout") +(repeat_layout / "blobs/sha256").mkdir(parents=True) +layout.write_layout(raw, repeat, repeat_layout, "x86_64", metadata) +for path in image.rglob("*"): + if path.is_file(): + assert path.read_bytes() == (repeat_layout / path.relative_to(image)).read_bytes() + +timestamp_root = pathlib.Path("timestamp-root") +timestamp_root.mkdir() +timestamp_file = timestamp_root / "data" +timestamp_file.write_bytes(b"deterministic EROFS fixture\n") +os.chmod(timestamp_file, 0o751) +os.link(timestamp_file, timestamp_root / "hardlink") +(timestamp_root / "relative").symlink_to("data") +timestamp_mkfs_args = [ + "env", + "-u", + "SOURCE_DATE_EPOCH", + "mkfs.erofs", + "-U", "00000000-0000-0000-0000-000000000001", + "-T", "1", + "--all-time", + "--force-uid=11000", + "--force-gid=11000", + "-E", "noinline_data", +] +timestamp_raw = pathlib.Path("timestamp.erofs") +subprocess.run( + timestamp_mkfs_args + [str(timestamp_raw), str(timestamp_root)], + check=True, +) +os.utime(timestamp_file, (23456, 23456)) +os.utime(timestamp_root, (34567, 34567)) +os.utime(timestamp_root / "relative", (45678, 45678), follow_symlinks=False) +timestamp_repeat = pathlib.Path("timestamp-repeat.erofs") +subprocess.run( + timestamp_mkfs_args + [str(timestamp_repeat), str(timestamp_root)], + check=True, +) +assert timestamp_raw.read_bytes() == timestamp_repeat.read_bytes() + +xattr_tar = pathlib.Path("xattr.tar") +xattr_value = b"nixmodules-fixture-xattr-value" +xattr_content = b"extended attribute fixture\n" + + +def write_xattr_tar(): + with tarfile.open(xattr_tar, "w", format=tarfile.PAX_FORMAT) as archive: + entry = tarfile.TarInfo("xattr-data") + entry.size = len(xattr_content) + entry.mode = 0o751 + entry.mtime = 1 + entry.pax_headers = {"SCHILY.xattr.user.fixture": xattr_value.decode()} + archive.addfile(entry, io.BytesIO(xattr_content)) + + +write_xattr_tar() +xattr_raw = pathlib.Path("xattr.erofs") +mkfs_args = [ + "env", + "-u", + "SOURCE_DATE_EPOCH", + "mkfs.erofs", + "-U", "00000000-0000-0000-0000-000000000001", + "-T", "1", + "--all-time", + "--force-uid=11000", + "--force-gid=11000", + "-E", "noinline_data", + "--tar=f", + str(xattr_raw), + str(xattr_tar), +] +subprocess.run(mkfs_args, check=True) +xattr_dump = subprocess.run( + ["dump.erofs", "--path=/xattr-data", str(xattr_raw)], + capture_output=True, + text=True, + check=True, +).stdout +assert "Layout: 0" in xattr_dump +assert "Uid: 11000 Gid: 11000" in xattr_dump +assert re.search(r"Xattr size: [1-9]\d*", xattr_dump) +assert xattr_value in xattr_raw.read_bytes() +xattr_extracted = pathlib.Path("xattr-extracted") +subprocess.run( + [ + "fsck.erofs", + "--preserve-perms", + *(["--preserve-owner"] if os.geteuid() == 0 else []), + f"--extract={xattr_extracted}", + str(xattr_raw), + ], + check=True, +) +assert (xattr_extracted / "xattr-data").read_bytes() == xattr_content +if os.geteuid() == 0: + assert (xattr_extracted / "xattr-data").stat().st_uid == 11000 + assert (xattr_extracted / "xattr-data").stat().st_gid == 11000 +print("Deterministic raw and compressed outputs, xattrs, forced ownership and invalid-input checks verified") diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix index ae357cff..51066c1f 100644 --- a/pkgs/bundle-oci/default.nix +++ b/pkgs/bundle-oci/default.nix @@ -3,8 +3,8 @@ , stdenv , bundle , closureInfo -, gnutar -, gzip +, erofs-utils +, zstd , python3 , writeShellApplication , skopeo @@ -17,7 +17,7 @@ let metadata = builtins.toJSON { inherit revision sourceTimestamp; }; image = runCommand "nixmodules-bundle-oci" { - nativeBuildInputs = [ gnutar gzip python3 ]; + nativeBuildInputs = [ erofs-utils zstd python3 ]; inherit bundle bundleClosure; architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; __structuredAttrs = true; @@ -39,15 +39,7 @@ let }; } '' - mkdir -p root/nix/store root/etc/nixmodules "$out/blobs/sha256" - cp -a --reflink=auto "$bundle/etc/nixmodules/." root/etc/nixmodules/ - while IFS= read -r path; do - cp -a --reflink=auto "$path" root/nix/store/ - done < "$bundleClosure/store-paths" - chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules - - bash ${./layer.sh} root layer.tar.gz - python3 ${./layout.py} layer.tar.gz "$out" "$architecture" ${lib.escapeShellArg metadata} + bash ${./build.sh} "$bundle" "$bundleClosure/store-paths" "$out" "$architecture" ${lib.escapeShellArg metadata} ${./layout.py} ''; in image diff --git a/pkgs/bundle-oci/layer.sh b/pkgs/bundle-oci/layer.sh deleted file mode 100644 index c215cd72..00000000 --- a/pkgs/bundle-oci/layer.sh +++ /dev/null @@ -1,8 +0,0 @@ -set -euo pipefail - -export LC_ALL=C -tar --create --format=pax --sort=name --mtime=@1 \ - --owner=11000 --group=11000 --numeric-owner \ - --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ - --xattrs --xattrs-include='*' \ - --directory="$1" . | gzip --no-name -9 > "$2" diff --git a/pkgs/bundle-oci/layout.py b/pkgs/bundle-oci/layout.py index 02f6b228..5b199c29 100644 --- a/pkgs/bundle-oci/layout.py +++ b/pkgs/bundle-oci/layout.py @@ -1,15 +1,47 @@ import hashlib import datetime -import gzip import json import pathlib import shutil +import subprocess import sys -def write_layout(layer, output, architecture, metadata): +def verify_layer(raw_image, compressed_layer): + raw_image = pathlib.Path(raw_image) + compressed_layer = pathlib.Path(compressed_layer) + raw_digest = hashlib.sha256() + raw_size = 0 + with raw_image.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + raw_digest.update(chunk) + raw_size += len(chunk) + + process = subprocess.Popen( + ["zstd", "-d", "-c", "--", str(compressed_layer)], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + decompressed_digest = hashlib.sha256() + decompressed_size = 0 + assert process.stdout is not None + for chunk in iter(lambda: process.stdout.read(1024 * 1024), b""): + decompressed_digest.update(chunk) + decompressed_size += len(chunk) + if process.wait() != 0: + raise ValueError("compressed EROFS layer failed zstd verification") + if decompressed_size != raw_size or decompressed_digest.digest() != raw_digest.digest(): + raise ValueError("compressed layer does not match the raw EROFS image") + + +def write_layout(raw_image, compressed_layer, output, architecture, metadata): architectures = {"x86_64": "amd64", "aarch64": "arm64"} + if architecture not in architectures: + raise ValueError(f"unsupported architecture: {architecture}") architecture = architectures[architecture] + raw_image = pathlib.Path(raw_image) + compressed_layer = pathlib.Path(compressed_layer) + output = pathlib.Path(output) blobs = output / "blobs" / "sha256" annotations = { "org.opencontainers.image.source": "https://github.com/replit/nixmodules", @@ -28,14 +60,22 @@ def write_layout(layer, output, architecture, metadata): annotations["org.opencontainers.image.created"] = timestamp created["created"] = timestamp - def descriptor(path, media_type): - digest = hashlib.sha256() - with path.open("rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - digest.update(chunk) + raw_digest = hashlib.sha256() + raw_size = 0 + with raw_image.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + raw_digest.update(chunk) + raw_size += len(chunk) + + compressed_digest = hashlib.sha256() + with compressed_layer.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + compressed_digest.update(chunk) + + def descriptor(path, media_type, digest=None): return { "mediaType": media_type, - "digest": "sha256:" + digest.hexdigest(), + "digest": "sha256:" + (digest or hashlib.sha256(path.read_bytes()).hexdigest()), "size": path.stat().st_size, } @@ -45,19 +85,22 @@ def store_json(value, media_type): path.write_bytes(data) return descriptor(path, media_type) - diff_digest = hashlib.sha256() - with gzip.open(layer, "rb") as source: - for chunk in iter(lambda: source.read(1024 * 1024), b""): - diff_digest.update(chunk) - layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.tar+gzip") - shutil.move(layer, blobs / layer_descriptor["digest"].split(":")[1]) + layer_descriptor = { + **descriptor( + compressed_layer, + "application/vnd.oci.image.layer.v1.erofs+zstd", + compressed_digest.hexdigest(), + ), + "annotations": {"dev.replit.erofs.uncompressed-size": str(raw_size)}, + } + shutil.move(compressed_layer, blobs / compressed_digest.hexdigest()) config = store_json( { **created, "architecture": architecture, "os": "linux", "config": {}, - "rootfs": {"type": "layers", "diff_ids": ["sha256:" + diff_digest.hexdigest()]}, + "rootfs": {"type": "layers", "diff_ids": ["sha256:" + raw_digest.hexdigest()]}, }, "application/vnd.oci.image.config.v1+json", ) @@ -80,6 +123,9 @@ def store_json(value, media_type): if __name__ == "__main__": write_layout( - pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), - sys.argv[3], json.loads(sys.argv[4]), + pathlib.Path(sys.argv[1]), + pathlib.Path(sys.argv[2]), + pathlib.Path(sys.argv[3]), + sys.argv[4], + json.loads(sys.argv[5]), ) diff --git a/pkgs/default.nix b/pkgs/default.nix index d75f53a9..95f7adc6 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -52,14 +52,6 @@ rec { bundle-oci-check = pkgs.callPackage ./bundle-oci/check.nix { }; - bundle-oci-erofs-zstd = pkgs.callPackage ./bundle-oci-erofs-zstd { - inherit bundle; - revision = self.rev or null; - sourceTimestamp = self.lastModified or null; - }; - - bundle-oci-erofs-zstd-check = pkgs.callPackage ./bundle-oci-erofs-zstd/check.nix { }; - custom-bundle = bundle-fn { moduleIds = dev-module-ids; }; diff --git a/scripts/ci_check.sh b/scripts/ci_check.sh index cf0f78bb..b964c939 100755 --- a/scripts/ci_check.sh +++ b/scripts/ci_check.sh @@ -20,4 +20,3 @@ nix develop "${NIX_FLAGS[@]}" --command echo Hello, world nix eval .#bundle nix build .#bundle-oci-check --no-link -nix build .#bundle-oci-erofs-zstd-check --no-link