From c14de592bbad1c910fc9b55ba2d2264db79fd6d8 Mon Sep 17 00:00:00 2001 From: rahul Date: Wed, 14 Oct 2026 21:22:55 +0000 Subject: [PATCH 1/2] Use gzip tar layers for bundle OCI images Assisted-by: Replit --- README.md | 19 ++++----- pkgs/bundle-oci/check.nix | 4 +- pkgs/bundle-oci/check.py | 80 ++++++++++++++++++++----------------- pkgs/bundle-oci/default.nix | 13 +++--- pkgs/bundle-oci/layer.sh | 8 ++++ pkgs/bundle-oci/layout.py | 9 ++++- 6 files changed, 76 insertions(+), 57 deletions(-) create mode 100644 pkgs/bundle-oci/layer.sh diff --git a/README.md b/README.md index 52741168..fee69f77 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ or nix build .#custom-bundle-squashfs ``` -To build the full bundle (including historical modules) as a native EROFS OCI image: +To build the full bundle as a gzip-compressed tar OCI image: ``` nix build .#bundle-oci @@ -100,20 +100,21 @@ Local sources without a clean revision omit the revision annotation; sources without a last-modified timestamp omit creation metadata. Publish from a revision-pinned Git flake when complete provenance is required. -It contains one `application/vnd.oci.image.layer.v1.erofs` layer, not a tar layer. -The whole filesystem uses zstd level 15 with 64 KiB clusters and tail packing, -without an uncompressed hot set. The consumer must support native EROFS layers -and guest-kernel zstd decompression; ordinary tar-only image unpackers cannot -use it. Its diffID equals the layer blob digest. +It contains one `application/vnd.oci.image.layer.v1.tar+gzip` layer. Gzip is +transport compression: the consumer decompresses it when unpacking the image. +The manifest describes the compressed blob's digest and size; the config's +diffID is the digest of the uncompressed tar stream. Tar entry order, +timestamps, ownership, extended headers and gzip headers are deterministic. The filesystem is generated directly from the same pinned full bundle used by `disk-script`: its complete store closure and `/etc/nixmodules` metadata, with unchanged store paths, file contents, modes and symlink targets. All guest UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and per-module OCI builders are unchanged. There is no disk conversion or dependency -update. Compressed files are not DAX-eligible: each guest decompresses into its -own page cache, paying decompression CPU and memory rather than sharing pages -through DAX. +update. The tar archive preserves hardlinks and extended attributes. A consumer +that converts the tar to EROFS must leave filesystem compression disabled for +files to remain DAX-eligible. This uses more unpacked disk space than compressed +EROFS; it does not automatically share all guest memory. A small fixture exercises the producer without building the full bundle: diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix index 0459eda1..f9720788 100644 --- a/pkgs/bundle-oci/check.nix +++ b/pkgs/bundle-oci/check.nix @@ -28,10 +28,10 @@ let in pkgs.runCommand "bundle-oci-check" { - nativeBuildInputs = [ pkgs.erofs-utils pkgs.python3 ]; + nativeBuildInputs = [ pkgs.gnutar pkgs.gzip pkgs.python3 ]; } '' - python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} + python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh} ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \ --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \ --digestfile "$PWD/copied.digest" diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index bf69e1ef..0fdfc83e 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -1,15 +1,16 @@ import hashlib +import gzip import importlib.util import json import os import pathlib -import re import stat import shutil import subprocess import sys +import tarfile -image, bundle, closure, dependency, layout_script = map(pathlib.Path, sys.argv[1:]) +image, bundle, closure, dependency, layout_script, layer_script = map(pathlib.Path, sys.argv[1:]) assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"} @@ -37,16 +38,20 @@ def blob(descriptor): } assert len(manifest["layers"]) == 1 layer = manifest["layers"][0] -assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs" -assert config["rootfs"] == {"type": "layers", "diff_ids": [layer["digest"]]} -erofs = blob(layer) +assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip" +archive = blob(layer) +tar_data = gzip.decompress(archive.read_bytes()) +diff_id = "sha256:" + hashlib.sha256(tar_data).hexdigest() +assert config["rootfs"] == {"type": "layers", "diff_ids": [diff_id]} +assert diff_id != layer["digest"] +assert archive.read_bytes()[3:8] == b"\x00" * 5 spec = importlib.util.spec_from_file_location("layout", layout_script) layout = importlib.util.module_from_spec(spec) spec.loader.exec_module(layout) repeated = pathlib.Path("repeated") (repeated / "blobs/sha256").mkdir(parents=True) -shutil.copyfile(erofs, "repeated.erofs") -layout.write_layout(pathlib.Path("repeated.erofs"), repeated, "x86_64", { +shutil.copyfile(archive, "repeated.tar.gz") +layout.write_layout(pathlib.Path("repeated.tar.gz"), repeated, "x86_64", { "revision": "0123456789abcdef0123456789abcdef01234567", "sourceTimestamp": 1700000000, }) @@ -55,8 +60,8 @@ def blob(descriptor): assert path.read_bytes() == (repeated / path.relative_to(image)).read_bytes() local = pathlib.Path("local") (local / "blobs/sha256").mkdir(parents=True) -shutil.copyfile(erofs, "local.erofs") -layout.write_layout(pathlib.Path("local.erofs"), local, "x86_64", { +shutil.copyfile(archive, "local.tar.gz") +layout.write_layout(pathlib.Path("local.tar.gz"), local, "x86_64", { "revision": None, "sourceTimestamp": None, }) local_descriptor = json.loads((local / "index.json").read_text())["manifests"][0] @@ -69,29 +74,24 @@ def blob(descriptor): (local / "blobs/sha256" / local_manifest["config"]["digest"].split(":")[1]).read_text() ) assert "created" not in local_config -subprocess.run(["fsck.erofs", "--extract=extracted", str(erofs)], check=True) root = pathlib.Path("extracted") +root.mkdir() +subprocess.run([ + "tar", "--extract", "--gzip", "--file", str(archive), + "--directory", str(root), "--no-same-owner", "--same-permissions", + "--xattrs", "--xattrs-include=*", +], check=True) +with tarfile.open(archive, "r:gz") as source: + members = source.getmembers() + assert all(m.uid == 11000 and m.gid == 11000 and m.mtime == 1 for m in members) + assert all(not m.uname and not m.gname for m in members) + assert all("atime" not in m.pax_headers and "ctime" not in m.pax_headers for m in members) + assert [m.name for m in members] == sorted(m.name for m in members) - -def assert_layout(path, expected): - info = subprocess.check_output( - ["dump.erofs", f"--path={path}", str(erofs)], text=True - ) - assert re.search(rf"Layout:\s*{expected}\b", info), info - - -assert_layout(f"{(bundle / 'fixture').resolve()}/bin/tool", 3) -assert_layout(f"{dependency}/data", 3) -for workers in (1, 2): - rebuilt = pathlib.Path(f"workers-{workers}.erofs") - subprocess.run([ - "mkfs.erofs", "-T", "1", "-U", "00000000-0000-0000-0000-000000000000", - "--force-uid=11000", "--force-gid=11000", - "-zzstd,level=15", "-C65536", "-Eztailpacking", - f"--workers={workers}", str(rebuilt), str(root), - ], check=True) - assert "sha256:" + hashlib.sha256(rebuilt.read_bytes()).hexdigest() == layer["digest"] -print("Compressed file layouts verified; layer digest identical with 1 and 2 workers") +rebuilt = pathlib.Path("rebuilt.tar.gz") +subprocess.run(["bash", str(layer_script), str(root), str(rebuilt)], check=True) +assert rebuilt.read_bytes() == archive.read_bytes() +print("Tar metadata and reproducible gzip layer verified") paths = closure.read_text().splitlines() assert str(dependency) in paths @@ -104,12 +104,6 @@ def compare(source, target): before, after = source.lstat(), target.lstat() assert stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode), target assert stat.S_IMODE(before.st_mode) == stat.S_IMODE(after.st_mode), target - guest = "/" + str(target.relative_to(root)) - info = subprocess.check_output( - ["dump.erofs", f"--path={guest}", str(erofs)], text=True - ) - assert re.search(r"Uid:\s*11000\b", info), info - assert re.search(r"Gid:\s*11000\b", info), info if source.is_symlink(): assert os.readlink(source) == os.readlink(target), target elif source.is_dir(): @@ -129,3 +123,17 @@ def compare(source, target): packed_dependency / "hardlink" ).stat().st_ino print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified") + +xattr_root = pathlib.Path("xattrs") +xattr_root.mkdir() +xattr_file = xattr_root / ("long-name-" * 20) +xattr_file.write_bytes(b"xattrs\n") +os.setxattr(xattr_file, "user.fixture", b"preserved") +subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs.tar.gz"], check=True) +with tarfile.open("xattrs.tar.gz", "r:gz") as source: + member = source.getmember("./" + xattr_file.name) + assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved" +os.utime(xattr_file, (12345, 67890)) +subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs-again.tar.gz"], check=True) +assert pathlib.Path("xattrs.tar.gz").read_bytes() == pathlib.Path("xattrs-again.tar.gz").read_bytes() +print("Long paths and extended attributes preserved; source timestamps do not change the layer") diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix index fc699d30..ae357cff 100644 --- a/pkgs/bundle-oci/default.nix +++ b/pkgs/bundle-oci/default.nix @@ -3,7 +3,8 @@ , stdenv , bundle , closureInfo -, erofs-utils +, gnutar +, gzip , python3 , writeShellApplication , skopeo @@ -16,7 +17,7 @@ let metadata = builtins.toJSON { inherit revision sourceTimestamp; }; image = runCommand "nixmodules-bundle-oci" { - nativeBuildInputs = [ erofs-utils python3 ]; + nativeBuildInputs = [ gnutar gzip python3 ]; inherit bundle bundleClosure; architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name; __structuredAttrs = true; @@ -45,12 +46,8 @@ let done < "$bundleClosure/store-paths" chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules - # Compress the whole layer instead of preserving DAX-eligible files: - # each guest decompresses into its own page cache, without DAX sharing. - mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \ - --force-uid=11000 --force-gid=11000 \ - -zzstd,level=15 -C65536 -Eztailpacking layer.erofs root - python3 ${./layout.py} layer.erofs "$out" "$architecture" ${lib.escapeShellArg metadata} + bash ${./layer.sh} root layer.tar.gz + python3 ${./layout.py} layer.tar.gz "$out" "$architecture" ${lib.escapeShellArg metadata} ''; in image diff --git a/pkgs/bundle-oci/layer.sh b/pkgs/bundle-oci/layer.sh new file mode 100644 index 00000000..c215cd72 --- /dev/null +++ b/pkgs/bundle-oci/layer.sh @@ -0,0 +1,8 @@ +set -euo pipefail + +export LC_ALL=C +tar --create --format=pax --sort=name --mtime=@1 \ + --owner=11000 --group=11000 --numeric-owner \ + --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ + --xattrs --xattrs-include='*' \ + --directory="$1" . | gzip --no-name -9 > "$2" diff --git a/pkgs/bundle-oci/layout.py b/pkgs/bundle-oci/layout.py index c2fe2c23..02f6b228 100644 --- a/pkgs/bundle-oci/layout.py +++ b/pkgs/bundle-oci/layout.py @@ -1,5 +1,6 @@ import hashlib import datetime +import gzip import json import pathlib import shutil @@ -44,7 +45,11 @@ def store_json(value, media_type): path.write_bytes(data) return descriptor(path, media_type) - layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.erofs") + diff_digest = hashlib.sha256() + with gzip.open(layer, "rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + diff_digest.update(chunk) + layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.tar+gzip") shutil.move(layer, blobs / layer_descriptor["digest"].split(":")[1]) config = store_json( { @@ -52,7 +57,7 @@ def store_json(value, media_type): "architecture": architecture, "os": "linux", "config": {}, - "rootfs": {"type": "layers", "diff_ids": [layer_descriptor["digest"]]}, + "rootfs": {"type": "layers", "diff_ids": ["sha256:" + diff_digest.hexdigest()]}, }, "application/vnd.oci.image.config.v1+json", ) From e5b986734f10959a09a125ff0356e5f98ee06ac5 Mon Sep 17 00:00:00 2001 From: rahul Date: Wed, 14 Oct 2026 21:26:57 +0000 Subject: [PATCH 2/2] Skip xattr probe when filesystem lacks support Assisted-by: Replit --- pkgs/bundle-oci/check.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index 0fdfc83e..f101a060 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -1,5 +1,6 @@ import hashlib import gzip +import errno import importlib.util import json import os @@ -128,12 +129,31 @@ def compare(source, target): xattr_root.mkdir() xattr_file = xattr_root / ("long-name-" * 20) xattr_file.write_bytes(b"xattrs\n") -os.setxattr(xattr_file, "user.fixture", b"preserved") +try: + os.setxattr(xattr_file, "user.fixture", b"preserved") + xattrs_supported = True +except OSError as error: + if error.errno != errno.ENOTSUP: + raise + xattrs_supported = False + print("Skipping xattr round-trip: build filesystem does not support xattrs") subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs.tar.gz"], check=True) with tarfile.open("xattrs.tar.gz", "r:gz") as source: member = source.getmember("./" + xattr_file.name) - assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved" + assert member.isfile() and source.extractfile(member).read() == b"xattrs\n" + if xattrs_supported: + assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved" +if xattrs_supported: + restored = pathlib.Path("xattrs-restored") + restored.mkdir() + subprocess.run([ + "tar", "--extract", "--gzip", "--file=xattrs.tar.gz", + "--directory", str(restored), "--no-same-owner", + "--xattrs", "--xattrs-include=*", + ], check=True) + assert os.getxattr(restored / xattr_file.name, "user.fixture") == b"preserved" + print("Extended attributes preserved in archive and extraction") os.utime(xattr_file, (12345, 67890)) subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs-again.tar.gz"], check=True) assert pathlib.Path("xattrs.tar.gz").read_bytes() == pathlib.Path("xattrs-again.tar.gz").read_bytes() -print("Long paths and extended attributes preserved; source timestamps do not change the layer") +print("Long paths preserved; source timestamps do not change the layer")