Use gzip tar layers for bundle OCI images

README.md

@@ -73,7 +73,7 @@ or
 nix build .#custom-bundle-squashfs
 ```
 
-To build the full bundle (including historical modules) as a native EROFS OCI image:
+To build the full bundle as a gzip-compressed tar OCI image:
 
 ```
 nix build .#bundle-oci
@@ -100,20 +100,21 @@ Local sources without a clean revision omit the revision annotation; sources
 without a last-modified timestamp omit creation metadata. Publish from a
 revision-pinned Git flake when complete provenance is required.
 
-It contains one `application/vnd.oci.image.layer.v1.erofs` layer, not a tar layer.
-The whole filesystem uses zstd level 15 with 64 KiB clusters and tail packing,
-without an uncompressed hot set. The consumer must support native EROFS layers
-and guest-kernel zstd decompression; ordinary tar-only image unpackers cannot
-use it. Its diffID equals the layer blob digest.
+It contains one `application/vnd.oci.image.layer.v1.tar+gzip` layer. Gzip is
+transport compression: the consumer decompresses it when unpacking the image.
+The manifest describes the compressed blob's digest and size; the config's
+diffID is the digest of the uncompressed tar stream. Tar entry order,
+timestamps, ownership, extended headers and gzip headers are deterministic.
 
 The filesystem is generated directly from the same pinned full bundle used by
 `disk-script`: its complete store closure and `/etc/nixmodules` metadata, with
 unchanged store paths, file contents, modes and symlink targets. All guest
 UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and
 per-module OCI builders are unchanged. There is no disk conversion or dependency
-update. Compressed files are not DAX-eligible: each guest decompresses into its
-own page cache, paying decompression CPU and memory rather than sharing pages
-through DAX.
+update. The tar archive preserves hardlinks and extended attributes. A consumer
+that converts the tar to EROFS must leave filesystem compression disabled for
+files to remain DAX-eligible. This uses more unpacked disk space than compressed
+EROFS; it does not automatically share all guest memory.
 
 A small fixture exercises the producer without building the full bundle:
 

pkgs/bundle-oci/check.nix

@@ -28,10 +28,10 @@ let
 in
 pkgs.runCommand "bundle-oci-check"
 {
-  nativeBuildInputs = [ pkgs.erofs-utils pkgs.python3 ];
+  nativeBuildInputs = [ pkgs.gnutar pkgs.gzip pkgs.python3 ];
 }
   ''
-    python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py}
+    python3 ${./check.py} ${image} ${bundle} ${closure}/store-paths ${dependency} ${./layout.py} ${./layer.sh}
     ${image.copyTo}/bin/copy-nixmodules-bundle-oci "oci:$PWD/copied:bundle" \
       --authfile ${pkgs.writeText "empty-auth.json" ''{"auths":{}}''} \
       --digestfile "$PWD/copied.digest"

pkgs/bundle-oci/check.py

@@ -1,15 +1,17 @@
 import hashlib
+import gzip
+import errno
 import importlib.util
 import json
 import os
 import pathlib
-import re
 import stat
 import shutil
 import subprocess
 import sys
+import tarfile
 
-image, bundle, closure, dependency, layout_script = map(pathlib.Path, sys.argv[1:])
+image, bundle, closure, dependency, layout_script, layer_script = map(pathlib.Path, sys.argv[1:])
 assert json.loads((image / "oci-layout").read_text()) == {"imageLayoutVersion": "1.0.0"}
 
 
@@ -37,16 +39,20 @@ def blob(descriptor):
 }
 assert len(manifest["layers"]) == 1
 layer = manifest["layers"][0]
-assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.erofs"
-assert config["rootfs"] == {"type": "layers", "diff_ids": [layer["digest"]]}
-erofs = blob(layer)
+assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip"
+archive = blob(layer)
+tar_data = gzip.decompress(archive.read_bytes())
+diff_id = "sha256:" + hashlib.sha256(tar_data).hexdigest()
+assert config["rootfs"] == {"type": "layers", "diff_ids": [diff_id]}
+assert diff_id != layer["digest"]
+assert archive.read_bytes()[3:8] == b"\x00" * 5
 spec = importlib.util.spec_from_file_location("layout", layout_script)
 layout = importlib.util.module_from_spec(spec)
 spec.loader.exec_module(layout)
 repeated = pathlib.Path("repeated")
 (repeated / "blobs/sha256").mkdir(parents=True)
-shutil.copyfile(erofs, "repeated.erofs")
-layout.write_layout(pathlib.Path("repeated.erofs"), repeated, "x86_64", {
+shutil.copyfile(archive, "repeated.tar.gz")
+layout.write_layout(pathlib.Path("repeated.tar.gz"), repeated, "x86_64", {
     "revision": "0123456789abcdef0123456789abcdef01234567",
     "sourceTimestamp": 1700000000,
 })
@@ -55,8 +61,8 @@ def blob(descriptor):
         assert path.read_bytes() == (repeated / path.relative_to(image)).read_bytes()
 local = pathlib.Path("local")
 (local / "blobs/sha256").mkdir(parents=True)
-shutil.copyfile(erofs, "local.erofs")
-layout.write_layout(pathlib.Path("local.erofs"), local, "x86_64", {
+shutil.copyfile(archive, "local.tar.gz")
+layout.write_layout(pathlib.Path("local.tar.gz"), local, "x86_64", {
     "revision": None, "sourceTimestamp": None,
 })
 local_descriptor = json.loads((local / "index.json").read_text())["manifests"][0]
@@ -69,29 +75,24 @@ def blob(descriptor):
     (local / "blobs/sha256" / local_manifest["config"]["digest"].split(":")[1]).read_text()
 )
 assert "created" not in local_config
-subprocess.run(["fsck.erofs", "--extract=extracted", str(erofs)], check=True)
 root = pathlib.Path("extracted")
+root.mkdir()
+subprocess.run([
+    "tar", "--extract", "--gzip", "--file", str(archive),
+    "--directory", str(root), "--no-same-owner", "--same-permissions",
+    "--xattrs", "--xattrs-include=*",
+], check=True)
+with tarfile.open(archive, "r:gz") as source:
+    members = source.getmembers()
+    assert all(m.uid == 11000 and m.gid == 11000 and m.mtime == 1 for m in members)
+    assert all(not m.uname and not m.gname for m in members)
+    assert all("atime" not in m.pax_headers and "ctime" not in m.pax_headers for m in members)
+    assert [m.name for m in members] == sorted(m.name for m in members)
 
-
-def assert_layout(path, expected):
-    info = subprocess.check_output(
-        ["dump.erofs", f"--path={path}", str(erofs)], text=True
-    )
-    assert re.search(rf"Layout:\s*{expected}\b", info), info
-
-
-assert_layout(f"{(bundle / 'fixture').resolve()}/bin/tool", 3)
-assert_layout(f"{dependency}/data", 3)
-for workers in (1, 2):
-    rebuilt = pathlib.Path(f"workers-{workers}.erofs")
-    subprocess.run([
-        "mkfs.erofs", "-T", "1", "-U", "00000000-0000-0000-0000-000000000000",
-        "--force-uid=11000", "--force-gid=11000",
-        "-zzstd,level=15", "-C65536", "-Eztailpacking",
-        f"--workers={workers}", str(rebuilt), str(root),
-    ], check=True)
-    assert "sha256:" + hashlib.sha256(rebuilt.read_bytes()).hexdigest() == layer["digest"]
-print("Compressed file layouts verified; layer digest identical with 1 and 2 workers")
+rebuilt = pathlib.Path("rebuilt.tar.gz")
+subprocess.run(["bash", str(layer_script), str(root), str(rebuilt)], check=True)
+assert rebuilt.read_bytes() == archive.read_bytes()
+print("Tar metadata and reproducible gzip layer verified")
 
 paths = closure.read_text().splitlines()
 assert str(dependency) in paths
@@ -104,12 +105,6 @@ def compare(source, target):
     before, after = source.lstat(), target.lstat()
     assert stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode), target
     assert stat.S_IMODE(before.st_mode) == stat.S_IMODE(after.st_mode), target
-    guest = "/" + str(target.relative_to(root))
-    info = subprocess.check_output(
-        ["dump.erofs", f"--path={guest}", str(erofs)], text=True
-    )
-    assert re.search(r"Uid:\s*11000\b", info), info
-    assert re.search(r"Gid:\s*11000\b", info), info
     if source.is_symlink():
         assert os.readlink(source) == os.readlink(target), target
     elif source.is_dir():
@@ -129,3 +124,36 @@ def compare(source, target):
     packed_dependency / "hardlink"
 ).stat().st_ino
 print("OCI descriptors, complete closure, metadata, content, modes, symlinks, ownership and hardlinks verified")
+
+xattr_root = pathlib.Path("xattrs")
+xattr_root.mkdir()
+xattr_file = xattr_root / ("long-name-" * 20)
+xattr_file.write_bytes(b"xattrs\n")
+try:
+    os.setxattr(xattr_file, "user.fixture", b"preserved")
+    xattrs_supported = True
+except OSError as error:
+    if error.errno != errno.ENOTSUP:
+        raise
+    xattrs_supported = False
+    print("Skipping xattr round-trip: build filesystem does not support xattrs")
+subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs.tar.gz"], check=True)
+with tarfile.open("xattrs.tar.gz", "r:gz") as source:
+    member = source.getmember("./" + xattr_file.name)
+    assert member.isfile() and source.extractfile(member).read() == b"xattrs\n"
+    if xattrs_supported:
+        assert member.pax_headers["SCHILY.xattr.user.fixture"] == "preserved"
+if xattrs_supported:
+    restored = pathlib.Path("xattrs-restored")
+    restored.mkdir()
+    subprocess.run([
+        "tar", "--extract", "--gzip", "--file=xattrs.tar.gz",
+        "--directory", str(restored), "--no-same-owner",
+        "--xattrs", "--xattrs-include=*",
+    ], check=True)
+    assert os.getxattr(restored / xattr_file.name, "user.fixture") == b"preserved"
+    print("Extended attributes preserved in archive and extraction")
+os.utime(xattr_file, (12345, 67890))
+subprocess.run(["bash", str(layer_script), str(xattr_root), "xattrs-again.tar.gz"], check=True)
+assert pathlib.Path("xattrs.tar.gz").read_bytes() == pathlib.Path("xattrs-again.tar.gz").read_bytes()
+print("Long paths preserved; source timestamps do not change the layer")

pkgs/bundle-oci/default.nix

@@ -3,7 +3,8 @@
 , stdenv
 , bundle
 , closureInfo
-, erofs-utils
+, gnutar
+, gzip
 , python3
 , writeShellApplication
 , skopeo
@@ -16,7 +17,7 @@ let
   metadata = builtins.toJSON { inherit revision sourceTimestamp; };
   image = runCommand "nixmodules-bundle-oci"
     {
-      nativeBuildInputs = [ erofs-utils python3 ];
+      nativeBuildInputs = [ gnutar gzip python3 ];
       inherit bundle bundleClosure;
       architecture = lib.toLower stdenv.hostPlatform.parsed.cpu.name;
       __structuredAttrs = true;
@@ -45,12 +46,8 @@ let
       done < "$bundleClosure/store-paths"
       chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules
 
-      # Compress the whole layer instead of preserving DAX-eligible files:
-      # each guest decompresses into its own page cache, without DAX sharing.
-      mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \
-        --force-uid=11000 --force-gid=11000 \
-        -zzstd,level=15 -C65536 -Eztailpacking layer.erofs root
-      python3 ${./layout.py} layer.erofs "$out" "$architecture" ${lib.escapeShellArg metadata}
+      bash ${./layer.sh} root layer.tar.gz
+      python3 ${./layout.py} layer.tar.gz "$out" "$architecture" ${lib.escapeShellArg metadata}
     '';
 in
 image

pkgs/bundle-oci/layer.sh

@@ -0,0 +1,8 @@
+set -euo pipefail
+
+export LC_ALL=C
+tar --create --format=pax --sort=name --mtime=@1 \
+  --owner=11000 --group=11000 --numeric-owner \
+  --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
+  --xattrs --xattrs-include='*' \
+  --directory="$1" . | gzip --no-name -9 > "$2"

pkgs/bundle-oci/layout.py

@@ -1,5 +1,6 @@
 import hashlib
 import datetime
+import gzip
 import json
 import pathlib
 import shutil
@@ -44,15 +45,19 @@ def store_json(value, media_type):
         path.write_bytes(data)
         return descriptor(path, media_type)
 
-    layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.erofs")
+    diff_digest = hashlib.sha256()
+    with gzip.open(layer, "rb") as source:
+        for chunk in iter(lambda: source.read(1024 * 1024), b""):
+            diff_digest.update(chunk)
+    layer_descriptor = descriptor(layer, "application/vnd.oci.image.layer.v1.tar+gzip")
     shutil.move(layer, blobs / layer_descriptor["digest"].split(":")[1])
     config = store_json(
         {
             **created,
             "architecture": architecture,
             "os": "linux",
             "config": {},
-            "rootfs": {"type": "layers", "diff_ids": [layer_descriptor["digest"]]},
+            "rootfs": {"type": "layers", "diff_ids": ["sha256:" + diff_digest.hexdigest()]},
         },
         "application/vnd.oci.image.config.v1+json",
     )