diff --git a/README.md b/README.md index 8cf7a992..52741168 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ Output might look like: { "bun-0.5": "/nix/store/l08f5vl1af9rpzd7kvr0l5gx9v7y8p12-replit-module-bun-0.5", "c-clang14.0": "/nix/store/n4vzd9rkpjs72xj9yvlakxh3bardvdki-replit-module-c-clang14.0", - "clojure-1.11": "/nix/store/giqq76fl3yphzsm6rkl1qxqh4mszknpl-replit-module-clojure-1.11", + "clojure-1.12": "/nix/store/69ai122m5grza3cm50l7ivb941av6ppb-replit-module-clojure-1.12", "cpp-clang14": "/nix/store/8iv4czda6j5nfhxs80ci625hj91ffpbn-replit-module-cpp-clang14", - "dart-2.18": "/nix/store/mhl58f8y3z6jv0javkmx28y5h9aacw39-replit-module-dart-2.18", - "dotnet-7.0": "/nix/store/06mjna44a7w9bby6r121a7i9a5027qqn-replit-module-dotnet-7.0", - "go-1.19": "/nix/store/y1gb1k8bkyd9jqxi4r1g9qibcqn3c6dm-replit-module-go-1.19", + "dart-3.10": "/nix/store/3i16jlnmcwdhmwqg46p2hrkqfzm3zvg4-replit-module-dart-3.10", + "dotnet-8.0": "/nix/store/dhrbdryqbkbyyi1wlnpvlwi28ni2hwcj-replit-module-dotnet-8.0", + "go-1.25": "/nix/store/l76y8m5y19biid1wmqy03442lx9kairb-replit-module-go-1.25", ... } ``` @@ -31,6 +31,13 @@ nix eval .#modules --json | jq ``` This will take a while. +Retired module IDs remain available through `pkgs/upgrade-map`: they resolve to +newer modules without keeping the retired implementations in the bundle. +For example, Haskell 9.0/9.2/9.4 resolve to 9.10, Python 3.8/3.9 to 3.10, +and Swift 5.6 to 5.8. The unversioned `swift` alias also resolves to 5.8. +The old Python and Node.js Prybar IDs now use the standard Python 3.10 and +Node.js 20 file runners, not Prybar. Projects using retired IDs get the newer +runtime and tools with this bundle and may need code or dependency changes. ## Building @@ -93,17 +100,20 @@ Local sources without a clean revision omit the revision annotation; sources without a last-modified timestamp omit creation metadata. Publish from a revision-pinned Git flake when complete provenance is required. -It contains one uncompressed `application/vnd.oci.image.layer.v1.erofs` layer, -not a tar layer. The consumer must support native EROFS layers; ordinary -tar-only image unpackers cannot use it. Its diffID equals the layer blob digest. +It contains one `application/vnd.oci.image.layer.v1.erofs` layer, not a tar layer. +The whole filesystem uses zstd level 15 with 64 KiB clusters and tail packing, +without an uncompressed hot set. The consumer must support native EROFS layers +and guest-kernel zstd decompression; ordinary tar-only image unpackers cannot +use it. Its diffID equals the layer blob digest. The filesystem is generated directly from the same pinned full bundle used by `disk-script`: its complete store closure and `/etc/nixmodules` metadata, with unchanged store paths, file contents, modes and symlink targets. All guest UIDs/GIDs are 11000, matching the legacy disk builders. Existing disk and -per-module OCI outputs are unchanged. There is no disk conversion or dependency -update. Uncompressed EROFS does not by itself guarantee DAX sharing; that also -depends on the consumer's backing storage and mount configuration. +per-module OCI builders are unchanged. There is no disk conversion or dependency +update. Compressed files are not DAX-eligible: each guest decompresses into its +own page cache, paying decompression CPU and memory rather than sharing pages +through DAX. A small fixture exercises the producer without building the full bundle: diff --git a/pkgs/bundle-oci/check.nix b/pkgs/bundle-oci/check.nix index eec9ab62..0459eda1 100644 --- a/pkgs/bundle-oci/check.nix +++ b/pkgs/bundle-oci/check.nix @@ -4,12 +4,14 @@ let dependency = pkgs.runCommand "bundle-oci-dependency" { } '' mkdir -p "$out" printf 'dependency\n' > "$out/data" + head -c 131072 /dev/zero >> "$out/data" ln "$out/data" "$out/hardlink" ''; module = pkgs.runCommand "bundle-oci-module" { } '' mkdir -p "$out/bin" printf '%s\n' '${dependency}' > "$out/module.json" printf '#!/bin/sh\nexit 0\n' > "$out/bin/tool" + head -c 131072 /dev/zero >> "$out/bin/tool" chmod 755 "$out/bin/tool" ln -s ../module.json "$out/bin/relative" ln -s '${dependency}/data' "$out/absolute" diff --git a/pkgs/bundle-oci/check.py b/pkgs/bundle-oci/check.py index b332fb73..bf69e1ef 100644 --- a/pkgs/bundle-oci/check.py +++ b/pkgs/bundle-oci/check.py @@ -71,6 +71,28 @@ def blob(descriptor): assert "created" not in local_config subprocess.run(["fsck.erofs", "--extract=extracted", str(erofs)], check=True) root = pathlib.Path("extracted") + + +def assert_layout(path, expected): + info = subprocess.check_output( + ["dump.erofs", f"--path={path}", str(erofs)], text=True + ) + assert re.search(rf"Layout:\s*{expected}\b", info), info + + +assert_layout(f"{(bundle / 'fixture').resolve()}/bin/tool", 3) +assert_layout(f"{dependency}/data", 3) +for workers in (1, 2): + rebuilt = pathlib.Path(f"workers-{workers}.erofs") + subprocess.run([ + "mkfs.erofs", "-T", "1", "-U", "00000000-0000-0000-0000-000000000000", + "--force-uid=11000", "--force-gid=11000", + "-zzstd,level=15", "-C65536", "-Eztailpacking", + f"--workers={workers}", str(rebuilt), str(root), + ], check=True) + assert "sha256:" + hashlib.sha256(rebuilt.read_bytes()).hexdigest() == layer["digest"] +print("Compressed file layouts verified; layer digest identical with 1 and 2 workers") + paths = closure.read_text().splitlines() assert str(dependency) in paths assert {p.name for p in (root / "nix/store").iterdir()} == { diff --git a/pkgs/bundle-oci/default.nix b/pkgs/bundle-oci/default.nix index 3693a580..fc699d30 100644 --- a/pkgs/bundle-oci/default.nix +++ b/pkgs/bundle-oci/default.nix @@ -45,10 +45,11 @@ let done < "$bundleClosure/store-paths" chmod 755 root root/nix root/nix/store root/etc root/etc/nixmodules - # Match the legacy disk's guest ownership. No compression: consumers may - # mount this blob directly; compression must not imply DAX page sharing. + # Compress the whole layer instead of preserving DAX-eligible files: + # each guest decompresses into its own page cache, without DAX sharing. mkfs.erofs -T 1 -U 00000000-0000-0000-0000-000000000000 \ - --force-uid=11000 --force-gid=11000 layer.erofs root + --force-uid=11000 --force-gid=11000 \ + -zzstd,level=15 -C65536 -Eztailpacking layer.erofs root python3 ${./layout.py} layer.erofs "$out" "$architecture" ${lib.escapeShellArg metadata} ''; in diff --git a/pkgs/default.nix b/pkgs/default.nix index ecf1dd5a..95f7adc6 100644 --- a/pkgs/default.nix +++ b/pkgs/default.nix @@ -13,11 +13,10 @@ let "nodejs-20" "nodejs-22" "nodejs-24" - "go-1.21" + "go-1.25" "docker" "replit" "replit-rtld-loader" - "ruby" "ruby-3.2" "postgresql-16" ]; diff --git a/pkgs/historical-modules/default.nix b/pkgs/historical-modules/default.nix index 61c576a2..bc850fe9 100644 --- a/pkgs/historical-modules/default.nix +++ b/pkgs/historical-modules/default.nix @@ -24,56 +24,6 @@ let displayVersion = "1.1.45"; }; } - { - moduleId = "clojure-1.11"; - commit = "4327245815e8500233ed3af1cbb674bd147f673b"; - overrides = { - displayVersion = "1.11"; - }; - } - { - moduleId = "dart-2.18"; - commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b"; - overrides = { - # /nix/store/05i6y0nwn8zgm6250my14xbcialzak9b-replit-module-dart-2.18 - # .runners.dart.displayVersion = "2.18.0"; - displayVersion = "2"; - }; - } - { - moduleId = "dart-3.0"; - commit = "0b7a60667d2c29f2686211e952924a9693916a20"; - overrides = { - # /nix/store/0wr2lk1h8vqzs38pfwiil6cf6s5mn54j-replit-module-dart-3.2 - # .runners.dart.displayVersion = "3.2.4"; - displayVersion = "3"; - }; - } - { - moduleId = "dart-3.1"; - commit = "3b22c787fd20b13fe5afb868589c574068303b5e"; - # displayVersion = "3.2.4"; - overrides = { - displayVersion = "3"; - }; - } - { - moduleId = "dart-3.2"; - commit = "9e3a33995f94a52e61ae962777e9bcbe6d75885e"; - # displayVersion = "3.2.4"; - overrides = { - displayVersion = "3"; - }; - } - { - moduleId = "dart-3.3"; - commit = "54ea6c58e71bf278c3ce8824de3dbbaa49662be4"; - overrides = { - # /nix/store/ampsihgvns831ig1w0warbpq1pyj2cg0-replit-module-dart-3.3 - # .runners.dart.displayVersion = "3.3.4"; - displayVersion = "3"; - }; - } { moduleId = "dart-3.8"; commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; @@ -81,13 +31,6 @@ let displayVersion = "3.8"; }; } - { - moduleId = "deno-1"; - commit = "4327245815e8500233ed3af1cbb674bd147f673b"; - overrides = { - displayVersion = "1"; - }; - } { moduleId = "docker"; commit = "185d7dc7645178f92fd891f9efa20b3343c0f20d"; @@ -96,124 +39,6 @@ let internal = true; }; } - { - moduleId = "dotnet-7.0"; - commit = "4327245815e8500233ed3af1cbb674bd147f673b"; - overrides = { - displayVersion = "7.0"; - }; - } - { - moduleId = "go-1.19"; - commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b"; - overrides = { - # /nix/store/7icmam9gmbfas0zbpm53f3ilw79kx0rj-replit-module-go-1.19 - # .runners["go-run"].displayVersion = "1.19.3"; - displayVersion = "1.19"; - }; - } - { - moduleId = "go-1.20"; - commit = "b5aa5df636c4cd8cd1aea251e8dea4fc0aa51781"; - overrides = { - # /nix/store/q6j9za8smkdbnlybg63qq5lmj7zbambr-replit-module-go-1.20 - # .runners["go-run"].displayVersion = "1.20.4"; - displayVersion = "1.20"; - }; - } - { - moduleId = "go-1.21"; - commit = "76ae6535d7d60e767fe9d54902400229ca0b9448"; - overrides = { - # /nix/store/...-replit-module-go-1.21 - # .runners["go-run"].displayVersion = "1..."; - displayVersion = "1.21"; - }; - } - { - moduleId = "haskell-ghc9.0"; - commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b"; - overrides = { - # /nix/store/h6hp6fd07dr8q9dhlz45nrs17wp2kmp4-replit-module-haskell-ghc9.0 - # .runners.runghc.displayVersion = "9.0.2"; - displayVersion = "9"; - }; - } - { - moduleId = "haskell-ghc9.2"; - commit = "4c6f2315da24b84bd5e9dfedb952e41677724aaa"; - overrides = { - # /nix/store/1r94iizf604jasg72f239fw0if4ldmd5-replit-module-haskell-ghc9.2 - # .runners.runghc.displayVersion = "9.2.8"; - displayVersion = "9"; - }; - } - { - moduleId = "haskell-ghc9.4"; - commit = "4d6f7cd9fd685a3319ac7b6e3fc0789b430d6289"; - overrides = { - # /nix/store/kkaxlri9s0bb7jydb1kpqzi2sjkf4rmj-replit-module-haskell-ghc9.4 - # .runners.runghc.displayVersion = "9.4.8"; - displayVersion = "9"; - }; - } - { - moduleId = "elixir-1_15"; - commit = "4d6f7cd9fd685a3319ac7b6e3fc0789b430d6289"; - overrides = { - # /nix/store/8nggvav01q7f1094m485nnd32094j522-replit-module-elixir-1_15 - # .runners.elixir.displayVersion = "1.15.7"; - displayVersion = "1"; - }; - } - { - moduleId = "nodejs-14"; - commit = "f4cd419a646009297c049a2f1eec434381e08f13"; - overrides = { - name = "Node.js Tools"; - displayVersion = "14"; - runners = { - nodeJS = { - displayVersion = "14.21.1"; - }; - }; - }; - } - { - moduleId = "nodejs-16"; - commit = "f4cd419a646009297c049a2f1eec434381e08f13"; - overrides = { - name = "Node.js Tools"; - displayVersion = "16"; - runners = { - nodeJS = { - displayVersion = "16.18.1"; - }; - }; - }; - } - { - moduleId = "nodejs-19"; - commit = "f4cd419a646009297c049a2f1eec434381e08f13"; - overrides = { - name = "Node.js Tools"; - displayVersion = "19"; - runners = { - nodeJS = { - displayVersion = "19.1.0"; - }; - }; - }; - } - { - moduleId = "php-8.1"; - commit = "0b7a60667d2c29f2686211e952924a9693916a20"; - overrides = { - # /nix/store/xc89rlwnmg5vh66hvrv1saq6md6kp1g5-replit-module-php-8.1 - # .runners.php.displayVersion = "8.1.20"; - displayVersion = "8"; - }; - } { moduleId = "php-8.2"; commit = "4327245815e8500233ed3af1cbb674bd147f673b"; @@ -221,38 +46,6 @@ let displayVersion = "8.2"; }; } - { - moduleId = "python-3.8"; - commit = "76ae6535d7d60e767fe9d54902400229ca0b9448"; - overrides = { - displayVersion = "3.8"; - }; - } - { - moduleId = "python-3.9"; - commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; - overrides = { - displayVersion = "3.9"; - }; - } - { - moduleId = "r-4.2"; - commit = "1e1bb663068482cdb7c04bf585daed00205c0140"; - overrides = { - # /nix/store/knizlzgmqgm488llvxxaai6kjb17mhm2-replit-module-r-4.2 - # .runners.r.displayVersion = "4.2.3"; - displayVersion = "4"; - }; - } - { - moduleId = "swift-5.6"; - commit = "c48c43c6c698223ed3ce2abc5a2d708735a77d5b"; - overrides = { - # /nix/store/f96l8blsvpx1dbgnxw2x611hhzbm199l-replit-module-swift-5.6 - # .runners.swift.displayVersion = "5.6.2"; - displayVersion = "5"; - }; - } { moduleId = "zig-0.11"; commit = "15426ef79793bf7c424eb40865d507eacfdd44e6"; @@ -262,19 +55,10 @@ let displayVersion = "0"; }; } - { - moduleId = "python-with-prybar-3.10"; - commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; - overrides = { - displayVersion = "3.10"; - }; - } { moduleId = "go-1.24"; commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; overrides = { - # /nix/store/...-replit-module-go-1.21 - # .runners["go-run"].displayVersion = "1..."; displayVersion = "1.24"; }; } @@ -285,13 +69,6 @@ let displayVersion = "3.1"; }; } - { - moduleId = "nodejs-with-prybar-18"; - commit = "ae6ed05c623804560afd1cf3d80d13b6170d5c24"; - overrides = { - displayVersion = "18"; - }; - } ]; moduleFromHistory = diff --git a/pkgs/modules/default.nix b/pkgs/modules/default.nix index ad2edbec..96f08482 100644 --- a/pkgs/modules/default.nix +++ b/pkgs/modules/default.nix @@ -25,10 +25,6 @@ let historical = pkgs.callPackage ../historical-modules { }; modulesList = [ - (import ./python { - python = pkgs-24_11.python39Full; - pypkgs = pkgs-24_11.python39Packages; - }) (import ./python { python = pkgs.python310; pypkgs = pkgs.python310Packages; @@ -56,18 +52,12 @@ let (import ./ty) (import ./ruff) - (import ./nodejs { - nodejs = pkgs-24_11.nodejs-18_x; - }) (import ./nodejs { nodejs = pkgs-master.nodejs_20; }) (import ./nodejs { nodejs = pkgs-master.nodejs_22; }) - (import ./nodejs { - nodejs = pkgs-24_11.nodejs_23; - }) (import ./nodejs { nodejs = pkgs-staging.nodejs_24; }) @@ -148,9 +138,6 @@ let (import ./vue { nodejs = pkgs.nodejs_20; }) - (import ./vue { - nodejs = pkgs-24_11.nodejs-18_x; - }) (import ./web) (import ./hermit) (import ./typescript-language-server { diff --git a/pkgs/upgrade-map/default.nix b/pkgs/upgrade-map/default.nix index 445f6b40..7a337d83 100644 --- a/pkgs/upgrade-map/default.nix +++ b/pkgs/upgrade-map/default.nix @@ -13,6 +13,8 @@ let "bun-1.0" = "bun-1.1"; "bun-1.1" = "bun-1.2"; "bun-1.2" = "bun-1.3"; + "clojure-1.11" = "clojure-1.12"; + "dart-2.18" = "dart-3.8"; "dart-3.0" = "dart-3.1"; "dart-3.1" = "dart-3.2"; "dart-3.2" = "dart-3.3"; @@ -20,13 +22,33 @@ let "dart-3.4" = "dart-3.5"; "dart-3.5" = "dart-3.8"; "dart-3.8" = "dart-3.10"; + "deno-1" = "deno-2"; + "dotnet-7.0" = "dotnet-8.0"; + "elixir-1_15" = "elixir-1_17"; "elixir-1_16" = "elixir-1_17"; "elixir-1_17" = "elixir-1_18"; - "go" = "go-1.19"; + "go" = "go-1.25"; + "go-1.19" = "go-1.24"; + "go-1.20" = "go-1.24"; + "go-1.21" = "go-1.24"; "go-1.23" = "go-1.24"; "go-1.24" = "go-1.25"; + "haskell-ghc9.0" = "haskell-ghc9.10"; + "haskell-ghc9.2" = "haskell-ghc9.10"; + "haskell-ghc9.4" = "haskell-ghc9.10"; + "nodejs-14" = "nodejs-20"; + "nodejs-16" = "nodejs-20"; + "nodejs-18" = "nodejs-20"; + "nodejs-19" = "nodejs-20"; + "nodejs-23" = "nodejs-24"; + "nodejs-with-prybar-18" = "nodejs-20"; + "php-8.1" = "php-8.2"; "pyright" = "ty"; "pyright-extended" = "ty"; + "python-3.8" = "python-3.10"; + "python-3.9" = "python-3.10"; + "python-with-prybar-3.10" = "python-3.10"; + "r-4.2" = "r-4.4"; "r-4.3" = "r-4.4"; "r-4.4" = "r-4.5"; "rust" = "rust-1.69"; @@ -34,7 +56,9 @@ let "rust-1.70" = "rust-1.72"; "rust-1.72" = "rust-stable"; "ruby-3.1" = "ruby-3.2"; - "swift" = "swift-5.6"; + "swift" = "swift-5.8"; + "swift-5.6" = "swift-5.8"; + "vue-node-18" = "vue-node-20"; }; upgrade-module =