Add erofs-utils package (#12971)

Author: hbeberman

LICENSES-AND-NOTICES/SPECS/LICENSES-MAP.md

@@ -212,6 +212,7 @@
                 "enchant2",
                 "enscript",
                 "environment-modules",
+                "erofs-utils",
                 "evemu",
                 "execstack",
                 "exempi",

LICENSES-AND-NOTICES/SPECS/data/licenses.json

@@ -0,0 +1,84 @@
+From bc8ec19a4ed741c9b457387dcba2b647e819b16d Mon Sep 17 00:00:00 2001
+From: Gao Xiang <hsiangkao@linux.alibaba.com>
+Date: Wed, 12 Feb 2025 20:36:33 +0800
+Subject: [PATCH 1/7] erofs-utils: mkfs: fragment: gracefully exit if temporary
+ storage is low
+
+Currently, EROFS keeps all fragments into a temporary file for later
+packed inode compression.  However, this could trigger an unexpected
+segfault if temporary storage runs low.
+
+Print a proper error message and gracefully exit.
+
+Closes: https://github.com/erofs/erofs-utils/issues/13
+Link: https://lore.kernel.org/r/20250212123633.40004-1-hsiangkao@linux.alibaba.com
+Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
+---
+ include/erofs/internal.h | 3 ++-
+ lib/fragments.c          | 4 ++++
+ lib/inode.c              | 8 +++-----
+ 3 files changed, 9 insertions(+), 6 deletions(-)
+
+diff --git a/include/erofs/internal.h b/include/erofs/internal.h
+index 5f5bc10..2f71557 100644
+--- a/include/erofs/internal.h
++++ b/include/erofs/internal.h
+@@ -256,7 +256,6 @@ struct erofs_inode {
+ 	unsigned int eof_tailrawsize;
+ 
+ 	union {
+-		void *compressmeta;
+ 		void *chunkindexes;
+ 		struct {
+ 			uint16_t z_advise;
+@@ -274,6 +273,8 @@ struct erofs_inode {
+ #define z_idata_size	idata_size
+ 		};
+ 	};
++	void *compressmeta;
++
+ #ifdef WITH_ANDROID
+ 	uint64_t capabilities;
+ #endif
+diff --git a/lib/fragments.c b/lib/fragments.c
+index 32ac6f5..9633a2b 100644
+--- a/lib/fragments.c
++++ b/lib/fragments.c
+@@ -266,6 +266,10 @@ int z_erofs_pack_file_from_fd(struct erofs_inode *inode, int fd, u32 tofcrc)
+ 	else
+ 		rc = 0;
+ out:
++	if (rc)
++		erofs_err("Failed to record %llu-byte fragment data @ %llu for nid %llu: %d",
++			  inode->fragment_size | 0ULL,
++			  inode->fragmentoff | 0ULL, inode->nid | 0ULL, rc);
+ 	if (memblock)
+ 		munmap(memblock, inode->i_size);
+ 	return rc;
+diff --git a/lib/inode.c b/lib/inode.c
+index 7f97030..743e915 100644
+--- a/lib/inode.c
++++ b/lib/inode.c
+@@ -141,16 +141,14 @@ unsigned int erofs_iput(struct erofs_inode *inode)
+ 		free(d);
+ 
+ 	free(inode->compressmeta);
+-	if (inode->eof_tailraw)
+-		free(inode->eof_tailraw);
++	free(inode->eof_tailraw);
+ 	list_del(&inode->i_hash);
+-	if (inode->i_srcpath)
+-		free(inode->i_srcpath);
++	free(inode->i_srcpath);
+ 
+ 	if (inode->datasource == EROFS_INODE_DATA_SOURCE_DISKBUF) {
+ 		erofs_diskbuf_close(inode->i_diskbuf);
+ 		free(inode->i_diskbuf);
+-	} else if (inode->i_link) {
++	} else {
+ 		free(inode->i_link);
+ 	}
+ 	free(inode);
+-- 
+2.48.1
+

SPECS/erofs-utils/0001-erofs-utils-mkfs-fragment-gracefully-exit-if-tempora.patch

@@ -0,0 +1,47 @@
+From d180c898bd0842eaeb54ca0073e6f444bdae8c40 Mon Sep 17 00:00:00 2001
+From: Gao Xiang <hsiangkao@linux.alibaba.com>
+Date: Thu, 13 Feb 2025 03:15:45 +0800
+Subject: [PATCH 2/7] erofs-utils: mkfs: reduce default dict size for LZMA
+
+Change the default dictionary size to 4 times the pcluster size.
+This will halve the LZMA internal dictionary size to a maximum of
+4MiB per LZMA worker (one worker per CPU in the kernel implementation
+unless the module parameter `lzma_streams=` is given.)
+
+It has a very slight impact on the final image sizes, yet users can
+always use `-zlzma,dictsize=` to specify a custom value.
+
+  _________________________________________________________________________
+ |______ Testset _____|_______ Vanilla _________|_________ After __________| Command Line
+ |  CoreOS            |   741978112 (708 MiB)   |   742293504 (708 MiB)    | -zlzma,6 -Eall-fragments,fragdedupe=inode -C8192
+ |                    |   687501312 (656 MiB)   |   687652864 (656 MiB)    | -zlzma,6 -Eall-fragments,fragdedupe=inode -C131072
+ |____________________|__ 658485248 (628 MiB) __|__ 658698240 (629 MiB)  __| -zlzma,6 -Eall-fragments,fragdedupe=inode -C1048576
+ |  Fedora KIWI       |  2974957568 (2838 MiB)  |  2977394688 (2840 MiB)   | -zlzma,6 -Eall-fragments,fragdedupe=inode -C8192
+ |                    |  2684272640 (2560 MiB)  |  2686750720 (2563 MiB)   | -zlzma,6 -Eall-fragments,fragdedupe=inode -C131072
+ |____________________|_ 2550800384 (2433 MiB) _|_ 2553278464 (2435 MiB) __| -zlzma,6 -Eall-fragments,fragdedupe=inode -C1048576
+ |  AOSP system       |   432562176 (413 MiB)   |   432738304 (413 MiB)    | -zlzma,6 -Eall-fragments,fragdedupe=inode -C8192
+ |  partition         |   393277440 (376 MiB)   |   393351168 (376 MiB)    | -zlzma,6 -Eall-fragments,fragdedupe=inode -C131072
+ |____________________|__ 379260928 (362 MiB) __|__ 379285504 (362 MiB)  __| -zlzma,6 -Eall-fragments,fragdedupe=inode -C1048576
+
+Link: https://lore.kernel.org/r/20250212191545.580768-1-hsiangkao@linux.alibaba.com
+Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
+---
+ lib/compressor_liblzma.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/lib/compressor_liblzma.c b/lib/compressor_liblzma.c
+index d609a28..c4ba585 100644
+--- a/lib/compressor_liblzma.c
++++ b/lib/compressor_liblzma.c
+@@ -75,7 +75,7 @@ static int erofs_compressor_liblzma_setdictsize(struct erofs_compress *c,
+ 			dict_size = erofs_compressor_lzma.default_dictsize;
+ 		} else {
+ 			dict_size = min_t(u32, Z_EROFS_LZMA_MAX_DICT_SIZE,
+-					  cfg.c_mkfs_pclustersize_max << 3);
++					  cfg.c_mkfs_pclustersize_max << 2);
+ 			if (dict_size < 32768)
+ 				dict_size = 32768;
+ 		}
+-- 
+2.48.1
+

SPECS/erofs-utils/0002-erofs-utils-mkfs-reduce-default-dict-size-for-LZMA.patch

@@ -0,0 +1,57 @@
+From 5e05ee09a3bc7331549d8e141dac0ccaf30feda5 Mon Sep 17 00:00:00 2001
+From: Gao Xiang <hsiangkao@linux.alibaba.com>
+Date: Fri, 14 Feb 2025 14:24:05 +0800
+Subject: [PATCH 3/7] erofs-utils: mkfs: add missing `errno = 0` before
+ strto[u]l
+
+strtoull(3) says:
+
+```
+Since strtoul() can legitimately return 0 or ULONG_MAX (ULLONG_MAX for
+strtoull()) on both success and failure, the calling program should set
+errno to 0 before the call, and then determine if an error occurred by
+checking whether errno has a nonzero value after the call.
+```
+
+Otherwise, `--workers=` could exit with `invalid worker number`.
+
+Fixes: 7894301e1a80 ("erofs-utils: mkfs: add `--workers=#` parameter")
+Fixes: 0132cb5ea7d0 ("erofs-utils: mkfs: add `--zfeature-bits` option")
+Fixes: 7550a30c332c ("erofs-utils: enable incremental builds")
+Signed-off-by: Gao Xiang <hsiangkao@linux.alibaba.com>
+Link: https://lore.kernel.org/r/20250214062407.3281416-1-hsiangkao@linux.alibaba.com
+---
+ mkfs/main.c | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/mkfs/main.c b/mkfs/main.c
+index 53d330b..b2396d0 100644
+--- a/mkfs/main.c
++++ b/mkfs/main.c
+@@ -814,6 +814,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
+ 		case 520: {
+ 			unsigned int processors;
+ 
++			errno = 0;
+ 			cfg.c_mt_workers = strtoul(optarg, &endptr, 0);
+ 			if (errno || *endptr != '\0') {
+ 				erofs_err("invalid worker number %s", optarg);
+@@ -828,6 +829,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
+ 		}
+ #endif
+ 		case 521:
++			errno = 0;
+ 			i = strtol(optarg, &endptr, 0);
+ 			if (errno || *endptr != '\0') {
+ 				erofs_err("invalid zfeature bits %s", optarg);
+@@ -844,6 +846,7 @@ static int mkfs_parse_options_cfg(int argc, char *argv[])
+ 			} else if (!strcmp(optarg, "rvsp")) {
+ 				dataimport_mode = EROFS_MKFS_DATA_IMPORT_RVSP;
+ 			} else {
++				errno = 0;
+ 				dataimport_mode = strtol(optarg, &endptr, 0);
+ 				if (errno || *endptr != '\0') {
+ 					erofs_err("invalid --%s=%s",
+-- 
+2.48.1
+