Add REPO_SNAPSHOT_TIME to the toolkit for package and image build. (#10369)

Author: sameluch

SPECS/tdnf/tdnf.spec

@@ -4,7 +4,7 @@
 Summary:        dnf equivalent using C libs
 Name:           tdnf
 Version:        3.5.8
-Release:        2%{?dist}
+Release:        3%{?dist}
 License:        LGPLv2.1 AND GPLv2
 Vendor:         Microsoft Corporation
 Distribution:   Azure Linux
@@ -237,7 +237,10 @@ fi
 /%{_lib}/systemd/system/tdnf*
 
 %changelog
-* Thu Aug 29 2024 Sam Meluch <sammeluch@microsoft.com> - 3.5.8-2
+* Fri Sep 20 2024 Sam Meluch <sammeluch@microsoft.com> - 3.5.8-3
+- Fix an issue with snapshottime config option
+
+* Wed Sep 18 2024 Sam Meluch <sammeluch@microsoft.com> - 3.5.8-2
 - Add virtual repo snapshot exclude repos
 
 * Mon Sep 09 2024 Sam Meluch <sammeluch@microsoft.com> - 3.5.8-1

SPECS/tdnf/virtual-repo-snapshot.patch

@@ -1,4 +1,4 @@
-From ef2e53818f93005d1dc2d96eaa36519069fa979d Mon Sep 17 00:00:00 2001
+From 7af976b8d0977f32f9903220d8b44378e72d4336 Mon Sep 17 00:00:00 2001
 From: Sam Meluch <sammeluch@microsoft.com>
 Date: Tue, 30 Apr 2024 13:56:44 -0700
 Subject: [PATCH] Add virtual repo snapshot feature to tdnf
@@ -21,7 +21,7 @@ Subject: [PATCH] Add virtual repo snapshot feature to tdnf
  14 files changed, 958 insertions(+), 17 deletions(-)
 
 diff --git a/client/config.c b/client/config.c
-index 8ddcc7a..805fff1 100644
+index 8ddcc7a..ef16467 100644
 --- a/client/config.c
 +++ b/client/config.c
 @@ -85,6 +85,7 @@ TDNFReadConfig(
@@ -38,7 +38,7 @@ index 8ddcc7a..805fff1 100644
          }
 +        else if (strcmp(cn->name, TDNF_CONF_KEY_SNAPSHOT_TIME) == 0)
 +        {
-+            pConf->pszSnapshotTime = cn->value; //assumes your system's time_t is typedef long
++            pConf->pszSnapshotTime = strdup(cn->value); //assumes your system's time_t is typedef long
 +        }
          else if (strcmp(cn->name, TDNF_CONF_KEY_CLEAN_REQ_ON_REMOVE) == 0)
          {

toolkit/Makefile

@@ -45,6 +45,8 @@ RUN_CHECK                            ?= n
 USE_PREVIEW_REPO                     ?= n
 DISABLE_UPSTREAM_REPOS               ?= n
 DISABLE_DEFAULT_REPOS                ?= n
+##help:var:REPO_SNAPSHOT_TIME:<posix_time>=Posix time to be used as a snapshot for remote repositories when fetching packages. Example: REPO_SNAPSHOT_TIME="1724119509".
+REPO_SNAPSHOT_TIME                   ?=
 TOOLCHAIN_CONTAINER_ARCHIVE          ?=
 TOOLCHAIN_ARCHIVE                    ?=
 TOOLCHAIN_SOURCES_ARCHIVE            ?=

toolkit/resources/imageconfigs/additionalfiles/iso_initrd/root/runliveinstaller

@@ -143,8 +143,13 @@ cd /installer
 # Turn off echoing while the installer runs to stop sensitive data from rendering in the TTY session.
 stty -echo
 
+# add call to static time file to populate liveinstaller option here
+if [[ -f "$CONFIG_ROOT/repo-snapshot-time.txt" ]]; then
+REPO_TIME=$(cat /"$CONFIG_ROOT/repo-snapshot-time.txt")
+fi
+
 ./liveinstaller --base-dir $CONFIG_ROOT --imager /installer/imager --input $UNATTENDED_CONFIG_FILE --template-config $CONFIG_ROOT/attended_config.json \
-                --build-dir $PWD --log-file=/installer/log.txt
+                --build-dir $PWD --log-file=/installer/log.txt --repo-snapshot-time="$REPO_TIME"
 installerExitCode=$?
 
 # Consume any buffered stdin to prevent it from being passed to any future programs,

toolkit/resources/manifests/package/pkggen_core_aarch64.txt

@@ -211,10 +211,10 @@ lua-5.4.6-1.azl3.aarch64.rpm
 lua-libs-5.4.6-1.azl3.aarch64.rpm
 azurelinux-rpm-macros-3.0-7.azl3.noarch.rpm
 azurelinux-check-macros-3.0-7.azl3.noarch.rpm
-tdnf-3.5.8-2.azl3.aarch64.rpm
-tdnf-cli-libs-3.5.8-2.azl3.aarch64.rpm
-tdnf-devel-3.5.8-2.azl3.aarch64.rpm
-tdnf-plugin-repogpgcheck-3.5.8-2.azl3.aarch64.rpm
+tdnf-3.5.8-3.azl3.aarch64.rpm
+tdnf-cli-libs-3.5.8-3.azl3.aarch64.rpm
+tdnf-devel-3.5.8-3.azl3.aarch64.rpm
+tdnf-plugin-repogpgcheck-3.5.8-3.azl3.aarch64.rpm
 libassuan-2.5.6-1.azl3.aarch64.rpm
 libassuan-devel-2.5.6-1.azl3.aarch64.rpm
 libgpg-error-1.47-1.azl3.aarch64.rpm

toolkit/resources/manifests/package/pkggen_core_x86_64.txt

@@ -211,10 +211,10 @@ lua-5.4.6-1.azl3.x86_64.rpm
 lua-libs-5.4.6-1.azl3.x86_64.rpm
 azurelinux-rpm-macros-3.0-7.azl3.noarch.rpm
 azurelinux-check-macros-3.0-7.azl3.noarch.rpm
-tdnf-3.5.8-2.azl3.x86_64.rpm
-tdnf-cli-libs-3.5.8-2.azl3.x86_64.rpm
-tdnf-devel-3.5.8-2.azl3.x86_64.rpm
-tdnf-plugin-repogpgcheck-3.5.8-2.azl3.x86_64.rpm
+tdnf-3.5.8-3.azl3.x86_64.rpm
+tdnf-cli-libs-3.5.8-3.azl3.x86_64.rpm
+tdnf-devel-3.5.8-3.azl3.x86_64.rpm
+tdnf-plugin-repogpgcheck-3.5.8-3.azl3.x86_64.rpm
 libassuan-2.5.6-1.azl3.x86_64.rpm
 libassuan-devel-2.5.6-1.azl3.x86_64.rpm
 libgpg-error-1.47-1.azl3.x86_64.rpm

toolkit/resources/manifests/package/toolchain_aarch64.txt

@@ -584,14 +584,14 @@ systemd-bootstrap-libs-250.3-17.azl3.aarch64.rpm
 systemd-bootstrap-rpm-macros-250.3-17.azl3.noarch.rpm
 tar-1.35-1.azl3.aarch64.rpm
 tar-debuginfo-1.35-1.azl3.aarch64.rpm
-tdnf-3.5.8-2.azl3.aarch64.rpm
-tdnf-autoupdate-3.5.8-2.azl3.aarch64.rpm
-tdnf-cli-libs-3.5.8-2.azl3.aarch64.rpm
-tdnf-debuginfo-3.5.8-2.azl3.aarch64.rpm
-tdnf-devel-3.5.8-2.azl3.aarch64.rpm
-tdnf-plugin-metalink-3.5.8-2.azl3.aarch64.rpm
-tdnf-plugin-repogpgcheck-3.5.8-2.azl3.aarch64.rpm
-tdnf-python-3.5.8-2.azl3.aarch64.rpm
+tdnf-3.5.8-3.azl3.aarch64.rpm
+tdnf-autoupdate-3.5.8-3.azl3.aarch64.rpm
+tdnf-cli-libs-3.5.8-3.azl3.aarch64.rpm
+tdnf-debuginfo-3.5.8-3.azl3.aarch64.rpm
+tdnf-devel-3.5.8-3.azl3.aarch64.rpm
+tdnf-plugin-metalink-3.5.8-3.azl3.aarch64.rpm
+tdnf-plugin-repogpgcheck-3.5.8-3.azl3.aarch64.rpm
+tdnf-python-3.5.8-3.azl3.aarch64.rpm
 texinfo-7.0.3-1.azl3.aarch64.rpm
 texinfo-debuginfo-7.0.3-1.azl3.aarch64.rpm
 unzip-6.0-20.azl3.aarch64.rpm

toolkit/resources/manifests/package/toolchain_x86_64.txt

@@ -590,14 +590,14 @@ systemd-bootstrap-libs-250.3-17.azl3.x86_64.rpm
 systemd-bootstrap-rpm-macros-250.3-17.azl3.noarch.rpm
 tar-1.35-1.azl3.x86_64.rpm
 tar-debuginfo-1.35-1.azl3.x86_64.rpm
-tdnf-3.5.8-2.azl3.x86_64.rpm
-tdnf-autoupdate-3.5.8-2.azl3.x86_64.rpm
-tdnf-cli-libs-3.5.8-2.azl3.x86_64.rpm
-tdnf-debuginfo-3.5.8-2.azl3.x86_64.rpm
-tdnf-devel-3.5.8-2.azl3.x86_64.rpm
-tdnf-plugin-metalink-3.5.8-2.azl3.x86_64.rpm
-tdnf-plugin-repogpgcheck-3.5.8-2.azl3.x86_64.rpm
-tdnf-python-3.5.8-2.azl3.x86_64.rpm
+tdnf-3.5.8-3.azl3.x86_64.rpm
+tdnf-autoupdate-3.5.8-3.azl3.x86_64.rpm
+tdnf-cli-libs-3.5.8-3.azl3.x86_64.rpm
+tdnf-debuginfo-3.5.8-3.azl3.x86_64.rpm
+tdnf-devel-3.5.8-3.azl3.x86_64.rpm
+tdnf-plugin-metalink-3.5.8-3.azl3.x86_64.rpm
+tdnf-plugin-repogpgcheck-3.5.8-3.azl3.x86_64.rpm
+tdnf-python-3.5.8-3.azl3.x86_64.rpm
 texinfo-7.0.3-1.azl3.x86_64.rpm
 texinfo-debuginfo-7.0.3-1.azl3.x86_64.rpm
 unzip-6.0-20.azl3.x86_64.rpm

toolkit/scripts/imggen.mk

@@ -75,6 +75,13 @@ clean-imagegen:
 	$(SCRIPTS_DIR)/safeunmount.sh "$(IMAGEGEN_DIR)" && \
 	rm -rf $(IMAGEGEN_DIR)
 
+# We need to clear the rpm package cache if we have a snapshot time. The filenames will all be
+# the same, but the actual .rpm files may be fundamentally different.
+$(STATUS_FLAGS_DIR)/imagegen_cleanup.flag: $(depend_REPO_SNAPSHOT_TIME)
+	@echo "REPO_SNAPSHOT_TIME has changed, sanitizing rpm cache"
+	rm -rf $(local_and_external_rpm_cache)
+	touch $@
+
 ##help:target:fetch-image-packages=Locate and download all packages required for an image build.
 fetch-image-packages: $(image_package_cache_summary)
 
@@ -113,7 +120,11 @@ ifeq ($(USE_PREVIEW_REPO),y)
 imagepkgfetcher_extra_flags += --use-preview-repo
 endif
 
-$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(toolchain_rpms) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(RPMS_DIR) $(imggen_rpms)
+ifneq ($(REPO_SNAPSHOT_TIME),)
+imagepkgfetcher_extra_flags += --repo-snapshot-time=$(REPO_SNAPSHOT_TIME)
+endif
+
+$(image_package_cache_summary): $(go-imagepkgfetcher) $(chroot_worker) $(toolchain_rpms) $(imggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(RPMS_DIR) $(imggen_rpms) $(depend_REPO_SNAPSHOT_TIME) $(STATUS_FLAGS_DIR)/imagegen_cleanup.flag
 	$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
 	$(go-imagepkgfetcher) \
 		--input=$(CONFIG_FILE) \
@@ -146,7 +157,7 @@ $(imager_disk_output_dir): $(STATUS_FLAGS_DIR)/imager_disk_output.flag
 	@touch $@
 	@echo Finished updating $@
 
-$(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_summary) $(license_results_file_img) $(imggen_local_repo) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(assets_files)
+$(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_summary) $(license_results_file_img) $(imggen_local_repo) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(assets_files) $(depend_REPO_SNAPSHOT_TIME)
 	$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
 	mkdir -p $(imager_disk_output_dir) && \
 	rm -rf $(imager_disk_output_dir)/* && \
@@ -159,6 +170,7 @@ $(STATUS_FLAGS_DIR)/imager_disk_output.flag: $(go-imager) $(image_package_cache_
 		--log-color=$(LOG_COLOR) \
 		--local-repo $(local_and_external_rpm_cache) \
 		--tdnf-worker $(chroot_worker) \
+		--repo-snapshot-time=$(REPO_SNAPSHOT_TIME) \
 		--repo-file=$(imggen_local_repo) \
 		--output-image-contents=$(image_package_manifest) \
 		--assets $(assets_dir) \
@@ -198,7 +210,7 @@ image: $(imager_disk_output_dir) $(imager_disk_output_files) $(go-roast) $(depen
 		$(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \
 		--timestamp-file=$(TIMESTAMP_DIR)/roast.jsonl
 
-$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(chroot_worker) $(graph_file) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config)
+$(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(chroot_worker) $(graph_file) $(depend_REPO_LIST) $(REPO_LIST) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(depend_REPO_SNAPSHOT_TIME) $(STATUS_FLAGS_DIR)/imagegen_cleanup.flag
 	$(if $(CONFIG_FILE),,$(error Must set CONFIG_FILE=))
 	$(go-imagepkgfetcher) \
 		--input=$(CONFIG_FILE) \
@@ -229,7 +241,7 @@ $(image_external_package_cache_summary): $(cached_file) $(go-imagepkgfetcher) $(
 
 # We need to ensure that initrd_img recursive build will never run concurrently with another build component, so add all ISO prereqs as
 # order-only-prerequisites to initrd_img
-iso_deps = $(go-isomaker) $(go-imager) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(image_package_cache_summary) $(license_results_file_img)
+iso_deps = $(go-isomaker) $(go-imager) $(depend_CONFIG_FILE) $(CONFIG_FILE) $(validate-config) $(image_package_cache_summary) $(license_results_file_img) $(depend_REPO_SNAPSHOT_TIME)
 # The initrd bundles these files into the image, we should rebuild it if they change
 initrd_bundled_files = $(go-liveinstaller) $(go-imager) $(assets_files) $(initrd_assets_files) $(imggen_local_repo)
 
@@ -250,6 +262,7 @@ iso: $(initrd_img) $(iso_deps)
 		--input $(CONFIG_FILE) \
 		--release-version $(RELEASE_VERSION) \
 		--resources $(RESOURCES_DIR) \
+		--repo-snapshot-time=$(REPO_SNAPSHOT_TIME) \
 		--iso-repo $(local_and_external_rpm_cache) \
 		--log-level=$(LOG_LEVEL) \
 		--log-file=$(LOGS_DIR)/imggen/isomaker.log \

toolkit/scripts/pkggen.mk

@@ -123,7 +123,7 @@ endif
 
 # Convert the dependency information in the json file into a graph structure
 # We require all the toolchain RPMs to be available here to help resolve unfixable cyclic dependencies
-$(graph_file): $(specs_file) $(go-grapher) $(toolchain_rpms) $(TOOLCHAIN_MANIFEST) $(pkggen_local_repo) $(graphpkgfetcher_cloned_repo) $(chroot_worker) $(depend_REPO_LIST) $(REPO_LIST)
+$(graph_file): $(specs_file) $(go-grapher) $(toolchain_rpms) $(TOOLCHAIN_MANIFEST) $(pkggen_local_repo) $(graphpkgfetcher_cloned_repo) $(chroot_worker) $(depend_REPO_LIST) $(REPO_LIST) $(depend_REPO_SNAPSHOT_TIME)
 	$(go-grapher) \
 		--input $(specs_file) \
 		$(logging_command) \
@@ -146,6 +146,7 @@ $(graph_file): $(specs_file) $(go-grapher) $(toolchain_rpms) $(TOOLCHAIN_MANIFES
 		--tls-key=$(TLS_KEY) \
 		--tmp-dir=$(grapher_working_dir) \
 		--tdnf-worker=$(chroot_worker) \
+		--repo-snapshot-time=$(REPO_SNAPSHOT_TIME) \
 		$(foreach repo, $(pkggen_local_repo) $(graphpkgfetcher_cloned_repo) $(REPO_LIST), --repo-file=$(repo))
 
 # We want to detect changes in the RPM cache, but we are not responsible for directly rebuilding any missing files.
@@ -192,12 +193,16 @@ graphpkgfetcher_extra_flags += $(if $(CONFIG_FILE),--base-dir="$(CONFIG_BASE_DIR
 $(cached_file): $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_TEST_RUN_LIST) $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST)
 endif
 
+ifneq ($(REPO_SNAPSHOT_TIME),)
+graphpkgfetcher_extra_flags += --repo-snapshot-time=$(REPO_SNAPSHOT_TIME)
+endif
+
 ifeq ($(PRECACHE),y)
 # Use highly parallel downlader to fully hydrate the cache before trying to use the package manager to download packages
 $(cached_file): $(STATUS_FLAGS_DIR)/precache.flag
 endif
 
-$(cached_file): $(graph_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(cached_remote_rpms) $(TOOLCHAIN_MANIFEST) $(toolchain_rpms) $(depend_EXTRA_BUILD_LAYERS)
+$(cached_file): $(graph_file) $(go-graphpkgfetcher) $(chroot_worker) $(pkggen_local_repo) $(depend_REPO_LIST) $(REPO_LIST) $(cached_remote_rpms) $(TOOLCHAIN_MANIFEST) $(toolchain_rpms) $(depend_EXTRA_BUILD_LAYERS) $(depend_REPO_SNAPSHOT_TIME) $(STATUS_FLAGS_DIR)/build_packages_cache_cleanup.flag
 	mkdir -p $(remote_rpms_cache_dir) && \
 	$(go-graphpkgfetcher) \
 		--input=$(graph_file) \
@@ -260,6 +265,13 @@ clean-compress-rpms:
 clean-compress-srpms:
 	rm -rf $(srpms_archive)
 
+# We need to clear the rpm package cache if we have a snapshot time. The filenames will all be
+# the same, but the actual .rpm files may be fundamentally different.
+$(STATUS_FLAGS_DIR)/build_packages_cache_cleanup.flag: $(depend_REPO_SNAPSHOT_TIME)
+	@echo "REPO_SNAPSHOT_TIME has changed, sanitizing rpm cache"
+	rm -rf $(remote_rpms_cache_dir)
+	touch $@
+
 ifeq ($(REBUILD_PACKAGES),y)
 $(RPMS_DIR): $(STATUS_FLAGS_DIR)/build-rpms.flag
 	@touch $@