Add support for kernel flavor versioning (#15578)

Author: mbykhovtsev-ms

toolkit/Makefile

@@ -33,6 +33,8 @@ TEST_RUN_LIST           ?=
 TEST_RERUN_LIST         ?=
 ##help:var:TEST_IGNORE_LIST:<spec_list>=List of space-separated spec folders to ignore for package tests. Must not overlap with "TEST_RERUN_LIST", may overlap with "TEST_RUN_LIST". Example: TEST_IGNORE_LIST="acl".
 TEST_IGNORE_LIST        ?=
+##help:var:EXTRA_MACROS_FILES:<file1> <file2>=Space separated list of additional files whose contents will be appended to the versions macro file used during package builds. Example: EXTRA_MACROS_FILES="file1 file2".
+EXTRA_MACROS_FILES ?=
 
 ######## SET INCREMENTAL BUILD FLAGS ########
 
@@ -168,7 +170,7 @@ endif
 VALIDATE_IMAGE_GPG ?= n
 
 # Default GPG keys for package GPG validation, used with VALIDATE_TOOLCHAIN_GPG and VALIDATE_IMAGE_GPG
-default_gpg_keys := $(wildcard $(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY) $(wildcard $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY)
+default_gpg_keys := $(strip $(wildcard $(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY) $(wildcard $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY))
 TOOLCHAIN_GPG_VALIDATION_KEYS ?= $(default_gpg_keys)
 IMAGE_GPG_VALIDATION_KEYS ?= $(default_gpg_keys)
 

toolkit/docs/building/building.md

@@ -814,6 +814,7 @@ To reproduce an ISO build, run the same make invocation as before, but set:
 | TEST_RUN_LIST                 |                                                                                                        | Explicit list of packages to test. The package test will be skipped if the build system thinks it is already up-to-date. The argument accepts both spec and package names. Example: for `python-werkzeug.spec`, which builds the `python3-werkzeug` package both `python-werkzeug` and `python3-werkzeug` are correct.
 | TEST_RERUN_LIST               |                                                                                                        | Always test these package, even if it its corresponding package is up-to-date. The argument accepts both spec and package names. Example: for `python-werkzeug.spec`, which builds the `python3-werkzeug` package both `python-werkzeug` and `python3-werkzeug` are correct.
 | TEST_IGNORE_LIST              |                                                                                                        | Ignore testing these packages. Ignoring and forcing the same test re-run is invalid and will fail the build. The argument accepts both spec and package names. Example: for `python-werkzeug.spec`, which builds the `python3-werkzeug` package both `python-werkzeug` and `python3-werkzeug` are correct.
+| EXTRA_MACROS_FILES              |                                                                                                        | Space separated list of additional `<macros.name>` containing additional RPM macros, which will be available to the build. Used to resolve versions of kernel Out of Tree Module packages.
 
 ---
 

toolkit/docs/how_it_works/3_package_building.md

@@ -117,6 +117,10 @@ The files can be ingested into the `graphviz` tools to visualize them, although
 dot -Tpng -o visualized.png < graph.dot
 ```
 
+### Dynamic versioning
+We have a versionsprocessor tool that iterates over all Specs and writes their release and versions into a macro file in a format of
+`azl_<package_name>_release`, `azl_<package_name>_version`, note that the `<package_name>` needs any `-` replaced with `_` due to macros not allowing `-`.
+
 ### Stage 1: Grapher
 
 The `grapher` tool reads the `specs.json` file and converts it into an acyclic directed graph. Inter-package dependencies are represented by directed edges in the graph.

toolkit/scripts/pkggen.mk

@@ -33,6 +33,7 @@ validate-pkggen-config = $(STATUS_FLAGS_DIR)/validate-image-config-pkggen.flag
 
 # Outputs
 specs_file               = $(PKGBUILD_DIR)/specs.json
+rel_versions_macro_file  = $(PKGBUILD_DIR)/macros.releaseversions
 graph_file               = $(PKGBUILD_DIR)/graph.dot
 cached_file              = $(PKGBUILD_DIR)/cached_graph.dot
 preprocessed_file        = $(PKGBUILD_DIR)/preprocessed_graph.dot
@@ -50,6 +51,8 @@ $(call create_folder,$(rpmbuilding_logs_dir))
 parse-specs: $(specs_file)
 ##help:target:graph-cache=Resolve package dependencies and cache the results.
 graph-cache: $(cached_file)
+##help:target:generate-versions-macros-file=Generate a macros file containing version and release macros for all specs.
+generate-versions-macros-file: $(rel_versions_macro_file)
 ##help:target:graph=Create the initial package build graph.
 workplan graph: $(graph_file)
 graph-preprocessed: $(preprocessed_file)
@@ -89,10 +92,29 @@ analyze-built-graph: $(go-graphanalytics)
 		exit 1; \
 	fi
 
+# Parse all SPECS in $(SPECS_DIR) and generate a release versions macros file containing macros of spec file versions and release.
+$(rel_versions_macro_file): $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $(go-versionsprocessor)
+	$(go-versionsprocessor) \
+		--dir $(SPECS_DIR) \
+		--dist-tag $(DIST_TAG) \
+		$(logging_command) \
+		--build-dir $(parse_working_dir) \
+		--worker-tar $(chroot_worker) \
+		--cpu-prof-file=$(PROFILE_DIR)/versionsprocessor.cpu.pprof \
+		--mem-prof-file=$(PROFILE_DIR)/versionsprocessor.mem.pprof \
+		--trace-file=$(PROFILE_DIR)/versionsprocessor.trace \
+		$(if $(EXTRA_MACROS_FILES),$(foreach file,$(EXTRA_MACROS_FILES),--extra-macros-file=$(file))) \
+		$(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \
+		$(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \
+		$(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \
+		--timestamp-file=$(TIMESTAMP_DIR)/versionsprocessor.jsonl \
+		$(if $(TARGET_ARCH),--target-arch="$(TARGET_ARCH)") \
+		--output $@
+
 # Parse specs in $(SPECS_DIR) and generate a specs.json file encoding all dependency information
 # We look at the same pack list as the srpmpacker tool via the target $(SRPM_PACK_LIST) if it is set.
 # We only parse the spec files we will actually pack.
-$(specs_file): $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $(go-specreader) $(depend_SPECS_DIR) $(depend_SRPM_PACK_LIST) $(depend_RUN_CHECK)
+$(specs_file): $(rel_versions_macro_file) $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $(go-specreader) $(depend_SPECS_DIR) $(depend_SRPM_PACK_LIST) $(depend_RUN_CHECK)
 	$(go-specreader) \
 		--dir $(SPECS_DIR) \
 		$(if $(SRPM_PACK_LIST),--spec-list="$(SRPM_PACK_LIST)") \
@@ -103,6 +125,7 @@ $(specs_file): $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $
 		--toolchain-rpms-dir="$(TOOLCHAIN_RPMS_DIR)" \
 		--dist-tag $(DIST_TAG) \
 		--worker-tar $(chroot_worker) \
+		--versions-macro-file $(rel_versions_macro_file) \
 		$(if $(filter y,$(RUN_CHECK)),--run-check) \
 		$(logging_command) \
 		--cpu-prof-file=$(PROFILE_DIR)/specreader.cpu.pprof \
@@ -297,7 +320,7 @@ $(RPMS_DIR):
 	@touch $@
 endif
 
-$(STATUS_FLAGS_DIR)/build-rpms.flag: $(no_repo_acl) $(preprocessed_file) $(chroot_worker) $(go-scheduler) $(go-pkgworker) $(depend_STOP_ON_PKG_FAIL) $(CONFIG_FILE) $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_MAX_CASCADING_REBUILDS) $(depend_TEST_RUN_LIST) $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(pkggen_rpms) $(srpms) $(BUILD_SRPMS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(depend_LICENSE_CHECK_MODE)
+$(STATUS_FLAGS_DIR)/build-rpms.flag: $(rel_versions_macro_file) $(no_repo_acl) $(preprocessed_file) $(chroot_worker) $(go-scheduler) $(go-pkgworker) $(depend_STOP_ON_PKG_FAIL) $(CONFIG_FILE) $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_MAX_CASCADING_REBUILDS) $(depend_TEST_RUN_LIST) $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(pkggen_rpms) $(srpms) $(BUILD_SRPMS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(depend_LICENSE_CHECK_MODE)
 	$(go-scheduler) \
 		--input="$(preprocessed_file)" \
 		--output="$(built_file)" \
@@ -315,6 +338,7 @@ $(STATUS_FLAGS_DIR)/build-rpms.flag: $(no_repo_acl) $(preprocessed_file) $(chroo
 		--distro-release-version="$(RELEASE_VERSION)" \
 		--distro-build-number="$(BUILD_NUMBER)" \
 		--rpmmacros-file="$(TOOLCHAIN_MANIFESTS_DIR)/macros.override" \
+		--versions-macro-file="$(rel_versions_macro_file)" \
 		--build-attempts="$$(($(PACKAGE_BUILD_RETRIES)+1))" \
 		--check-attempts="$$(($(CHECK_BUILD_RETRIES)+1))" \
 		$(if $(MAX_CASCADING_REBUILDS),--max-cascading-rebuilds="$(MAX_CASCADING_REBUILDS)") \

toolkit/scripts/srpm_pack.mk

@@ -13,9 +13,9 @@
 # update  - Check signatures and updating any mismatches in the signatures file
 SRPM_FILE_SIGNATURE_HANDLING ?= enforce
 
-SRPM_BUILD_CHROOT_DIR = $(BUILD_DIR)/SRPM_packaging
-SRPM_BUILD_LOGS_DIR = $(LOGS_DIR)/pkggen/srpms
-
+SRPM_BUILD_CHROOT_DIR   = $(BUILD_DIR)/SRPM_packaging
+SRPM_BUILD_LOGS_DIR     = $(LOGS_DIR)/pkggen/srpms
+rel_versions_macro_file = $(PKGBUILD_DIR)/macros.releaseversions
 # Configure the list of packages we want to process into SRPMs
 # Strip any whitespace from user input and reasign using override so we can compare it with the empty string
 override SRPM_PACK_LIST := $(strip $(SRPM_PACK_LIST))
@@ -77,7 +77,7 @@ $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(local_
 $(STATUS_FLAGS_DIR)/build_toolchain_srpms.flag: $(STATUS_FLAGS_DIR)/build_srpms.flag
 	@touch $@
 else
-$(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(go-srpmpacker) $(depend_SRPM_PACK_LIST) $(local_spec_sources)
+$(STATUS_FLAGS_DIR)/build_srpms.flag: $(rel_versions_macro_file) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(go-srpmpacker) $(depend_SRPM_PACK_LIST) $(local_spec_sources)
 	GODEBUG=netdns=go $(go-srpmpacker) \
 		--dir=$(SPECS_DIR) \
 		--output-dir=$(BUILD_SRPMS_DIR) \
@@ -88,6 +88,7 @@ $(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(local_specs) $(local_sp
 		--tls-cert=$(TLS_CERT) \
 		--tls-key=$(TLS_KEY) \
 		--build-dir=$(SRPM_BUILD_CHROOT_DIR) \
+		--versions-macro-file=$(rel_versions_macro_file) \
 		--signature-handling=$(SRPM_FILE_SIGNATURE_HANDLING) \
 		--worker-tar=$(chroot_worker) \
 		$(if $(filter y,$(RUN_CHECK)),--run-check) \

toolkit/scripts/toolkit.mk

@@ -25,6 +25,7 @@ rpms_snapshot_dir_name = rpms_snapshots
 rpms_snapshot_build_dir = $(BUILD_DIR)/$(rpms_snapshot_dir_name)
 rpms_snapshot_logs_path = $(LOGS_DIR)/$(rpms_snapshot_dir_name)/rpms_snapshot.log
 rpms_snapshot_per_specs = $(rpms_snapshot_build_dir)/$(specs_dir_name)_$(rpms_snapshot_name)
+rel_versions_macro_file = $(PKGBUILD_DIR)/macros.releaseversions
 
 valid_arch_spec_names_build_dir = $(BUILD_DIR)/valid_arch_spec_names
 valid_arch_spec_names           = $(valid_arch_spec_names_build_dir)/valid_arch_spec_names.txt
@@ -75,14 +76,15 @@ $(toolkit_archive_versioned_compressed): $(toolkit_archive) $(rpms_snapshot) $(d
 	tar --update -f $(toolkit_archive_versioned) -C $(toolkit_build_dir) $(toolkit_release_file_relative_path) $(toolkit_rpms_snapshot_file_relative_path) && \
 	$(ARCHIVE_TOOL) --best -c $(toolkit_archive_versioned) > $(toolkit_archive_versioned_compressed)
 
-$(toolkit_archive): $(go_tool_targets) $(mariner_repos_files) $(toolkit_component_extra_files) $(toolkit_root_files)
+$(toolkit_archive): $(go_tool_targets) $(mariner_repos_files) $(toolkit_component_extra_files) $(toolkit_root_files) $(rel_versions_macro_file)
 	rm -rf $(toolkit_prep_dir) && \
 	mkdir -p $(toolkit_prep_dir) && \
 	mkdir -p $(toolkit_repos_dir) && \
 	mkdir -p $(toolkit_tools_dir) && \
 	cp -r $(toolkit_root_files) $(toolkit_prep_dir) && \
 	cp $(mariner_repos_files) $(toolkit_repos_dir) && \
 	cp $(toolkit_component_extra_files) $(toolkit_prep_dir) && \
+	cp $(rel_versions_macro_file) $(toolkit_prep_dir) && \
 	cp $(go_tool_targets) $(toolkit_tools_dir) && \
 	rm -rf $(toolkit_prep_dir)/out && \
 	tar -cvp -f $(toolkit_archive) -C $(dir $(toolkit_prep_dir)) $(notdir $(toolkit_prep_dir))
@@ -93,12 +95,13 @@ rpms-snapshot: $(rpms_snapshot)
 $(rpms_snapshot): $(rpms_snapshot_per_specs) $(depend_SPECS_DIR)
 	cp $(rpms_snapshot_per_specs) $(rpms_snapshot)
 
-$(rpms_snapshot_per_specs): $(go-rpmssnapshot) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR)
+$(rpms_snapshot_per_specs): $(go-rpmssnapshot) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(rel_versions_macro_file)
 	@mkdir -p "$(rpms_snapshot_build_dir)"
 	$(go-rpmssnapshot) \
 		--input="$(SPECS_DIR)" \
 		--output="$(rpms_snapshot_per_specs)" \
 		--build-dir="$(rpms_snapshot_build_dir)" \
+		--versions-macro-file $(rel_versions_macro_file) \
 		--dist-tag=$(DIST_TAG) \
 		--worker-tar="$(chroot_worker)" \
 		--log-level=$(LOG_LEVEL) \
@@ -114,13 +117,14 @@ run-specarchchecker: $(valid_arch_spec_names)
 	@cat $(valid_arch_spec_names) && echo "" # File doesn't have a newline at the end, so add one via echo.
 	@echo "Valid arch spec names generated under '$(valid_arch_spec_names)'."
 
-$(valid_arch_spec_names): $(go-specarchchecker) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST)
+$(valid_arch_spec_names): $(go-specarchchecker) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(rel_versions_macro_file)
 	$(go-specarchchecker) \
 		--input="$(SPECS_DIR)" \
 		--output="$@" \
 		--packages="$(PACKAGE_BUILD_LIST)" \
 		--rebuild-packages="$(PACKAGE_REBUILD_LIST)" \
 		--build-dir="$(valid_arch_spec_names_build_dir)" \
+		--versions-macro-file $(rel_versions_macro_file) \
 		$(if $(filter y,$(RUN_CHECK)),--test-only) \
 		--dist-tag=$(DIST_TAG) \
 		--worker-tar="$(chroot_worker)" \

toolkit/scripts/tools.mk

@@ -62,6 +62,7 @@ go_tool_list = \
 	scheduler \
 	specarchchecker \
 	specreader \
+	versionsprocessor \
 	srpmpacker \
 	validatechroot \
 

toolkit/tools/internal/safechroot/chrootinterface.go

@@ -8,4 +8,5 @@ type ChrootInterface interface {
 	Run(toRun func() error) error
 	UnsafeRun(toRun func() error) error
 	AddFiles(filesToCopy ...FileToCopy) error
+	AddRPMMacrosFile(macrosFilePath string) error
 }

toolkit/tools/internal/safechroot/dummychroot.go

@@ -23,3 +23,7 @@ func (d *DummyChroot) UnsafeRun(toRun func() error) (err error) {
 func (d *DummyChroot) AddFiles(filesToCopy ...FileToCopy) (err error) {
 	return AddFilesToDestination(d.RootDir(), filesToCopy...)
 }
+
+func (d *DummyChroot) AddRPMMacrosFile(macrosFilePath string) error {
+	return AddRPMMacrosFile(d, macrosFilePath)
+}

toolkit/tools/internal/safechroot/safechroot.go

@@ -14,9 +14,11 @@ import (
 	"time"
 
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/buildpipeline"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/directory"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/file"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/logger"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/retry"
+	"github.com/microsoft/azurelinux/toolkit/tools/internal/rpm"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/shell"
 	"github.com/microsoft/azurelinux/toolkit/tools/internal/systemdependency"
 
@@ -160,6 +162,60 @@ func NewOverlayMountPoint(chrootDir, source, target, lowerDir, upperDir, workDir
 	return
 }
 
+func (c *Chroot) AddRPMMacrosFile(macrosFilePath string) error {
+	return AddRPMMacrosFile(c, macrosFilePath)
+}
+
+func AddRPMMacrosFile(c ChrootInterface, macrosFilePath string) (err error) {
+	var macroDir string
+
+	doGetMacroDir := func() error {
+		var macroErr error
+
+		macroDir, macroErr = rpm.GetMacroDir()
+		if macroErr != nil {
+			logger.Log.Errorf("Failed to get RPM macro directory: %s", macroErr)
+			return macroErr
+		}
+
+		return macroErr
+	}
+
+	c.Run(doGetMacroDir)
+
+	// Destination path inside the chroot (same path as on the host).
+	macrosDestDir := filepath.Join(c.RootDir(), macroDir)
+	macrosDestFile := filepath.Join(macrosDestDir, filepath.Base(macrosFilePath))
+
+	exists, existsErr := file.PathExists(macrosDestFile)
+	if existsErr != nil {
+		logger.Log.Errorf("Failed to check if macros file exists (%s): %s", macrosDestFile, existsErr)
+		return existsErr
+	}
+
+	if exists {
+		logger.Log.Warningf("Macros file (%s) already exists, skipping copy", macrosDestFile)
+		return nil
+	}
+
+	// Ensure destination directory exists and copy the file.
+	mkdirErr := directory.EnsureDirExists(macrosDestDir)
+	if mkdirErr != nil {
+		logger.Log.Errorf("Failed to create macros directory inside chroot (%s): %s", macrosDestDir, mkdirErr)
+		return mkdirErr
+	}
+
+	copyErr := file.Copy(macrosFilePath, macrosDestFile)
+	if copyErr != nil {
+		logger.Log.Errorf("Failed to copy release version macros file into chroot (%s -> %s): %s", macrosFilePath, macrosDestFile, copyErr)
+		return copyErr
+	}
+
+	logger.Log.Infof("Copied release version macros file into chroot (%s -> %s)", macrosFilePath, macrosDestFile)
+
+	return
+}
+
 // GetSource gets the source device of the mount.
 func (m *MountPoint) GetSource() string {
 	return m.source
@@ -209,7 +265,7 @@ func NewChroot(rootDir string, isExistingDir bool) *Chroot {
 // This call will block until the chroot initializes successfully.
 // Only one Chroot will initialize at a given time.
 func (c *Chroot) Initialize(tarPath string, extraDirectories []string, extraMountPoints []*MountPoint,
-	includeDefaultMounts bool,
+	includeDefaultMounts bool, releaseVersionMacrosFile ...string,
 ) (err error) {
 	// On failed initialization, cleanup all chroot files
 	const leaveChrootOnDisk = false
@@ -319,6 +375,16 @@ func (c *Chroot) Initialize(tarPath string, extraDirectories []string, extraMoun
 		activeChroots = append(activeChroots, c)
 	}
 
+	// If a release version macros file is provided, copy it into the default RPM macros directory
+	// inside the chroot so rpmspec/rpmbuild pick it up automatically.
+	if len(releaseVersionMacrosFile) > 0 && releaseVersionMacrosFile[0] != "" {
+		err = c.AddRPMMacrosFile(releaseVersionMacrosFile[0])
+		if err != nil {
+			err = fmt.Errorf("failed to add release version macros file to chroot:\n%w", err)
+			return
+		}
+	}
+
 	return
 }