feat: enable multiple config files on command line (#456)

Co-authored-by: reuben olinsky <reubeno@users.noreply.github.com>
Co-authored-by: Daniel McIlvaney <Daniel.McIlvaney@microsoft.com>

Author: reubeno

internal/app/azldev/app.go

@@ -46,6 +46,7 @@ type App struct {
 	dryRun               bool
 	reportFormat         ReportFormat
 	disableDefaultConfig bool
+	configFiles          []string
 	colorMode            ColorMode
 
 	// Root command for the CLI.
@@ -152,15 +153,14 @@ func NewApp(fsFactory opctx.FileSystemFactory, osEnvFactory opctx.OSEnvFactory)
 		"disable default configuration")
 	app.cmd.PersistentFlags().StringVarP(&app.explicitProjectDir, "project", "C", "",
 		"path to Azure Linux project")
+	app.cmd.PersistentFlags().StringArrayVar(&app.configFiles, "config-file", nil,
+		"additional TOML config file(s) to merge (may be repeated)")
 	app.cmd.PersistentFlags().BoolVarP(&app.dryRun, "dry-run", "n", false, "dry run only (do not take action)")
 	app.cmd.PersistentFlags().VarP(&app.reportFormat, "output-format", "O",
 		"output format {csv, json, markdown, table}")
 	app.cmd.PersistentFlags().Var(&app.colorMode, "color",
 		"output colorization mode {always, auto, never}")
 
-	// Manually specifying a config file should only be used if you really know what you're doing.
-	_ = app.cmd.PersistentFlags().MarkHidden("config-file")
-
 	return app
 }
 
@@ -365,6 +365,7 @@ func (a *App) initializeProjectConfig(envOptions *EnvOptions, earlyTempDirPath s
 	projectDir, config, err := a.findAndLoadConfig(
 		envOptions.DryRunnable,
 		earlyTempDirPath,
+		a.configFiles,
 	)
 
 	if errors.Is(err, projectconfig.ErrConfigFileNotFound) {
@@ -438,6 +439,11 @@ func (a *App) handParseConfigFlags(args []string) {
 			}
 		case "--no-default-config":
 			a.disableDefaultConfig = true
+		case "--config-file":
+			index++
+			if index < len(args) {
+				a.configFiles = append(a.configFiles, args[index])
+			}
 		case "-n", "--dry-run":
 			a.dryRun = true
 		case "--color":
@@ -446,21 +452,28 @@ func (a *App) handParseConfigFlags(args []string) {
 				_ = a.colorMode.Set(args[index])
 			}
 		default:
-			switch {
-			case strings.HasPrefix(arg, "-C"):
-				a.explicitProjectDir = strings.TrimPrefix(arg, "-C")
-			case strings.HasPrefix(arg, "--project="):
-				a.explicitProjectDir = strings.TrimPrefix(arg, "--project=")
-			case strings.HasPrefix(arg, "--color="):
-				_ = a.colorMode.Set(strings.TrimPrefix(arg, "--color="))
-			}
+			a.handParsePrefixedFlags(arg)
 		}
 	}
 }
 
+// handParsePrefixedFlags handles flag variants with = assignment syntax (e.g., --project=value).
+func (a *App) handParsePrefixedFlags(arg string) {
+	switch {
+	case strings.HasPrefix(arg, "-C"):
+		a.explicitProjectDir = strings.TrimPrefix(arg, "-C")
+	case strings.HasPrefix(arg, "--project="):
+		a.explicitProjectDir = strings.TrimPrefix(arg, "--project=")
+	case strings.HasPrefix(arg, "--color="):
+		_ = a.colorMode.Set(strings.TrimPrefix(arg, "--color="))
+	case strings.HasPrefix(arg, "--config-file="):
+		a.configFiles = append(a.configFiles, strings.TrimPrefix(arg, "--config-file="))
+	}
+}
+
 // Initializes the configuration for the azldev CLI. This includes finding the project.
 // loading configuration, etc.
-func (a *App) findAndLoadConfig(dryRunnable opctx.DryRunnable, tempDirPath string) (
+func (a *App) findAndLoadConfig(dryRunnable opctx.DryRunnable, tempDirPath string, extraConfigFiles []string) (
 	projectDir string, config *projectconfig.ProjectConfig, err error,
 ) {
 	// If no explicit project dir was specified, then fall back to the current working directory.
@@ -480,6 +493,7 @@ func (a *App) findAndLoadConfig(dryRunnable opctx.DryRunnable, tempDirPath strin
 		referenceDir,
 		a.disableDefaultConfig,
 		tempDirPath,
+		extraConfigFiles,
 	)
 	if err != nil {
 		return projectDir, config, fmt.Errorf("failed to load project configuration:\n%w", err)

internal/app/azldev/cmds/image/customize.go

@@ -31,7 +31,7 @@ func NewImageCustomizeCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&options.imageTag, "image-tag", "", "",
 		"Container tag for the MCR base Azure Linux image to be downloaded and customized."+
 			" Cannot be specified with --image-file")
-	cmd.Flags().StringVarP(&options.configFile, "config-file", "", "",
+	cmd.Flags().StringVarP(&options.imageConfigFile, "image-config", "", "",
 		"Path of the image customization config file")
 	cmd.Flags().StringVar(&options.outputImageFormat, "output-image-format", "",
 		"Format of output image ("+getImageCustomizerImageFormatsString()+")")
@@ -56,7 +56,7 @@ func NewImageCustomizeCmd() *cobra.Command {
 	// Customizer, we are requiring them in azldev so that we can deduce which
 	// folders to mount into the container.
 	// See: https://dev.azure.com/mariner-org/polar/_workitems/edit/15282
-	_ = cmd.MarkFlagRequired("config-file")
+	_ = cmd.MarkFlagRequired("image-config")
 	_ = cmd.MarkFlagRequired("output-path")
 
 	cmd.MarkFlagsMutuallyExclusive("image-file", "image-tag")

internal/app/azldev/cmds/image/imageutils.go

@@ -33,7 +33,7 @@ const (
 type imageCustomizerOptions struct {
 	imageFile                string
 	imageTag                 string
-	configFile               string
+	imageConfigFile          string
 	outputImageFormat        string
 	outputPath               string
 	rpmSources               []string
@@ -58,7 +58,7 @@ func getImageCustomizerImageFormatsString() string {
 func buildDockerArgs(
 	options *imageCustomizerOptions, buildDir string, logsDir string, rpmSources []rpmSourceInfo,
 ) []string {
-	configDir := path.Dir(options.configFile)
+	configDir := path.Dir(options.imageConfigFile)
 	outputPathDir := path.Dir(options.outputPath)
 
 	args := []string{
@@ -141,7 +141,7 @@ func buildImageCustomizerArgs(
 
 	args = append(args, []string{
 		"--build-dir", buildDir,
-		"--config-file", options.configFile,
+		"--config-file", options.imageConfigFile,
 		"--output-image-format", options.outputImageFormat,
 		"--output-path", options.outputPath,
 		"--log-level", logLevel,

internal/app/azldev/cmds/image/injectfiles.go

@@ -28,7 +28,7 @@ func NewImageInjectFilesCmd() *cobra.Command {
 
 	cmd.Flags().StringVarP(&options.imageFile, "image-file", "", "",
 		"Path of the base Azure Linux image which the customization will be applied to")
-	cmd.Flags().StringVarP(&options.configFile, "config-file", "", "",
+	cmd.Flags().StringVarP(&options.imageConfigFile, "image-config", "", "",
 		"Path of the image customization config file")
 	cmd.Flags().StringVar(&options.outputImageFormat, "output-image-format", "",
 		"Format of output image ("+getImageCustomizerImageFormatsString()+")")
@@ -47,7 +47,7 @@ func NewImageInjectFilesCmd() *cobra.Command {
 	// folders to mount into the container.
 	// See: https://dev.azure.com/mariner-org/polar/_workitems/edit/15282
 	_ = cmd.MarkFlagRequired("image-file")
-	_ = cmd.MarkFlagRequired("config-file")
+	_ = cmd.MarkFlagRequired("image-config")
 	_ = cmd.MarkFlagRequired("output-path")
 
 	return cmd

internal/projectconfig/config.go

@@ -22,6 +22,7 @@ func LoadProjectConfig(
 	referenceDir string,
 	disableDefaultConfig bool,
 	tempDirPath string,
+	extraConfigFilePaths []string,
 ) (projectDir string, config *ProjectConfig, err error) {
 	// Look for project root and azldev.toml file.
 	projectDir, projectFilePath, err := FindProjectRootAndConfigFile(fs, referenceDir)
@@ -47,9 +48,13 @@ func LoadProjectConfig(
 		configFilePaths = append(configFilePaths, defaultConfigFilePath)
 	}
 
-	// Load the project config file last.
+	// Load the project config file next.
 	configFilePaths = append(configFilePaths, projectFilePath)
 
+	// Append any extra config files specified by the user (e.g., via --config-file flags).
+	// These are loaded last, so they can override/merge with settings from the project config.
+	configFilePaths = append(configFilePaths, extraConfigFilePaths...)
+
 	// Actually load and process the config file (and any linked config files it references).
 	//
 	// NOTE: We don't wrap the error returned back here (if one is returned) because we already have

internal/projectconfig/distro.go

@@ -4,8 +4,10 @@
 package projectconfig
 
 import (
+	"fmt"
 	"runtime"
 
+	"dario.cat/mergo"
 	"github.com/brunoga/deep"
 )
 
@@ -80,15 +82,31 @@ type DistroVersionDefinition struct {
 	MockConfigPathAarch64 string `toml:"mock-config-aarch64,omitempty" json:"mockConfigAarch64,omitempty" validate:"omitempty,filepath" jsonschema:"title=Mock config file,description=Path to the aarch64 mock config file for this version"`
 }
 
+// MergeUpdatesFrom mutates the distro definition, updating it with overrides present in other.
+// Uses [mergo.WithOverride] without WithAppendSlice so that slice fields like
+// [DistroDefinition.PackageRepositories] are replaced, not appended. This supports the primary
+// use case of swapping between package sources via --config-file overrides.
+//
+// For map fields like [DistroDefinition.Versions], mergo replaces the entire value for a
+// matching key rather than doing a field-level merge within the value struct.
+func (d *DistroDefinition) MergeUpdatesFrom(other *DistroDefinition) error {
+	err := mergo.Merge(d, other, mergo.WithOverride)
+	if err != nil {
+		return fmt.Errorf("failed to merge distro definition:\n%w", err)
+	}
+
+	return nil
+}
+
 // Returns a copy of the distro definition with relative file paths converted to absolute
 // file paths (relative to referenceDir, not the current working directory).
-func (d DistroDefinition) WithAbsolutePaths(referenceDir string) DistroDefinition {
+func (d *DistroDefinition) WithAbsolutePaths(referenceDir string) DistroDefinition {
 	// First deep-copy ourselves.
 	//
 	// NOTE: We use the panicking MustCopy() because copying should only fail if the input *type*
 	// is invalid. Since we're always using the same type, we never expect to see a runtime error
 	// here.
-	result := deep.MustCopy(d)
+	result := deep.MustCopy(*d)
 
 	for name := range result.Versions {
 		result.Versions[name] = result.Versions[name].WithAbsolutePaths(referenceDir)
@@ -101,13 +119,13 @@ func (d DistroDefinition) WithAbsolutePaths(referenceDir string) DistroDefinitio
 	return result
 }
 
-func (d DistroDefinition) WithResolvedConfigs() DistroDefinition {
+func (d *DistroDefinition) WithResolvedConfigs() DistroDefinition {
 	// First deep-copy ourselves.
 	//
 	// NOTE: We use the panicking MustCopy() because copying should only fail if the input *type*
 	// is invalid. Since we're always using the same type, we never expect to see a runtime error
 	// here.
-	result := deep.MustCopy(d)
+	result := deep.MustCopy(*d)
 
 	for name := range result.Versions {
 		result.Versions[name] = result.Versions[name].WithResolvedConfigs()

internal/projectconfig/distro_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/gim-home/azldev-preview/internal/projectconfig"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestDistroReferenceStringer(t *testing.T) {
@@ -38,3 +39,88 @@ func TestDistroReferenceStringer(t *testing.T) {
 		assert.Equal(t, "(default) (default)", s.String())
 	})
 }
+
+func TestDistroDefinition_MergeUpdatesFrom(t *testing.T) {
+	t.Run("override description", func(t *testing.T) {
+		base := &projectconfig.DistroDefinition{
+			Description:    "original",
+			DefaultVersion: "1.0",
+		}
+
+		other := &projectconfig.DistroDefinition{
+			Description: "updated",
+		}
+
+		err := base.MergeUpdatesFrom(other)
+		require.NoError(t, err)
+		assert.Equal(t, "updated", base.Description)
+		assert.Equal(t, "1.0", base.DefaultVersion)
+	})
+
+	t.Run("add new version", func(t *testing.T) {
+		base := &projectconfig.DistroDefinition{
+			Description: "distro",
+			Versions: map[string]projectconfig.DistroVersionDefinition{
+				"1.0": {Description: "v1", ReleaseVer: "1.0"},
+			},
+		}
+
+		other := &projectconfig.DistroDefinition{
+			Versions: map[string]projectconfig.DistroVersionDefinition{
+				"2.0": {Description: "v2", ReleaseVer: "2.0"},
+			},
+		}
+
+		err := base.MergeUpdatesFrom(other)
+		require.NoError(t, err)
+		assert.Equal(t, "distro", base.Description)
+		assert.Len(t, base.Versions, 2)
+		assert.Equal(t, "v1", base.Versions["1.0"].Description)
+		assert.Equal(t, "v2", base.Versions["2.0"].Description)
+	})
+
+	t.Run("override existing version field", func(t *testing.T) {
+		base := &projectconfig.DistroDefinition{
+			Versions: map[string]projectconfig.DistroVersionDefinition{
+				"1.0": {Description: "old desc", ReleaseVer: "1.0", DistGitBranch: "main"},
+			},
+		}
+
+		other := &projectconfig.DistroDefinition{
+			Versions: map[string]projectconfig.DistroVersionDefinition{
+				"1.0": {Description: "new desc"},
+			},
+		}
+
+		err := base.MergeUpdatesFrom(other)
+		require.NoError(t, err)
+
+		version := base.Versions["1.0"]
+		assert.Equal(t, "new desc", version.Description)
+		// mergo.WithOverride replaces the entire map value for the same key,
+		// so non-specified fields will be zeroed out. This is intentional:
+		// override configs are expected to fully redefine any version they touch.
+		assert.Empty(t, version.ReleaseVer)
+		assert.Empty(t, version.DistGitBranch)
+	})
+
+	t.Run("replace package repositories", func(t *testing.T) {
+		base := &projectconfig.DistroDefinition{
+			PackageRepositories: []projectconfig.PackageRepository{
+				{BaseURI: "https://old-repo.example.com"},
+				{BaseURI: "https://another-old-repo.example.com"},
+			},
+		}
+
+		other := &projectconfig.DistroDefinition{
+			PackageRepositories: []projectconfig.PackageRepository{
+				{BaseURI: "https://new-repo.example.com"},
+			},
+		}
+
+		err := base.MergeUpdatesFrom(other)
+		require.NoError(t, err)
+		require.Len(t, base.PackageRepositories, 1)
+		assert.Equal(t, "https://new-repo.example.com", base.PackageRepositories[0].BaseURI)
+	})
+}

internal/projectconfig/loader.go

@@ -25,8 +25,6 @@ var (
 	ErrDuplicateComponentGroups = errors.New("duplicate component group")
 	// ErrDuplicateImages is returned when duplicate conflicting image definitions are found.
 	ErrDuplicateImages = errors.New("duplicate image")
-	// ErrDuplicateDistros is returned when duplicate conflicting distro definitions are found.
-	ErrDuplicateDistros = errors.New("duplicate distro")
 )
 
 // Loads and resolves the project configuration files located at the given path. Referenced include files
@@ -95,11 +93,19 @@ func mergeConfigFile(resolvedCfg *ProjectConfig, loadedCfg *ConfigFile) error {
 
 	// Merge in distros.
 	for distroName, distro := range loadedCfg.Distros {
-		if _, ok := resolvedCfg.Distros[distroName]; ok {
-			return fmt.Errorf("%w: %s", ErrDuplicateDistros, distroName)
-		}
+		resolvedDistro := distro.WithResolvedConfigs()
+		resolvedDistro = resolvedDistro.WithAbsolutePaths(loadedCfg.dir)
 
-		resolvedCfg.Distros[distroName] = distro.WithResolvedConfigs().WithAbsolutePaths(loadedCfg.dir)
+		if existing, ok := resolvedCfg.Distros[distroName]; ok {
+			err := existing.MergeUpdatesFrom(&resolvedDistro)
+			if err != nil {
+				return fmt.Errorf("failed to merge distro %#q:\n%w", distroName, err)
+			}
+
+			resolvedCfg.Distros[distroName] = existing
+		} else {
+			resolvedCfg.Distros[distroName] = resolvedDistro
+		}
 	}
 
 	// Merge in component groups.