feat: Local components build /w upstream sources and arbitrary files (#439)

* Local components build /w upstream sources

* Extra validation on urls and file ref def

* Added origins back

* Test fix

* Refactored local source retrieval

Author: Antonio Salinas

internal/app/azldev/core/componentbuilder/componentbuilder_test.go

@@ -59,7 +59,7 @@ func setupBuilder(t *testing.T) *componentBuilderTestParams {
 		},
 	)
 
-	sourceManager.EXPECT().FetchFile(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
+	sourceManager.EXPECT().FetchFiles(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 
 	preparer, err := sources.NewPreparer(sourceManager, testEnv.Env.FS(), testEnv.Env, testEnv.Env)
 

internal/app/azldev/core/sources/sourceprep.go

@@ -86,8 +86,16 @@ func NewPreparer(
 func (p *sourcePreparerImpl) PrepareSources(
 	ctx context.Context, component components.Component, outputDir string, applyOverlays bool,
 ) error {
-	// Use the source manager to fetch components
-	err := p.sourceManager.FetchComponent(ctx, component, outputDir)
+	// Download any explicitly configured source files first.
+	// Files defined here take precedence over any sources implicitly defined by the fedora upstream.
+	err := p.sourceManager.FetchFiles(ctx, component, outputDir)
+	if err != nil {
+		return fmt.Errorf("failed to fetch source files for component %#q:\n%w",
+			component.GetName(), err)
+	}
+
+	// Fetch the component sources.
+	err = p.sourceManager.FetchComponent(ctx, component, outputDir)
 	if err != nil {
 		return fmt.Errorf("failed to fetch sources for component %#q:\n%w",
 			component.GetName(), err)

internal/app/azldev/core/sources/sourceprep_test.go

@@ -55,6 +55,7 @@ func TestPrepareSources_Success(t *testing.T) {
 
 	component.EXPECT().GetName().AnyTimes().Return("test-component")
 	component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{})
+	sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil)
 	sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn(
 		func(_ interface{}, _ interface{}, outputDir string) error {
 			// Create the expected spec file.
@@ -87,10 +88,10 @@ func TestPrepareSources_SourceManagerError(t *testing.T) {
 	sourceManager := sourceproviders_test.NewMockSourceManager(ctrl)
 	ctx := testctx.NewCtx()
 
-	expectedErr := errors.New("failed to fetch component")
+	expectedErr := errors.New("failed to fetch files")
 
 	component.EXPECT().GetName().AnyTimes().Return("test-component")
-	sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).Return(expectedErr)
+	sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(expectedErr)
 
 	preparer, err := sources.NewPreparer(sourceManager, ctx.FS(), ctx, ctx)
 	require.NoError(t, err)
@@ -112,6 +113,7 @@ func TestPrepareSources_WritesMacrosFile(t *testing.T) {
 			With: []string{"feature"},
 		},
 	})
+	sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil)
 	sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn(
 		func(_ interface{}, _ interface{}, outputDir string) error {
 			// Create the expected spec file.
@@ -358,6 +360,7 @@ func TestPrepareSources_CheckSkip(t *testing.T) {
 			},
 		},
 	})
+	sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil)
 	sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn(
 		func(_ interface{}, _ interface{}, outputDir string) error {
 			// Create the expected spec file with a %check section.
@@ -414,6 +417,7 @@ func TestPrepareSources_CheckSkipDisabled(t *testing.T) {
 			},
 		},
 	})
+	sourceManager.EXPECT().FetchFiles(gomock.Any(), component, testOutputDir).Return(nil)
 	sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).DoAndReturn(
 		func(_ interface{}, _ interface{}, outputDir string) error {
 			// Create the expected spec file with a %check section.

internal/projectconfig/component.go

@@ -29,35 +29,20 @@ type ComponentReference struct {
 	Version *rpm.Version
 }
 
-// Origin Types.
+// OriginType indicates the type of origin for a source file.
 type OriginType string
 
 const (
-	OriginTypeURL            OriginType = "url"
-	OriginTypeGitHub         OriginType = "github"
-	OriginTypeLocal          OriginType = "local"
-	OriginTypeStorageAccount OriginType = "storageAccount"
-	OriginTypeGenerated      OriginType = "generated"
+	// OriginTypeURI indicates that the source file is fetched from a URI.
+	OriginTypeURI OriginType = "download"
 )
 
-// Origin represents a single source location with its type and configuration.
+// Origin describes where a source file comes from and how to retrieve it.
 type Origin struct {
-	// Type of source: "url", "github", "local", "blobstore"
-	Type OriginType `toml:"Type" json:"type"`
-
-	// Location - interpretation depends on Type:
-	// - "url": the full URL
-	// - "local": relative or absolute path
-	Location string `toml:"Location,omitempty" json:"location,omitempty"`
-
-	// Optional: Script for generated files
-	Script string `toml:"Script,omitempty" json:"script,omitempty"`
-
-	// Optional: Additional arguments specific to the origin type
-	// Examples:
-	// - GitHub: {"release": "v1.2.3", "asset": "foo.tar.gz"}
-	// - Generated: {"arg1": "value1", "arg2": "value2"} (passed as --arg1=value1)
-	Args []string `toml:"Args,omitempty" json:"args,omitempty"`
+	// Type indicates how the source file should be acquired.
+	Type OriginType `toml:"type" json:"type" jsonschema:"required,enum=download,title=Origin type,description=Type of origin for this source file"`
+	// Uri to download the source file from if origin type is 'download'. Ignored for other origin types.
+	Uri string `toml:"uri,omitempty" json:"uri,omitempty" jsonschema:"title=URI,description=URI to download the source file from if origin type is 'download',example=https://example.com/source.tar.gz"`
 }
 
 // SourceFileReference encapsulates a reference to a specific source file artifact.
@@ -66,16 +51,16 @@ type SourceFileReference struct {
 	Component ComponentReference `toml:"-" json:"-"`
 
 	// Name of the source file; must be non-empty.
-	Filename string `toml:"-" json:"-"`
+	Filename string `toml:"filename" json:"filename"`
 
 	// Hash of the source file, expressed as a hex string.
-	Hash string `toml:"Hash,omitempty" json:"hash,omitempty"`
+	Hash string `toml:"hash,omitempty" json:"hash,omitempty"`
 
-	// Type of hash used by `Hash` (if present).
-	HashType fileutils.HashType `toml:"HashType,omitempty" json:"hashType,omitempty"`
+	// Type of hash used by Hash (e.g., "sha256", "sha512").
+	HashType fileutils.HashType `toml:"hash-type,omitempty" json:"hashType,omitempty"`
 
-	// Ordered list of source origins to try in priority order
-	Origins []Origin `toml:"Origins,omitempty" json:"origins,omitempty"`
+	// Type of origin for this source file (e.g., URI, custom).
+	Origin Origin `toml:"origin" json:"origin"`
 }
 
 // Defines a component group. Component groups are logical groupings of components (see [ComponentConfig]).
@@ -128,6 +113,9 @@ type ComponentConfig struct {
 
 	// Configuration for building the component.
 	Build ComponentBuildConfig `toml:"build,omitempty" json:"build,omitempty" table:"-" jsonschema:"title=Build configuration,description=Configuration for building the component"`
+
+	// Source file references for this component.
+	SourceFiles []SourceFileReference `toml:"source-files,omitempty" json:"sourceFiles,omitempty" table:"-" jsonschema:"title=Source files,description=Source files to download for this component"`
 }
 
 // Mutates the component config, updating it with overrides present in other.
@@ -151,6 +139,7 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi
 		SourceConfigFile: c.SourceConfigFile,
 		Spec:             deep.MustCopy(c.Spec),
 		Build:            deep.MustCopy(c.Build),
+		SourceFiles:      deep.MustCopy(c.SourceFiles),
 	}
 
 	// Fix up paths.

internal/providers/sourceproviders/fedorasource/fedorasource.go

@@ -162,11 +162,24 @@ func (g *FedoraSourceDownloaderImpl) downloadAndVerifySources(
 	repoDir string,
 ) error {
 	for _, sourceFile := range sourceFiles {
+		destFilePath := filepath.Join(repoDir, sourceFile.fileName)
+
+		exists, err := fileutils.Exists(g.fileSystem, destFilePath)
+		if err != nil {
+			return fmt.Errorf("failed to check if file exists at %#q:\n%w", destFilePath, err)
+		}
+
+		if exists {
+			slog.Debug("File already exists, skipping download", "fileName", sourceFile.fileName, "path", destFilePath)
+
+			continue
+		}
+
 		slog.Info("Downloading source file...",
 			"fileName", sourceFile.fileName,
-			"URI", sourceFile.uri)
-
-		destFilePath := filepath.Join(repoDir, sourceFile.fileName)
+			"URI", sourceFile.uri,
+			"destPath", destFilePath,
+		)
 
 		if err := g.downloader.Download(ctx, sourceFile.uri, destFilePath); err != nil {
 			return fmt.Errorf("failed to download from %#q to %#q:\n%w", sourceFile.uri, destFilePath, err)

internal/providers/sourceproviders/fedorasourceprovider.go

@@ -16,6 +16,7 @@ import (
 	"github.com/gim-home/azldev-preview/internal/global/opctx"
 	"github.com/gim-home/azldev-preview/internal/projectconfig"
 	"github.com/gim-home/azldev-preview/internal/providers/sourceproviders/fedorasource"
+	"github.com/gim-home/azldev-preview/internal/utils/fileutils"
 	"github.com/gim-home/azldev-preview/internal/utils/git"
 )
 
@@ -27,6 +28,7 @@ type DistroResolver func(distroRef projectconfig.DistroReference) (
 // FedoraSourcesProviderImpl implements ComponentSourceProvider for Git repositories.
 type FedoraSourcesProviderImpl struct {
 	fs             opctx.FS
+	dryRunnable    opctx.DryRunnable
 	gitProvider    git.GitProvider
 	downloader     fedorasource.FedoraSourceDownloader
 	distroResolver DistroResolver
@@ -36,6 +38,7 @@ var _ ComponentSourceProvider = (*FedoraSourcesProviderImpl)(nil)
 
 func NewFedoraSourcesProviderImpl(
 	fs opctx.FS,
+	dryRunnable opctx.DryRunnable,
 	gitProvider git.GitProvider,
 	downloader fedorasource.FedoraSourceDownloader,
 	distroResolver DistroResolver,
@@ -44,6 +47,10 @@ func NewFedoraSourcesProviderImpl(
 		return nil, errors.New("filesystem cannot be nil")
 	}
 
+	if dryRunnable == nil {
+		return nil, errors.New("dryRunnable cannot be nil")
+	}
+
 	if gitProvider == nil {
 		return nil, errors.New("git provider cannot be nil")
 	}
@@ -58,6 +65,7 @@ func NewFedoraSourcesProviderImpl(
 
 	return &FedoraSourcesProviderImpl{
 		fs:             fs,
+		dryRunnable:    dryRunnable,
 		gitProvider:    gitProvider,
 		downloader:     downloader,
 		distroResolver: distroResolver,
@@ -66,7 +74,7 @@ func NewFedoraSourcesProviderImpl(
 
 func (g *FedoraSourcesProviderImpl) GetComponent(
 	ctx context.Context, component components.Component, destDirPath string,
-) error {
+) (err error) {
 	componentName := component.GetName()
 	if componentName == "" {
 		return errors.New("component name cannot be empty")
@@ -97,45 +105,80 @@ func (g *FedoraSourcesProviderImpl) GetComponent(
 		"upstreamComponent", upstreamNameToUse,
 		"branch", distroGitBranch)
 
-	// Clone the repository directly to the destination
+	// Clone to a temp directory first, then copy files to destination.
+	tempDir, err := fileutils.MkdirTempInTempDir(g.fs, "azldev-clone-")
+	if err != nil {
+		return fmt.Errorf("failed to create temp directory for clone:\n%w", err)
+	}
+
+	defer fileutils.RemoveAllAndUpdateErrorIfNil(g.fs, tempDir, &err)
+
+	// Clone the repository to temp directory
 	err = g.gitProvider.Clone(
 		ctx,
 		gitRepoURL,
-		destDirPath,
+		tempDir,
 		git.WithGitBranch(distroGitBranch))
 	if err != nil {
 		return fmt.Errorf("failed to clone git repository %#q:\n%w", gitRepoURL, err)
 	}
 
 	// Checkout the appropriate commit based on component/distro config
-	err = g.checkoutTargetCommit(ctx, effectiveDistroRef, destDirPath)
+	err = g.checkoutTargetCommit(ctx, effectiveDistroRef, tempDir)
 	if err != nil {
 		return fmt.Errorf("failed to checkout target commit:\n%w", err)
 	}
 
-	// Delete the .git directory so it's not packaged up.
-	err = g.fs.RemoveAll(filepath.Join(destDirPath, ".git"))
+	// Delete the .git directory so it's not copied to destination.
+	err = g.fs.RemoveAll(filepath.Join(tempDir, ".git"))
 	if err != nil {
 		return fmt.Errorf("failed to remove .git directory from cloned repository at %#q:\n%w",
-			destDirPath, err)
+			tempDir, err)
 	}
 
-	// Extract sources from repo (downloads lookaside files into the repo)
-	err = g.downloader.ExtractSourcesFromRepo(ctx, destDirPath, upstreamNameToUse, lookasideBaseURI)
+	// Extract sources from repo (downloads lookaside files into the temp dir)
+	err = g.downloader.ExtractSourcesFromRepo(ctx, tempDir, upstreamNameToUse, lookasideBaseURI)
 	if err != nil {
 		return fmt.Errorf("failed to extract sources from git repository:\n%w", err)
 	}
 
-	// If the upstream name differs from the component name, we will need to rename the spec.
-	if upstreamNameToUse != componentName {
-		downloadedSpecPath := filepath.Join(destDirPath, upstreamNameToUse+".spec")
-		desiredSpecPath := filepath.Join(destDirPath, componentName+".spec")
+	// If the upstream name differs from the component name, rename the spec in temp dir.
+	err = g.renameSpecIfNeeded(tempDir, upstreamNameToUse, componentName)
+	if err != nil {
+		return err
+	}
 
-		err := g.fs.Rename(downloadedSpecPath, desiredSpecPath)
-		if err != nil {
-			return fmt.Errorf("failed to rename fetched spec file from %#q to %#q:\n%w",
-				downloadedSpecPath, desiredSpecPath, err)
-		}
+	// Copy files from temp dir to destination, skipping files that already exist.
+	// This preserves any files downloaded by FetchFiles, giving them precedence.
+	copyOptions := fileutils.CopyDirOptions{
+		CopyFileOptions: fileutils.CopyFileOptions{
+			PreserveFileMode: true,
+		},
+		FileFilter: fileutils.SkipExistingFiles,
+	}
+
+	err = fileutils.CopyDirRecursive(g.dryRunnable, g.fs, tempDir, destDirPath, copyOptions)
+	if err != nil {
+		return fmt.Errorf("failed to copy files to destination:\n%w", err)
+	}
+
+	return nil
+}
+
+// renameSpecIfNeeded renames the spec file in the given directory if the upstream name
+// differs from the desired component name.
+func (g *FedoraSourcesProviderImpl) renameSpecIfNeeded(dir, upstreamName, componentName string) error {
+	if upstreamName == componentName {
+		return nil
+	}
+
+	downloadedSpecPath := filepath.Join(dir, upstreamName+".spec")
+	desiredSpecPath := filepath.Join(dir, componentName+".spec")
+
+	err := g.fs.Rename(downloadedSpecPath, desiredSpecPath)
+	if err != nil {
+		return fmt.Errorf("failed to rename fetched spec file from %#q to %#q:\n%w",
+			downloadedSpecPath, desiredSpecPath, err)
 	}
 
 	return nil