feat: Source Retrieval refactor and Source Manager Implementation (#357)

* Refactor of prepare sources and implementation of source manager.

* Source manager tests

* Fixed failing source manager test

* Updated scenario test and local contents provider

Author: Antonio Salinas

internal/app/azldev/cmds/component/build.go

@@ -16,7 +16,6 @@ import (
 	"github.com/microsoft/azldev/internal/buildenv"
 	"github.com/microsoft/azldev/internal/orchestration/artifacts"
 	"github.com/microsoft/azldev/internal/orchestration/orchestrator"
-	"github.com/microsoft/azldev/internal/projectconfig"
 	"github.com/microsoft/azldev/internal/providers/sourceproviders"
 	"github.com/microsoft/azldev/internal/utils/defers"
 	"github.com/samber/lo"
@@ -149,15 +148,10 @@ func BuildComponent(
 	workDirFactory *workdir.Factory,
 	options *ComponentBuildOptions,
 ) (results ComponentBuildResults, err error) {
-	var rpmContentsProvider sourceproviders.RPMContentsProvider
-
-	// If the component has external sources, then we build an RPM contents provider;
-	// otherwise, we pass nil to the preparer.
-	if component.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeUpstream {
-		rpmContentsProvider, err = buildRPMContentsProvider(env, component)
-		if err != nil {
-			return ComponentBuildResults{}, fmt.Errorf("failed to create RPM contents provider:\n%w", err)
-		}
+	sourceManager, err := sourceproviders.NewSourceManager(env)
+	if err != nil {
+		return ComponentBuildResults{},
+			fmt.Errorf("failed to setup source retrieval manager for component %q:\n%w", component.GetName(), err)
 	}
 
 	var buildEnv buildenv.RPMAwareBuildEnv
@@ -177,7 +171,12 @@ func BuildComponent(
 		return nil
 	}, &err)
 
-	sourcePreparer := sources.NewPreparer(env, env, env.FS(), rpmContentsProvider)
+	sourcePreparer, err := sources.NewPreparer(env, sourceManager)
+	if err != nil {
+		return ComponentBuildResults{},
+			fmt.Errorf("failed to create source preparer for component %q:\n%w", component.GetName(), err)
+	}
+
 	builder := componentbuilder.New(env, env.FS(), env, sourcePreparer, buildEnv, workDirFactory)
 
 	return buildComponentUsingBuilder(env, orchestrator, component, builder,

internal/app/azldev/cmds/component/preparesources.go

@@ -10,12 +10,7 @@ import (
 	"github.com/microsoft/azldev/internal/app/azldev"
 	"github.com/microsoft/azldev/internal/app/azldev/core/components"
 	"github.com/microsoft/azldev/internal/app/azldev/core/sources"
-	"github.com/microsoft/azldev/internal/projectconfig"
-	"github.com/microsoft/azldev/internal/providers/rpmprovider"
 	"github.com/microsoft/azldev/internal/providers/sourceproviders"
-	"github.com/microsoft/azldev/internal/rpm"
-	"github.com/microsoft/azldev/internal/utils/downloader"
-	"github.com/samber/lo"
 	"github.com/spf13/cobra"
 )
 
@@ -78,93 +73,23 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er
 	event := env.StartEvent("Preparing sources", "component", component.GetName())
 	defer event.End()
 
-	var rpmContentsProvider sourceproviders.RPMContentsProvider
+	var sourceManager sourceproviders.SourceManager
 
-	// If the component has external sources, then we build an RPM contents provider;
-	// otherwise, we pass nil to the preparer.
-	if component.GetConfig().Spec.SourceType == projectconfig.SpecSourceTypeUpstream {
-		rpmContentsProvider, err = buildRPMContentsProvider(env, component)
-		if err != nil {
-			return fmt.Errorf("failed to create RPM contents provider:\n%w", err)
-		}
-	}
-
-	preparer := sources.NewPreparer(env, env, env.FS(), rpmContentsProvider)
-
-	err = preparer.PrepareSources(env, component, options.OutputDir)
-	if err != nil {
-		return fmt.Errorf("failed to prepare sources for component %q:\n%w", component.GetName(), err)
-	}
-
-	return nil
-}
-
-func buildRPMContentsProvider(
-	env *azldev.Env, component components.Component,
-) (*sourceproviders.RPMContentsProviderImpl, error) {
-	repoQuerierOptions, err := composeRepoQuerierOptionsForComponent(env, component)
-	if err != nil {
-		return nil, fmt.Errorf("failed to compose repo querying options:\n%w", err)
-	}
-
-	rpmExtractor, err := rpm.NewRPMExtractorImpl(env.FS())
+	// Create source manager to handle all source fetching, both local and upstream.
+	sourceManager, err = sourceproviders.NewSourceManager(env)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create RPM extractor:\n%w", err)
+		return fmt.Errorf("failed to create source manager:\n%w", err)
 	}
 
-	httpDownloader, err := downloader.NewHTTPDownloader(env, env, env.FS())
+	preparer, err := sources.NewPreparer(env, sourceManager)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create HTTP downloader:\n%w", err)
+		return fmt.Errorf("failed to create source preparer:\n%w", err)
 	}
 
-	repoQuerier, err := rpm.NewRQRepoQuerier(env, repoQuerierOptions...)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create RPM repo querier:\n%w", err)
-	}
-
-	rpmProvider, err := rpmprovider.NewRPMProviderImpl(env, httpDownloader, repoQuerier)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create RPM provider:\n%w", err)
-	}
-
-	rpmContentsProvider, err := sourceproviders.NewRPMContentsProviderImpl(rpmExtractor, rpmProvider)
+	err = preparer.PrepareSources(env, component, options.OutputDir)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create RPM contents provider:\n%w", err)
-	}
-
-	return rpmContentsProvider, nil
-}
-
-func composeRepoQuerierOptionsForComponent(
-	env *azldev.Env, component components.Component,
-) ([]rpm.RepoQuerierOption, error) {
-	specSource := component.GetConfig().Spec
-
-	options := []rpm.RepoQuerierOption{}
-
-	switch specSource.SourceType {
-	case projectconfig.SpecSourceTypeUnspecified:
-		break
-	case projectconfig.SpecSourceTypeLocal:
-		break
-	case projectconfig.SpecSourceTypeUpstream:
-		distro, distroVer, err := env.ResolveDistroRef(specSource.UpstreamDistro)
-		if err != nil {
-			return nil, fmt.Errorf("failed to resolve upstream distro for component %q:\n%w", component.GetName(), err)
-		}
-
-		repoURLs := lo.Map(distro.PackageRepositories, func(repo projectconfig.PackageRepository, _ int) string {
-			return repo.BaseURI
-		})
-
-		options = append(options, rpm.WithBaseURLs(repoURLs...))
-
-		if distroVer.ReleaseVer != "" {
-			options = append(options, rpm.WithReleaseVer(distroVer.ReleaseVer))
-		}
-	default:
-		break
+		return fmt.Errorf("failed to prepare sources for component %q:\n%w", component.GetName(), err)
 	}
 
-	return options, nil
+	return nil
 }

internal/app/azldev/core/componentbuilder/componentbuilder_test.go

@@ -47,14 +47,16 @@ func setupBuilder(t *testing.T) *componentBuilderTestParams {
 	workDirFactory, err := workdir.NewFactory(testEnv.Env.FS(), testEnv.Env.WorkDir(), testEnv.Env.ConstructionTime())
 	require.NoError(t, err)
 
-	rpmContentsProvider := sourceproviders_test.NewNoOpMockRPMContentsProvider(ctrl)
+	sourceManager := sourceproviders_test.NewNoOpMockSourceManager(ctrl)
+
+	preparer, err := sources.NewPreparer(testEnv.Env, sourceManager)
+
+	require.NoError(t, err)
 
 	params := &componentBuilderTestParams{
-		ctrl:    ctrl,
-		testEnv: testEnv,
-		sourcePreparer: sources.NewPreparer(
-			testEnv.DryRunnable, testEnv.Env, testEnv.Env.FS(), rpmContentsProvider,
-		),
+		ctrl:           ctrl,
+		testEnv:        testEnv,
+		sourcePreparer: preparer,
 		workDirFactory: workDirFactory,
 	}