feat: Prepares sources by cloning Fedora package repo (#368)

* Git contents provider and git util

* Git provider Unit tests and minor refactoring

* Added testuser to docker group

* Added tests and a few refactors

* Revert dockerfile

* Refactored fedora source provider/tests

* Small refactors to sourcemanager & fedoraSourceProvider

* Added hash validation and download header for lookaside anti-bot detection

* Refactored some hash functions

* Changed hash.go to make use of Hashtype

* Hash util change

* Added back Accept header comment

Author: Antonio Salinas

defaultconfigs/content/distros/fedora/fedora.distro.toml

@@ -1,6 +1,8 @@
 [distros.fedora]
 description = "Fedora Linux"
 default-version = "42"
+dist-git-base-uri = "https://src.fedoraproject.org/rpms/$pkg.git"
+lookaside-base-uri = "https://src.fedoraproject.org/repo/pkgs/$pkg/$filename/$hashtype/$hash/$filename"
 repos = [
     # NOTE: These are source repositories; we will want to tag them as such in the future.
     {"base-uri" = "https://na.edge.kernel.org/fedora/releases/$releasever/Everything/source/tree" },
@@ -11,15 +13,19 @@ repos = [
 [distros.fedora.versions.41]
 description = "Fedora Linux 41"
 release-ver = "41"
+dist-git-branch = "f41"
 
 [distros.fedora.versions.42]
 description = "Fedora Linux 42"
 release-ver = "42"
+dist-git-branch = "f42"
 
 [distros.fedora.versions.43]
 description = "Fedora Linux 43"
 release-ver = "43"
+dist-git-branch = "f43"
 
 [distros.fedora.versions.rawhide]
 description = "Fedora Linux Rawhide"
 release-ver = "rawhide"
+dist-git-branch = "rawhide"

internal/app/azldev/cmds/component/build.go

@@ -171,7 +171,7 @@ func BuildComponent(
 		return nil
 	}, &err)
 
-	sourcePreparer, err := sources.NewPreparer(env, sourceManager)
+	sourcePreparer, err := sources.NewPreparer(sourceManager)
 	if err != nil {
 		return ComponentBuildResults{},
 			fmt.Errorf("failed to create source preparer for component %q:\n%w", component.GetName(), err)

internal/app/azldev/cmds/component/preparesources.go

@@ -84,7 +84,7 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er
 		return fmt.Errorf("failed to create source manager:\n%w", err)
 	}
 
-	preparer, err := sources.NewPreparer(env, sourceManager)
+	preparer, err := sources.NewPreparer(sourceManager)
 	if err != nil {
 		return fmt.Errorf("failed to create source preparer:\n%w", err)
 	}

internal/app/azldev/core/componentbuilder/componentbuilder_test.go

@@ -49,7 +49,7 @@ func setupBuilder(t *testing.T) *componentBuilderTestParams {
 
 	sourceManager := sourceproviders_test.NewNoOpMockSourceManager(ctrl)
 
-	preparer, err := sources.NewPreparer(testEnv.Env, sourceManager)
+	preparer, err := sources.NewPreparer(sourceManager)
 
 	require.NoError(t, err)
 

internal/app/azldev/core/sources/sourceprep.go

@@ -8,7 +8,6 @@ import (
 	"errors"
 	"fmt"
 
-	"github.com/microsoft/azldev/internal/app/azldev"
 	"github.com/microsoft/azldev/internal/app/azldev/core/components"
 	"github.com/microsoft/azldev/internal/providers/sourceproviders"
 )
@@ -29,25 +28,18 @@ type SourcePreparer interface {
 
 // Standard implementation of the [SourcePreparer] interface.
 type sourcePreparerImpl struct {
-	env           *azldev.Env
 	sourceManager sourceproviders.SourceManager
 }
 
 // NewPreparer creates a new [SourcePreparer] instance. All arguments are required.
 func NewPreparer(
-	env *azldev.Env,
 	sourceManager sourceproviders.SourceManager,
 ) (SourcePreparer, error) {
-	if env == nil {
-		return nil, errors.New("environment cannot be nil")
-	}
-
 	if sourceManager == nil {
 		return nil, errors.New("source manager cannot be nil")
 	}
 
 	return &sourcePreparerImpl{
-		env:           env,
 		sourceManager: sourceManager,
 	}, nil
 }

internal/app/azldev/core/sources/sourceprep_test.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/microsoft/azldev/internal/app/azldev/core/components/components_testutils"
 	"github.com/microsoft/azldev/internal/app/azldev/core/sources"
-	"github.com/microsoft/azldev/internal/app/azldev/core/testutils"
 	"github.com/microsoft/azldev/internal/providers/sourceproviders/sourceproviders_test"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/mock/gomock"
@@ -19,32 +18,29 @@ const testOutputDir = "/output"
 
 func TestNewPreparer(t *testing.T) {
 	ctrl := gomock.NewController(t)
-	env := testutils.NewTestEnv(t)
 	sourceManager := sourceproviders_test.NewMockSourceManager(ctrl)
 
-	preparer, err := sources.NewPreparer(env.Env, sourceManager)
+	preparer, err := sources.NewPreparer(sourceManager)
 	require.NotNil(t, preparer)
 	require.NoError(t, err)
 }
 
 func TestPrepareSources_Success(t *testing.T) {
 	ctrl := gomock.NewController(t)
-	env := testutils.NewTestEnv(t)
 	component := components_testutils.NewMockComponent(ctrl)
 	sourceManager := sourceproviders_test.NewMockSourceManager(ctrl)
 
 	component.EXPECT().GetName().AnyTimes().Return("test-component")
 	sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).Return(nil)
 
-	preparer, err := sources.NewPreparer(env.Env, sourceManager)
+	preparer, err := sources.NewPreparer(sourceManager)
 	require.NoError(t, err)
 	err = preparer.PrepareSources(t.Context(), component, testOutputDir)
 	require.NoError(t, err)
 }
 
 func TestPrepareSources_SourceManagerError(t *testing.T) {
 	ctrl := gomock.NewController(t)
-	env := testutils.NewTestEnv(t)
 	component := components_testutils.NewMockComponent(ctrl)
 	sourceManager := sourceproviders_test.NewMockSourceManager(ctrl)
 
@@ -53,7 +49,7 @@ func TestPrepareSources_SourceManagerError(t *testing.T) {
 	component.EXPECT().GetName().AnyTimes().Return("test-component")
 	sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).Return(expectedErr)
 
-	preparer, err := sources.NewPreparer(env.Env, sourceManager)
+	preparer, err := sources.NewPreparer(sourceManager)
 	require.NoError(t, err)
 
 	err = preparer.PrepareSources(t.Context(), component, testOutputDir)

internal/app/azldev/core/testutils/testenv.go

@@ -161,11 +161,14 @@ func constructProjectConfig(testMockConfigPath string) *projectconfig.ProjectCon
 	config.Project.DefaultDistro.Version = "1.0"
 
 	distro := projectconfig.DistroDefinition{
-		Versions: make(map[string]projectconfig.DistroVersionDefinition),
+		Versions:         make(map[string]projectconfig.DistroVersionDefinition),
+		LookasideBaseURI: "https://example.com/lookaside/$pkg/$filename/$hashtype/$hash/$filename",
+		DistGitBaseURI:   "https://example.com/upstream/$pkg.git",
 	}
 
 	distro.Versions["1.0"] = projectconfig.DistroVersionDefinition{
 		MockConfigPath: testMockConfigPath,
+		DistGitBranch:  "main",
 	}
 
 	config.Distros["test-distro"] = distro

internal/projectconfig/component.go

@@ -9,17 +9,15 @@ import (
 	"dario.cat/mergo"
 	"github.com/brunoga/deep"
 	"github.com/microsoft/azldev/internal/rpm"
+	"github.com/microsoft/azldev/internal/utils/fileutils"
 )
 
-// HashType represents a type of hash used for source file verification.
-type HashType string
-
 const (
 	// HashTypeSHA256 represents the SHA-256 hash algorithm.
-	HashTypeSHA256 HashType = "sha256"
+	HashTypeSHA256 = fileutils.HashTypeSHA256
 
 	// HashTypeSHA512 represents the SHA-512 hash algorithm.
-	HashTypeSHA512 HashType = "sha512"
+	HashTypeSHA512 = fileutils.HashTypeSHA512
 )
 
 // ComponentReference encapsulates a reference to a source component.
@@ -74,7 +72,7 @@ type SourceFileReference struct {
 	Hash string `toml:"Hash,omitempty" json:"Hash,omitempty"`
 
 	// Type of hash used by `Hash` (if present).
-	HashType HashType `toml:"HashType,omitempty" json:"HashType,omitempty"`
+	HashType fileutils.HashType `toml:"HashType,omitempty" json:"HashType,omitempty"`
 
 	// Ordered list of source origins to try in priority order
 	Origins []Origin `toml:"Origins,omitempty" json:"Origins,omitempty"`