Skip to content

Commit 1b768f3

Browse files
author
Antonio Salinas
authored
feat: Prepares sources by cloning Fedora package repo (#368)
* Git contents provider and git util * Git provider Unit tests and minor refactoring * Added testuser to docker group * Added tests and a few refactors * Revert dockerfile * Refactored fedora source provider/tests * Small refactors to sourcemanager & fedoraSourceProvider * Added hash validation and download header for lookaside anti-bot detection * Refactored some hash functions * Changed hash.go to make use of Hashtype * Hash util change * Added back Accept header comment
1 parent f7dfbc1 commit 1b768f3

30 files changed

Lines changed: 1779 additions & 390 deletions

defaultconfigs/content/distros/fedora/fedora.distro.toml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
[distros.fedora]
22
description = "Fedora Linux"
33
default-version = "42"
4+
dist-git-base-uri = "https://src.fedoraproject.org/rpms/$pkg.git"
5+
lookaside-base-uri = "https://src.fedoraproject.org/repo/pkgs/$pkg/$filename/$hashtype/$hash/$filename"
46
repos = [
57
# NOTE: These are source repositories; we will want to tag them as such in the future.
68
{"base-uri" = "https://na.edge.kernel.org/fedora/releases/$releasever/Everything/source/tree" },
@@ -11,15 +13,19 @@ repos = [
1113
[distros.fedora.versions.41]
1214
description = "Fedora Linux 41"
1315
release-ver = "41"
16+
dist-git-branch = "f41"
1417

1518
[distros.fedora.versions.42]
1619
description = "Fedora Linux 42"
1720
release-ver = "42"
21+
dist-git-branch = "f42"
1822

1923
[distros.fedora.versions.43]
2024
description = "Fedora Linux 43"
2125
release-ver = "43"
26+
dist-git-branch = "f43"
2227

2328
[distros.fedora.versions.rawhide]
2429
description = "Fedora Linux Rawhide"
2530
release-ver = "rawhide"
31+
dist-git-branch = "rawhide"

internal/app/azldev/cmds/component/build.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -171,7 +171,7 @@ func BuildComponent(
171171
return nil
172172
}, &err)
173173

174-
sourcePreparer, err := sources.NewPreparer(env, sourceManager)
174+
sourcePreparer, err := sources.NewPreparer(sourceManager)
175175
if err != nil {
176176
return ComponentBuildResults{},
177177
fmt.Errorf("failed to create source preparer for component %q:\n%w", component.GetName(), err)

internal/app/azldev/cmds/component/preparesources.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er
8484
return fmt.Errorf("failed to create source manager:\n%w", err)
8585
}
8686

87-
preparer, err := sources.NewPreparer(env, sourceManager)
87+
preparer, err := sources.NewPreparer(sourceManager)
8888
if err != nil {
8989
return fmt.Errorf("failed to create source preparer:\n%w", err)
9090
}

internal/app/azldev/core/componentbuilder/componentbuilder_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ func setupBuilder(t *testing.T) *componentBuilderTestParams {
4949

5050
sourceManager := sourceproviders_test.NewNoOpMockSourceManager(ctrl)
5151

52-
preparer, err := sources.NewPreparer(testEnv.Env, sourceManager)
52+
preparer, err := sources.NewPreparer(sourceManager)
5353

5454
require.NoError(t, err)
5555

internal/app/azldev/core/sources/sourceprep.go

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ import (
88
"errors"
99
"fmt"
1010

11-
"github.com/microsoft/azldev/internal/app/azldev"
1211
"github.com/microsoft/azldev/internal/app/azldev/core/components"
1312
"github.com/microsoft/azldev/internal/providers/sourceproviders"
1413
)
@@ -29,25 +28,18 @@ type SourcePreparer interface {
2928

3029
// Standard implementation of the [SourcePreparer] interface.
3130
type sourcePreparerImpl struct {
32-
env *azldev.Env
3331
sourceManager sourceproviders.SourceManager
3432
}
3533

3634
// NewPreparer creates a new [SourcePreparer] instance. All arguments are required.
3735
func NewPreparer(
38-
env *azldev.Env,
3936
sourceManager sourceproviders.SourceManager,
4037
) (SourcePreparer, error) {
41-
if env == nil {
42-
return nil, errors.New("environment cannot be nil")
43-
}
44-
4538
if sourceManager == nil {
4639
return nil, errors.New("source manager cannot be nil")
4740
}
4841

4942
return &sourcePreparerImpl{
50-
env: env,
5143
sourceManager: sourceManager,
5244
}, nil
5345
}

internal/app/azldev/core/sources/sourceprep_test.go

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99

1010
"github.com/microsoft/azldev/internal/app/azldev/core/components/components_testutils"
1111
"github.com/microsoft/azldev/internal/app/azldev/core/sources"
12-
"github.com/microsoft/azldev/internal/app/azldev/core/testutils"
1312
"github.com/microsoft/azldev/internal/providers/sourceproviders/sourceproviders_test"
1413
"github.com/stretchr/testify/require"
1514
"go.uber.org/mock/gomock"
@@ -19,32 +18,29 @@ const testOutputDir = "/output"
1918

2019
func TestNewPreparer(t *testing.T) {
2120
ctrl := gomock.NewController(t)
22-
env := testutils.NewTestEnv(t)
2321
sourceManager := sourceproviders_test.NewMockSourceManager(ctrl)
2422

25-
preparer, err := sources.NewPreparer(env.Env, sourceManager)
23+
preparer, err := sources.NewPreparer(sourceManager)
2624
require.NotNil(t, preparer)
2725
require.NoError(t, err)
2826
}
2927

3028
func TestPrepareSources_Success(t *testing.T) {
3129
ctrl := gomock.NewController(t)
32-
env := testutils.NewTestEnv(t)
3330
component := components_testutils.NewMockComponent(ctrl)
3431
sourceManager := sourceproviders_test.NewMockSourceManager(ctrl)
3532

3633
component.EXPECT().GetName().AnyTimes().Return("test-component")
3734
sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).Return(nil)
3835

39-
preparer, err := sources.NewPreparer(env.Env, sourceManager)
36+
preparer, err := sources.NewPreparer(sourceManager)
4037
require.NoError(t, err)
4138
err = preparer.PrepareSources(t.Context(), component, testOutputDir)
4239
require.NoError(t, err)
4340
}
4441

4542
func TestPrepareSources_SourceManagerError(t *testing.T) {
4643
ctrl := gomock.NewController(t)
47-
env := testutils.NewTestEnv(t)
4844
component := components_testutils.NewMockComponent(ctrl)
4945
sourceManager := sourceproviders_test.NewMockSourceManager(ctrl)
5046

@@ -53,7 +49,7 @@ func TestPrepareSources_SourceManagerError(t *testing.T) {
5349
component.EXPECT().GetName().AnyTimes().Return("test-component")
5450
sourceManager.EXPECT().FetchComponent(gomock.Any(), component, testOutputDir).Return(expectedErr)
5551

56-
preparer, err := sources.NewPreparer(env.Env, sourceManager)
52+
preparer, err := sources.NewPreparer(sourceManager)
5753
require.NoError(t, err)
5854

5955
err = preparer.PrepareSources(t.Context(), component, testOutputDir)

internal/app/azldev/core/testutils/testenv.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,11 +161,14 @@ func constructProjectConfig(testMockConfigPath string) *projectconfig.ProjectCon
161161
config.Project.DefaultDistro.Version = "1.0"
162162

163163
distro := projectconfig.DistroDefinition{
164-
Versions: make(map[string]projectconfig.DistroVersionDefinition),
164+
Versions: make(map[string]projectconfig.DistroVersionDefinition),
165+
LookasideBaseURI: "https://example.com/lookaside/$pkg/$filename/$hashtype/$hash/$filename",
166+
DistGitBaseURI: "https://example.com/upstream/$pkg.git",
165167
}
166168

167169
distro.Versions["1.0"] = projectconfig.DistroVersionDefinition{
168170
MockConfigPath: testMockConfigPath,
171+
DistGitBranch: "main",
169172
}
170173

171174
config.Distros["test-distro"] = distro

internal/projectconfig/component.go

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,17 +9,15 @@ import (
99
"dario.cat/mergo"
1010
"github.com/brunoga/deep"
1111
"github.com/microsoft/azldev/internal/rpm"
12+
"github.com/microsoft/azldev/internal/utils/fileutils"
1213
)
1314

14-
// HashType represents a type of hash used for source file verification.
15-
type HashType string
16-
1715
const (
1816
// HashTypeSHA256 represents the SHA-256 hash algorithm.
19-
HashTypeSHA256 HashType = "sha256"
17+
HashTypeSHA256 = fileutils.HashTypeSHA256
2018

2119
// HashTypeSHA512 represents the SHA-512 hash algorithm.
22-
HashTypeSHA512 HashType = "sha512"
20+
HashTypeSHA512 = fileutils.HashTypeSHA512
2321
)
2422

2523
// ComponentReference encapsulates a reference to a source component.
@@ -74,7 +72,7 @@ type SourceFileReference struct {
7472
Hash string `toml:"Hash,omitempty" json:"Hash,omitempty"`
7573

7674
// Type of hash used by `Hash` (if present).
77-
HashType HashType `toml:"HashType,omitempty" json:"HashType,omitempty"`
75+
HashType fileutils.HashType `toml:"HashType,omitempty" json:"HashType,omitempty"`
7876

7977
// Ordered list of source origins to try in priority order
8078
Origins []Origin `toml:"Origins,omitempty" json:"Origins,omitempty"`

0 commit comments

Comments
 (0)