Fixed tdnf provides parsing to recognize epochs in package names. (#10587)

Author: PawelWMS

toolkit/tools/imagegen/installutils/installutils.go

@@ -820,7 +820,7 @@ func calculateTotalPackages(packages []string, installRoot string) (installedPac
 		// end with an empty line.
 		for _, line := range splitStdout {
 			matches := tdnf.InstallPackageRegex.FindStringSubmatch(line)
-			if len(matches) != tdnf.InstallMaxMatchLen {
+			if len(matches) != tdnf.InstallPackageMaxMatchLen {
 				// This line contains output other than a package information; skip it
 				continue
 			}

toolkit/tools/internal/packagerepo/repocloner/rpmrepocloner/rpmrepocloner.go

@@ -474,8 +474,8 @@ func (r *RpmRepoCloner) WhatProvides(pkgVer *pkgjson.PackageVer) (packageNames [
 			// MUST keep order of packages printed by TDNF.
 			// TDNF will print the packages starting from the highest version, which allows us to work around an RPM bug:
 			// https://github.com/rpm-software-management/rpm/issues/2359
-			for _, matches := range tdnf.PackageLookupNameMatchRegex.FindAllStringSubmatch(stdout, -1) {
-				packageName := matches[tdnf.PackageNameIndex]
+			for _, matches := range tdnf.PackageProvidesRegex.FindAllStringSubmatch(stdout, -1) {
+				packageName := matches[tdnf.PackageProvidesNameIndex]
 				if lookupIgnoredCase {
 					logger.Log.Warnf("'%s' was found by case-insensitive lookup of '%s', but this is not valid and will be ignored", packageName, pkgVer.Name)
 					// This is not a valid mapping of requires -> provides, so we skip it. This is not a fatal error since
@@ -556,15 +556,15 @@ func (r *RpmRepoCloner) ClonedRepoContents() (repoContents *repocloner.RepoConte
 	repoContents = &repocloner.RepoContents{}
 	onStdout := func(line string) {
 		matches := tdnf.ListedPackageRegex.FindStringSubmatch(line)
-		if len(matches) != tdnf.ListMaxMatchLen {
+		if len(matches) != tdnf.ListedPackageMaxMatchLen {
 			return
 		}
 
 		pkg := &repocloner.RepoPackage{
-			Name:         matches[tdnf.ListPackageName],
-			Version:      matches[tdnf.ListPackageVersion],
-			Architecture: matches[tdnf.ListPackageArch],
-			Distribution: matches[tdnf.ListPackageDist],
+			Name:         matches[tdnf.ListedPackageName],
+			Version:      matches[tdnf.ListedPackageVersion],
+			Architecture: matches[tdnf.ListedPackageArch],
+			Distribution: matches[tdnf.ListedPackageDist],
 		}
 
 		pkgID := pkg.ID()

toolkit/tools/internal/tdnf/tdnf.go

@@ -28,8 +28,7 @@ var (
 	//		Repo	: [repo_name]
 	//
 	// NOTE: we ignore packages installed in the build environment denoted by "Repo	: @System".
-	PackageLookupNameMatchRegex = regexp.MustCompile(`([^:\s]+(x86_64|aarch64|noarch))\s*:[^\n]*\nRepo\s+:\s+[^@]`)
-	PackageNameIndex            = 1
+	PackageProvidesRegex = regexp.MustCompile(`(\S+)\s+:[^\n]*\nRepo\s+:\s+[^@]`)
 
 	// Tdnf may opt to ignore case when doing a provides lookup. While this is useful for a user, it will give
 	// bad results when we're trying to match a package name to a package in the repo. This regex will match the
@@ -63,21 +62,27 @@ var (
 )
 
 const (
-	InstallMatchSubString = iota
-	InstallPackageName    = iota
-	InstallPackageArch    = iota
-	InstallPackageVersion = iota
-	InstallPackageDist    = iota
-	InstallMaxMatchLen    = iota
+	InstallPackageMatchSubString = iota
+	InstallPackageName           = iota
+	InstallPackageArch           = iota
+	InstallPackageVersion        = iota
+	InstallPackageDist           = iota
+	InstallPackageMaxMatchLen    = iota
 )
 
 const (
-	ListMatchSubString = iota
-	ListPackageName    = iota
-	ListPackageArch    = iota
-	ListPackageVersion = iota
-	ListPackageDist    = iota
-	ListMaxMatchLen    = iota
+	PackageProvidesMatchSubString = iota
+	PackageProvidesNameIndex      = iota
+	PackageProvidesMaxMatchLen    = iota
+)
+
+const (
+	ListedPackageMatchSubString = iota
+	ListedPackageName           = iota
+	ListedPackageArch           = iota
+	ListedPackageVersion        = iota
+	ListedPackageDist           = iota
+	ListedPackageMaxMatchLen    = iota
 )
 
 const (

toolkit/tools/internal/tdnf/tdnf_test.go

@@ -78,7 +78,7 @@ func TestInstallPackageRegex_MatchesPackageName(t *testing.T) {
 
 	matches := InstallPackageRegex.FindStringSubmatch(line)
 
-	assert.Len(t, matches, InstallMaxMatchLen)
+	assert.Len(t, matches, InstallPackageMaxMatchLen)
 	assert.Equal(t, "X", matches[InstallPackageName])
 }
 
@@ -93,7 +93,7 @@ func TestInstallPackageRegex_MatchesPackageArch(t *testing.T) {
 
 	matches := InstallPackageRegex.FindStringSubmatch(line)
 
-	assert.Len(t, matches, InstallMaxMatchLen)
+	assert.Len(t, matches, InstallPackageMaxMatchLen)
 	assert.Equal(t, "aarch64", matches[InstallPackageArch])
 }
 
@@ -108,7 +108,7 @@ func TestInstallPackageRegex_MatchesPackageVersionNoEpoch(t *testing.T) {
 
 	matches := InstallPackageRegex.FindStringSubmatch(line)
 
-	assert.Len(t, matches, InstallMaxMatchLen)
+	assert.Len(t, matches, InstallPackageMaxMatchLen)
 	assert.Equal(t, "1.1b.8_X-22~rc1", matches[InstallPackageVersion])
 }
 
@@ -117,7 +117,7 @@ func TestInstallPackageRegex_MatchesPackageVersionWithEpoch(t *testing.T) {
 
 	matches := InstallPackageRegex.FindStringSubmatch(line)
 
-	assert.Len(t, matches, InstallMaxMatchLen)
+	assert.Len(t, matches, InstallPackageMaxMatchLen)
 	assert.Equal(t, "5:1.1b.8_X-22~rc1", matches[InstallPackageVersion])
 }
 
@@ -132,7 +132,7 @@ func TestInstallPackageRegex_MatchesPackageDist(t *testing.T) {
 
 	matches := InstallPackageRegex.FindStringSubmatch(line)
 
-	assert.Len(t, matches, InstallMaxMatchLen)
+	assert.Len(t, matches, InstallPackageMaxMatchLen)
 	assert.Equal(t, "azl3", matches[InstallPackageDist])
 }
 
@@ -153,3 +153,92 @@ func TestInstallPackageRegex_DoesNotMatchInvalidLine(t *testing.T) {
 
 	assert.False(t, InstallPackageRegex.MatchString(line))
 }
+func TestPackageLookupNameMatchRegex_MatchesExternalRepo(t *testing.T) {
+	const line = "xz-devel-5.4.4-1.azl3.x86_64 : Header and development files for xz\nRepo : toolchain-repo"
+
+	matches := PackageProvidesRegex.FindStringSubmatch(line)
+
+	assert.Len(t, matches, PackageProvidesMaxMatchLen)
+	assert.Equal(t, "xz-devel-5.4.4-1.azl3.x86_64", matches[PackageProvidesNameIndex])
+}
+
+func TestPackageLookupNameMatchRegex_MatchesPackageWithEpoch(t *testing.T) {
+	const line = "xz-devel-2:5.4.4-1.azl3.x86_64 : Header and development files for xz\nRepo : toolchain-repo"
+
+	matches := PackageProvidesRegex.FindStringSubmatch(line)
+
+	assert.Len(t, matches, PackageProvidesMaxMatchLen)
+	assert.Equal(t, "xz-devel-2:5.4.4-1.azl3.x86_64", matches[PackageProvidesNameIndex])
+}
+
+func TestPackageLookupNameMatchRegex_FailsForOutputWithoutRepo(t *testing.T) {
+	const line = "xz-devel-5.4.4-1.azl3.x86_64 : Header and development files for xz"
+
+	assert.False(t, PackageProvidesRegex.MatchString(line))
+}
+
+func TestPackageLookupNameMatchRegex_FailsForOutputWithSystemRepo(t *testing.T) {
+	const line = "xz-devel-5.4.4-1.azl3.x86_64 : Header and development files for xz\nRepo : @System"
+
+	assert.False(t, PackageProvidesRegex.MatchString(line))
+}
+
+func TestPackageLookupNameMatchRegex_FailsForEmptyOutput(t *testing.T) {
+	const line = ""
+
+	assert.False(t, PackageProvidesRegex.MatchString(line))
+}
+
+func TestPackageLookupNameMatchRegex_FailsForInvalidOutput(t *testing.T) {
+	const line = "Invalid output line"
+
+	assert.False(t, PackageProvidesRegex.MatchString(line))
+}
+
+func TestPackageLookupNameMatchRegex_MatchesOutputWithCapabilityMatch(t *testing.T) {
+	const line = "[using capability match for 'pkgconfig(liblzma)'] xz-devel-5.4.4-1.azl3.x86_64 : Header and development files for xz\nRepo : toolchain-repo"
+
+	matches := PackageProvidesRegex.FindStringSubmatch(line)
+
+	assert.Len(t, matches, PackageProvidesMaxMatchLen)
+	assert.Equal(t, "xz-devel-5.4.4-1.azl3.x86_64", matches[PackageProvidesNameIndex])
+}
+
+func TestPackageLookupNameMatchRegex_MatchesOutputWithMultiplePackages(t *testing.T) {
+	const line = "xz-devel-5.4.4-1.azl3.x86_64 : ABC\nRepo : toolchain-repo\nother-package-4.4.4-1.azl3.x86_64 : ABC2\nRepo : other-repo\n"
+
+	allMatches := PackageProvidesRegex.FindAllStringSubmatch(line, -1)
+
+	assert.Len(t, allMatches, 2)
+	assert.Len(t, allMatches[0], PackageProvidesMaxMatchLen)
+	assert.Equal(t, "xz-devel-5.4.4-1.azl3.x86_64", allMatches[0][PackageProvidesNameIndex])
+
+	assert.Len(t, allMatches[1], PackageProvidesMaxMatchLen)
+	assert.Equal(t, "other-package-4.4.4-1.azl3.x86_64", allMatches[1][PackageProvidesNameIndex])
+}
+
+func TestPackageLookupNameMatchRegex_MatchesOutputWithExternalAndSystemMix(t *testing.T) {
+	const line = "xz-devel-5.4.4-1.azl3.x86_64 : ABC\nRepo : toolchain-repo\nother-package-4.4.4-1.azl3.x86_64 : ABC2\nRepo : @System\n"
+
+	allMatches := PackageProvidesRegex.FindAllStringSubmatch(line, -1)
+
+	assert.Len(t, allMatches, 1)
+	assert.Len(t, allMatches[0], PackageProvidesMaxMatchLen)
+	assert.Equal(t, "xz-devel-5.4.4-1.azl3.x86_64", allMatches[0][PackageProvidesNameIndex])
+}
+
+func TestPackageLookupNameMatchRegex_MatchesOutputWithSystemFirstExternalSecond(t *testing.T) {
+	const line = "other-package-4.4.4-1.azl3.x86_64 : ABC2\nRepo : @System\nxz-devel-5.4.4-1.azl3.x86_64 : ABC\nRepo : toolchain-repo"
+
+	allMatches := PackageProvidesRegex.FindAllStringSubmatch(line, -1)
+
+	assert.Len(t, allMatches, 1)
+	assert.Len(t, allMatches[0], PackageProvidesMaxMatchLen)
+	assert.Equal(t, "xz-devel-5.4.4-1.azl3.x86_64", allMatches[0][PackageProvidesNameIndex])
+}
+
+func TestPackageLookupNameMatchRegex_FailsForOutputWithOnlyPluginLoaded(t *testing.T) {
+	const line = "Loaded plugin: tdnfrepogpgcheck"
+
+	assert.False(t, PackageProvidesRegex.MatchString(line))
+}