Merge branch '3.0-dev' into anphel/3.0-prod-update-sep19

Author: anphel31

.github/CODEOWNERS

@@ -62,7 +62,6 @@
 /SPECS/virtiofsd/* @microsoft/cbl-mariner-kata-containers
 
 /SPECS/cloud-hypervisor-cvm/* @microsoft/cbl-mariner-kata-containers
-/SPECS/hvloader/* @microsoft/cbl-mariner-kata-containers
 
 /SPECS/cloud-init/* @microsoft/cbl-mariner-provisioning
 /SPECS/walinuxagent/* @microsoft/cbl-mariner-provisioning

.github/workflows/check-circular-deps.yml

@@ -5,9 +5,9 @@ name: Circular dependency check
 
 on:
   push:
-    branches: [3.0*]
+    branches: [3.0*, fasttrack/*, "!fasttrack/2.0"]
   pull_request:
-    branches: [3.0*]
+    branches: [3.0*, fasttrack/*, "!fasttrack/2.0"]
 
 jobs:
   spec-check:

.github/workflows/go-test-coverage.yml

@@ -86,7 +86,7 @@ jobs:
         sudo env "PATH=$PATH" make go-test-coverage
 
     - name: Upload test coverage
-      uses: actions/upload-artifact@v2.1.4
+      uses: actions/upload-artifact@v4
       with:
         name: TestCoverage
         path: toolkit/out/tools/test_coverage_report.html

.github/workflows/lint-specs.yml

@@ -95,7 +95,7 @@ jobs:
           fi
           exit 0
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: ${{ always() }}
         with:
           name: linted_specs

.pipelines/containerSourceData/busybox/Dockerfile-Busybox

@@ -33,8 +33,7 @@ RUN mkdir /staging \
     && pushd /staging \
     && rm -rf boot media mnt opt run \
     && rm -rf usr/lib/sysimage \
-    && rm -rf var/cache \
-    && rm -rf var/lib/rpm; \
+    && rm -rf var/cache; \
 	ln -vL /staging/usr/sbin/busybox /staging/bin/; \
 	chroot /staging /bin/busybox --install -s /bin
 

.pipelines/containerSourceData/scripts/BuildGoldenContainer.sh

@@ -58,7 +58,7 @@ set -e
 #     -j OUTPUT -k ./rpms.tar.gz -l ~/azurelinux/.pipelines/containerSourceData \
 #     -m "false" -n "false" -p development -q "false" -u "true"
 
-while getopts ":a:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:t:u:v:" OPTIONS; do
+while getopts ":a:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:t:u:v:w:" OPTIONS; do
     case ${OPTIONS} in
     a ) BASE_IMAGE_NAME_FULL=$OPTARG;;
     b ) ACR=$OPTARG;;
@@ -82,6 +82,7 @@ while getopts ":a:b:c:d:e:f:g:h:i:j:k:l:m:n:o:p:q:r:s:t:u:v:" OPTIONS; do
     t ) SBOM_SCRIPT=$OPTARG;;
     u ) DISTROLESS=$OPTARG;;
     v ) VERSION_EXTRACT_CMD=$OPTARG;;
+    w ) TOOLCHAIN_RPMS_TARBALL=$OPTARG;;
 
     \? )
         echo "Error - Invalid Option: -$OPTARG" 1>&2
@@ -125,6 +126,7 @@ function print_inputs {
     echo "SBOM_TOOL_PATH                -> $SBOM_TOOL_PATH"
     echo "SBOM_SCRIPT                   -> $SBOM_SCRIPT"
     echo "DISTROLESS                    -> $DISTROLESS"
+    echo "TOOLCHAIN_RPMS_TARBALL -> $TOOLCHAIN_RPMS_TARBALL"
 }
 
 function validate_inputs {
@@ -168,6 +170,11 @@ function validate_inputs {
         exit 1
     fi
 
+    if [[ ! -f $TOOLCHAIN_RPMS_TARBALL ]]; then
+        echo "Error - No TOOLCHAIN_RPMS tarball found under '$TOOLCHAIN_RPMS_TARBALL'."
+        exit 1
+    fi
+
     if [ ! -d "$CONTAINER_SRC_DIR" ]; then
         echo "Error - Container source directory does not exist."
         exit 1
@@ -236,7 +243,9 @@ function prepare_docker_directory {
     mkdir -pv "$HOST_MOUNTED_DIR"
 
     # Copy files into docker context directory
-    tar -xf "$RPMS_TARBALL" -C "$HOST_MOUNTED_DIR"/
+    tar -xvf "$RPMS_TARBALL" -C "$HOST_MOUNTED_DIR"/
+    # we look for the toolchain rpms in the same directory as the rpms tarball
+    tar -xvf "$TOOLCHAIN_RPMS_TARBALL" -C "$HOST_MOUNTED_DIR/RPMS"/
     cp -v "$CONTAINER_SRC_DIR/azurelinuxlocal.repo" "$HOST_MOUNTED_DIR"/
 }
 

.pipelines/prchecks/PackageBuildPRCheck.yml

@@ -1,10 +1,8 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
-# Since we're boosting our builds by using a private, pre-compiled raw toolchain
-# the pipeline requires defining the following variables outside of the YAML:
-# - rawToolchainCacheURL_AMD64
-# - rawToolchainCacheURL_ARM64
+# The "agentPool" parameter is defined in the "Agent pools (DEV)" variable group.
+# The "rawToolchain*" parameters are defined in the "Raw toolchain info" variable group.
 
 trigger: none
 
@@ -13,15 +11,15 @@ parameters:
     type: object
     default:
       - name: "AMD64"
-        agentPool: "$(DEV_AMD64_Managed)" # Pool defined inside the "Agent pools (DEV)" variable group.
+        agentPool: "$(DEV_AMD64_Managed)"
         maxCPUs: "$(($(nproc) / 2))"
-        rawToolchainCacheURL: "$(rawToolchainCacheURL_AMD64)"
-        rawToolchainExpectedHash: "f56df34b90915c93f772d3961bf5e9eeb8c1233db43dd92070214e4ce6b72894"
+        rawToolchainCacheURL: "$(rawToolchainCacheURL_AMD64_3.0)"
+        rawToolchainExpectedHash: "$(rawToolchainCacheHash_AMD64_3.0)"
       - name: "ARM64"
-        agentPool: "$(DEV_ARM64_Managed)" # Pool defined inside the "Agent pools (DEV)" variable group.
+        agentPool: "$(DEV_ARM64_Managed)"
         maxCPUs: "$(($(nproc) / 3))"
-        rawToolchainCacheURL: "$(rawToolchainCacheURL_ARM64)"
-        rawToolchainExpectedHash: "65de43b3bdcfdaac71df1f11fd1f830a8109b1eb9d7cb6cbc2e2d0e929d0ef76"
+        rawToolchainCacheURL: "$(rawToolchainCacheURL_ARM64_3.0)"
+        rawToolchainExpectedHash: "$(rawToolchainCacheHash_ARM64_3.0)"
   - name: debug
     type: boolean
     default: false
@@ -36,12 +34,15 @@ resources:
 
 variables:
   - group: "Agent pools (DEV)"
+  - group: "Raw toolchain info"
   - name: rpmsArtifactNameBase
     value: RPMs
   - name: toolchainArtifactNameBase
     value: Toolchain
+  - name: toolchainTestsArtifactNameBase
+    value: Toolchain_tests
   - name: system.debug
-    value: '${{ parameters.debug }}'
+    value: "${{ parameters.debug }}"
 
 extends:
   template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates
@@ -59,7 +60,7 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
-                  ob_artifactBaseName: $(toolchainArtifactNameBase)_${{ configuration.name }}
+                  ob_artifactBaseName: $(toolchainArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
                 steps:
                   - template: .pipelines/templates/RawToolchainDownload.yml@self
@@ -75,14 +76,21 @@ extends:
                   # Toolchain package tests should be run through the full package build, calculate the list of packages that should be re-tested
                   # and make it available to the next stage via an output variable: 'CalculateToolchainPackageRetestList.toolchainPackageRetestList'
                   - template: .pipelines/templates/ToolchainCalculatePackageRetests.yml@self
+                    parameters:
+                      # GCC fails to build as a regular package.
+                      ignoredSpecs: ["gcc"]
+
+                  - script: echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                    name: "ToolchainArtifactName"
+                    displayName: "Set variable for published artifact name"
 
                   # 1. Automatic publishing won't work if 'isCustom: true' is set on the pool. We cannot do 'isCustom: false' because
                   #    then OneBranch attempts to perform additional actions (adding build tags for instance), which require additional permissions
                   #    that the PR check pipeline does not have.
                   # 2. The value for 'artifact' must equal $(ob_artifactBaseName), as this is the only value OneBranch accepts.
                   - task: PublishPipelineArtifact@1
                     inputs:
-                      artifact: $(toolchainArtifactNameBase)_${{ configuration.name }}
+                      artifact: $(toolchainArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                       targetPath: $(ob_outputDirectory)
                     condition: always()
                     displayName: "Publish toolchain artifacts"
@@ -96,38 +104,83 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
-                  ob_artifactBaseName: ${{ variables.rpmsArtifactNameBase }}_${{ configuration.name }}
+                  ob_artifactBaseName: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
-                  testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
+                  toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
                 steps:
                   - template: .pipelines/templates/PackageBuild.yml@self
                     parameters:
-                      customToolchainArtifactName: $(toolchainArtifactNameBase)_${{ configuration.name }}
+                      checkBuildRetries: "1"
+                      customToolchainArtifactName: $(toolchainArtifactName)
                       isCheckBuild: true
                       isQuickRebuildPackages: true
-                      outputArtifactsFolder: $(ob_outputDirectory)
+                      isUseCCache: true
                       maxCPU: "${{ configuration.maxCPUs }}"
+                      outputArtifactsFolder: $(ob_outputDirectory)
                       pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
                       selfRepoName: self
                       testSuiteName: "[${{ configuration.name }}] Package test"
-                      testRerunList: "$(testListFromToolchain)"
+
+                  - script: echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                    name: "RPMsArtifactName"
+                    displayName: "Set variable for published artifact name"
 
                   - task: PublishPipelineArtifact@1
                     inputs:
-                      artifact: ${{ variables.rpmsArtifactNameBase }}_${{ configuration.name }}
+                      artifact: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                       targetPath: $(ob_outputDirectory)
                     condition: always()
                     displayName: "Publish packages build artifacts"
 
-          - stage: sodiff_${{ configuration.name }}
+          - stage: Toolchain_tests_${{ configuration.name }}
+            dependsOn: Toolchain_${{ configuration.name }}
+            jobs:
+              - job: TestToolchainPackages
+                condition: stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList']
+                pool:
+                  type: linux
+                  isCustom: true
+                  name: ${{ configuration.agentPool }}
+                variables:
+                  ob_artifactBaseName: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
+                  ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+                  testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
+                  toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                steps:
+                  - template: .pipelines/templates/PackageBuild.yml@self
+                    parameters:
+                      checkBuildRetries: "1"
+                      customToolchainArtifactName: $(toolchainArtifactName)
+                      isAllowToolchainRebuilds: true
+                      isCheckBuild: true
+                      isQuickRebuildPackages: true
+                      isUseCCache: true
+                      maxCPU: "${{ configuration.maxCPUs }}"
+                      outputArtifactsFolder: $(ob_outputDirectory)
+                      pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
+                      selfRepoName: self
+                      srpmPackList: "$(testListFromToolchain)"
+                      testRerunList: "$(testListFromToolchain)"
+                      testSuiteName: "[${{ configuration.name }}] Toolchain test"
+
+                  - task: PublishPipelineArtifact@1
+                    inputs:
+                      artifact: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
+                      targetPath: $(ob_outputDirectory)
+                    condition: always()
+                    displayName: "Publish toolchain build artifacts"
+
+          - stage: Sodiff_${{ configuration.name }}
             dependsOn: RPMs_${{ configuration.name }}
             jobs:
               - job: Sodiff_Check
                 pool:
                   type: linux
                   isCustom: true
                   name: ${{ configuration.agentPool }}
+                variables:
+                  rpmsArtifactName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsArtifactName'] ]
                 steps:
                   - template: .pipelines/templatesWithCheckout/SodiffCheck.yml@self
                     parameters:
-                      inputArtifactName: ${{ variables.rpmsArtifactNameBase }}_${{ configuration.name }}
+                      inputArtifactName: $(rpmsArtifactName)