Merge branch 'main' into 2.0

Author: jslobodzian

.CodeQL.yml

@@ -0,0 +1,29 @@
+path_classifiers:
+  library:
+    # Treat source files for all compiled languages in the specs directories
+    # as 3rd party library sources because they are not owned by us.
+    #
+    # Extensions from https://codeql.github.com/docs/codeql-overview/supported-languages-and-frameworks/.
+    - "SPECS*/**/*.c"
+    - "SPECS*/**/*.c++"
+    - "SPECS*/**/*.cc"
+    - "SPECS*/**/*.cpp"
+    - "SPECS*/**/*.cs"
+    - "SPECS*/**/*.cshtml"
+    - "SPECS*/**/*.csproj"
+    - "SPECS*/**/*.cts"
+    - "SPECS*/**/*.cxx"
+    - "SPECS*/**/*.go"
+    - "SPECS*/**/*.h"
+    - "SPECS*/**/*.h++"
+    - "SPECS*/**/*.hh"
+    - "SPECS*/**/*.hpp"
+    - "SPECS*/**/*.hxx"
+    - "SPECS*/**/*.java"
+    - "SPECS*/**/*.kt"
+    - "SPECS*/**/*.mts"
+    - "SPECS*/**/*.sln"
+    - "SPECS*/**/*.swift"
+    - "SPECS*/**/*.ts"
+    - "SPECS*/**/*.tsx"
+    - "SPECS*/**/*.xaml"

.github/workflows/check-entangled-specs.yml

@@ -25,10 +25,10 @@ jobs:
         uses: actions/checkout@v4
 
       # For consistency, we use the same major/minor version of Python that CBL-Mariner ships
-      - name: Setup Python 3.7
+      - name: Setup Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Get Python dependencies
         run: python3 -m pip install -r toolkit/scripts/requirements.txt

.github/workflows/lint-specs.yml

@@ -62,10 +62,10 @@ jobs:
           path: 'spec-cleaner'
 
       # For consistency, we use the same major/minor version of Python that CBL-Mariner ships
-      - name: Setup Python 3.7
+      - name: Setup Python 3.9
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       # We take our version of the linting tool from the master branch to ensure rules
       # are consistent across all branches

.pipelines/prchecks/PackageBuildPRCheck.yml

@@ -80,9 +80,11 @@ extends:
                       # GCC fails to build as a regular package.
                       ignoredSpecs: ["gcc"]
 
-                  - script: echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                  - script: |
+                      echo "##vso[task.setvariable variable=toolchainArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                      echo "##vso[task.setvariable variable=toolchainTarballName;isOutput=true]toolchain_built_rpms_all.tar.gz"
                     name: "ToolchainArtifactName"
-                    displayName: "Set variable for published artifact name"
+                    displayName: "Set variables for published toolchain tarball"
 
                   # 1. Automatic publishing won't work if 'isCustom: true' is set on the pool. We cannot do 'isCustom: false' because
                   #    then OneBranch attempts to perform additional actions (adding build tags for instance), which require additional permissions
@@ -104,24 +106,38 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   ob_artifactBaseName: $(rpmsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
+                  outputRPMsTarballName: "rpms.tar.gz"
                   toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                  toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download toolchain"
+                    inputs:
+                      artifact: $(toolchainArtifactName)
+                      patterns: "**/$(toolchainTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templates/PackageBuild.yml@self
                     parameters:
                       checkBuildRetries: "1"
-                      customToolchainArtifactName: $(toolchainArtifactName)
+                      customToolchainTarballName: $(toolchainTarballName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
                       isCheckBuild: true
                       isQuickRebuildPackages: true
                       isUseCCache: true
                       maxCPU: "${{ configuration.maxCPUs }}"
                       outputArtifactsFolder: $(ob_outputDirectory)
+                      outputRPMsTarballName: $(outputRPMsTarballName)
                       pipArtifactFeeds: "mariner/Mariner-Pypi-Feed"
                       selfRepoName: self
                       testSuiteName: "[${{ configuration.name }}] Package test"
 
-                  - script: echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                  - script: |
+                      echo "##vso[task.setvariable variable=rpmsArtifactName;isOutput=true]$(ob_artifactBaseName)"
+                      echo "##vso[task.setvariable variable=rpmsTarballName;isOutput=true]$(outputRPMsTarballName)"
                     name: "RPMsArtifactName"
                     displayName: "Set variable for published artifact name"
 
@@ -142,15 +158,25 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   ob_artifactBaseName: $(toolchainTestsArtifactNameBase)_${{ configuration.name }}_$(System.JobAttempt)
                   ob_outputDirectory: $(Build.ArtifactStagingDirectory)
                   testListFromToolchain: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['CalculateToolchainPackageRetestList.toolchainPackageRetestList'] ]
                   toolchainArtifactName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainArtifactName'] ]
+                  toolchainTarballName: $[ stageDependencies.Toolchain_${{ configuration.name }}.Build.outputs['ToolchainArtifactName.toolchainTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download toolchain"
+                    inputs:
+                      artifact: $(toolchainArtifactName)
+                      patterns: "**/$(toolchainTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templates/PackageBuild.yml@self
                     parameters:
                       checkBuildRetries: "1"
-                      customToolchainArtifactName: $(toolchainArtifactName)
+                      customToolchainTarballName: $(toolchainTarballName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
                       isAllowToolchainRebuilds: true
                       isCheckBuild: true
                       isQuickRebuildPackages: true
@@ -179,8 +205,18 @@ extends:
                   isCustom: true
                   name: ${{ configuration.agentPool }}
                 variables:
+                  inputArtifactsLocation: $(Agent.TempDirectory)
                   rpmsArtifactName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsArtifactName'] ]
+                  rpmsTarballName: $[ stageDependencies.RPMs_${{ configuration.name }}.BuildAndTest.outputs['RPMsArtifactName.rpmsTarballName'] ]
                 steps:
+                  - task: DownloadPipelineArtifact@2
+                    displayName: "Download RPMs tarball"
+                    inputs:
+                      artifact: $(rpmsArtifactName)
+                      patterns: "**/$(rpmsTarballName)"
+                      targetPath: $(inputArtifactsLocation)
+
                   - template: .pipelines/templatesWithCheckout/SodiffCheck.yml@self
                     parameters:
-                      inputArtifactName: $(rpmsArtifactName)
+                      inputArtifactsFolder: $(inputArtifactsLocation)
+                      inputRPMsTarballName: $(rpmsTarballName)

.pipelines/templates/PackageBuild.yml

@@ -14,13 +14,9 @@ parameters:
     type: number
     default: 12
 
-  - name: customToolchainArtifactName
-    type: string
-    default: ""
-
   - name: customToolchainTarballName
     type: string
-    default: "toolchain_built_rpms_all.tar.gz"
+    default: ""
 
   - name: extraPackageRepos
     type: string
@@ -30,12 +26,16 @@ parameters:
     type: boolean
     default: true
 
-  - name: inputCacheArtifacts
+  - name: inputArtifactsFolder
+    type: string
+    default: "$(Agent.TempDirectory)"
+
+  - name: inputCacheRPMsTarballs
     type: object
     default: []
     # Sample:
-    # - name: build-artifacts
-    #   rpmsTarball: cache.tar.gz
+    #   - cache.tar.gz
+    #   - cache2.tar.gz
 
   - name: isAllowToolchainRebuilds
     type: string
@@ -160,15 +160,9 @@ steps:
           artifactFeeds: "${{ parameters.pipArtifactFeeds }}"
         displayName: "Authenticate to custom pip artifact feeds"
 
-  - ${{ if parameters.customToolchainArtifactName }}:
-      - task: DownloadPipelineArtifact@2
-        displayName: "Download toolchain"
-        inputs:
-          artifact: "${{ parameters.customToolchainArtifactName }}"
-          patterns: "**/${{ parameters.customToolchainTarballName }}"
-
+  - ${{ if parameters.customToolchainTarballName }}:
       - script: |
-          toolchain_archive="$(find "$(Pipeline.Workspace)" -name "${{ parameters.customToolchainTarballName }}" -print -quit)"
+          toolchain_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.customToolchainTarballName }}" -print -quit)"
           if [[ ! -f "$toolchain_archive" ]]; then
             echo "ERROR: toolchain archive not found!" >&2
             exit 1
@@ -178,17 +172,11 @@ steps:
           sudo make -C "${{ parameters.buildRepoRoot }}/toolkit" toolchain TOOLCHAIN_ARCHIVE="$toolchain_archive"
         displayName: "Populate toolchain"
 
-  - ${{ each inputCacheArtifact in parameters.inputCacheArtifacts }}:
-      - task: DownloadPipelineArtifact@2
-        displayName: "Download input cache RPM from ${{ inputCacheArtifact.name }}"
-        inputs:
-          artifact: "${{ inputCacheArtifact.name }}"
-          patterns: "**/${{ inputCacheArtifact.rpmsTarball }}"
-
+  - ${{ each inputCacheRPMsTarball in parameters.inputCacheRPMsTarballs }}:
       - script: |
-          rpms_archive="$(find "$(Pipeline.Workspace)" -name "${{ inputCacheArtifact.rpmsTarball }}" -print -quit)"
+          rpms_archive="$(find "${{ parameters.inputArtifactsFolder }}" -name "${{ inputCacheRPMsTarball }}" -print -quit)"
           if [[ ! -f "$rpms_archive" ]]; then
-            echo "ERROR: cache RPMs archive '${{ inputCacheArtifact.rpmsTarball }}' not found!" >&2
+            echo "ERROR: cache RPMs archive '${{ inputCacheRPMsTarball }}' not found!" >&2
             exit 1
           fi
 
@@ -200,7 +188,7 @@ steps:
         check_build_retries_arg="CHECK_BUILD_RETRIES=${{ parameters.checkBuildRetries }}"
       fi
 
-      if [[ -n "${{ parameters.customToolchainArtifactName }}" ]]; then
+      if [[ -n "${{ parameters.customToolchainTarballName }}" ]]; then
         toolchain_archive_arg="TOOLCHAIN_ARCHIVE=$(toolchainArchive)"
       fi
 

.pipelines/templatesWithCheckout/SodiffCheck.yml

@@ -6,8 +6,9 @@ parameters:
     type: string
     default: "$(Build.SourcesDirectory)"
 
-  - name: inputArtifactName
+  - name: inputArtifactsFolder
     type: string
+    default: "$(Agent.TempDirectory)"
 
   - name: inputRPMsTarballName
     type: string
@@ -26,19 +27,11 @@ parameters:
     default: "$(Agent.TempDirectory)/SourcesWorkspace"
 
 steps:
-  - task: DownloadPipelineArtifact@2
-    displayName: "Download sources for signing"
-    inputs:
-      artifact: ${{ parameters.inputArtifactName }}
-      patterns: |
-        **/${{ parameters.inputRPMsTarballName }}
-      targetPath: "$(Agent.TempDirectory)"
-
   - script: |
       set -e
 
       mkdir -p "${{ parameters.sourcesWorkspace }}"
-      find "$(Agent.TempDirectory)" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf
+      find "${{ parameters.inputArtifactsFolder }}" -name "${{ parameters.inputRPMsTarballName }}" -print0 | xargs -0 -n 1 tar -C "${{ parameters.sourcesWorkspace }}" -xkf
     displayName: "Extract sources tarball"
 
   - script: |