Merge branch 'main' into 2.0

Author: jslobodzian

.github/pull_request_template.md

@@ -17,6 +17,7 @@ Feel free to delete sections of the template which do not apply to your PR, or a
 - [ ] All source files have up-to-date hashes in the `*.signatures.json` files
 - [ ] `sudo make go-tidy-all` and `sudo make go-test-coverage` pass
 - [ ] Documentation has been updated to match any changes to the build system
+- [ ] If you are adding/removing a .spec file that has multiple-versions supported, please add [@microsoft/cbl-mariner-multi-package-reviewers](https://github.com/orgs/microsoft/teams/cbl-mariner-multi-package-reviewers) team as reviewer [(Eg. golang has 2 versions 1.18, 1.21+)](https://github.com/microsoft/azurelinux/tree/2.0/SPECS/golang)
 - [ ] Ready to merge
 
 ---

.github/workflows/check-kernel-config.yml

@@ -227,7 +227,7 @@ do
       # Parsing output instead of using error codes because 'wget' returns code 8 for FTP, even if the file exists.
       # Sample HTTP(S) output:  Remote file exists.
       # Sample FTP output:      File ‘time-1.9.tar.gz’ exists.
-      if ! wget --secure-protocol=TLSv1_2 --spider --timeout=2 --tries=10 "${manifesturl}" 2>&1 | grep -qP "^(Remote file|File ‘.*’) exists.*"
+      if ! wget --secure-protocol=TLSv1_2 --spider --timeout=30 --tries=10 "${manifesturl}" 2>&1 | grep -qP "^(Remote file|File ‘.*’) exists.*"
       then
         echo "Registration for $name:$version has invalid URL '$manifesturl' (could not download)"  >> bad_registrations.txt
       fi

.github/workflows/validate-cg-manifest.sh

@@ -3,6 +3,8 @@
 
 name: CodeQL CBL-Mariner repository
 
+trigger: none
+
 resources:
   repositories:
     - repository: CBL-Mariner-Pipelines

.pipelines/CodeQL/CodeQL.yml

@@ -1,2 +1,3 @@
 distroless-packages-base
 nodejs18
+prebuilt-ca-certificates

.pipelines/containerSourceData/nodejs/distroless/nodejs18.pkg

@@ -1,2 +1,3 @@
 distroless-packages-base
 prometheus
+prebuilt-ca-certificates

.pipelines/containerSourceData/prometheus/distroless/prometheus.pkg

@@ -1,2 +1,3 @@
 distroless-packages-base
 prometheus-adapter
+prebuilt-ca-certificates

.pipelines/containerSourceData/prometheusadapter/distroless/prometheusadapter.pkg

@@ -1,2 +1,3 @@
 distroless-packages-base
 python3
+prebuilt-ca-certificates

.pipelines/containerSourceData/python/distroless/python.pkg

@@ -28,6 +28,8 @@ set -e
 #   │   container_tarballs
 #   │   ├── container_base
 #   │   │   ├── core-2.0.20230607.tar.gz
+#   │   ├── core_container_builder
+#   │   │   ├── core-container-builder-2.0.20230607.tar.gz
 #   │   ├── distroless_base
 #   │   │   ├── distroless-base-2.0.20230607.tar.gz
 #   │   ├── distroless_debug
@@ -100,6 +102,7 @@ function validate_inputs {
     fi
 
     BASE_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "core-[0-9.]*.tar.gz")
+    BASE_BUILDER_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "core-container-builder-[0-9.]*.tar.gz")
     DISTROLESS_BASE_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "distroless-base-[0-9.]*.tar.gz")
     DISTROLESS_DEBUG_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "distroless-debug-[0-9.]*.tar.gz")
     DISTROLESS_MINIMAL_TARBALL=$(find "$CONTAINER_TARBALLS_DIR" -name "distroless-minimal-[0-9.]*.tar.gz")
@@ -162,6 +165,7 @@ function initialization {
     EULA_FILE_NAME="EULA-Container.txt"
 
     # Image types
+    BASE_BUILDER="base-builder"
     BASE="base"
     DISTROLESS="distroless"
     MARINARA="marinara"
@@ -199,8 +203,15 @@ function initialization {
     echo "DISTROLESS_DEBUG_IMAGE_NAME           -> $DISTROLESS_DEBUG_IMAGE_NAME"
     echo "DISTROLESS_DEBUG_NONROOT_IMAGE_NAME   -> $DISTROLESS_DEBUG_NONROOT_IMAGE_NAME"
     echo "MARINARA_IMAGE_NAME                   -> $MARINARA_IMAGE_NAME"
+
+    ROOT_FOLDER="$(git rev-parse --show-toplevel)"
+    EULA_FILE_PATH="$ROOT_FOLDER/.pipelines/container_artifacts/data"
 }
 
+function build_builder_image {
+    echo "+++ Build builder image"
+    docker import - "$BASE_BUILDER" < "$BASE_BUILDER_TARBALL"
+}
 function docker_build {
     local image_type=$1
     local image_full_name=$2
@@ -214,8 +225,6 @@ function docker_build {
     local build_dir="$WORK_DIR/container_build_dir"
     mkdir -p "$build_dir"
 
-    ROOT_FOLDER="$(git rev-parse --show-toplevel)"
-    EULA_FILE_PATH="$ROOT_FOLDER/.pipelines/container_artifacts/data"
     if [ -d "$EULA_FILE_PATH" ]; then
         cp "$EULA_FILE_PATH/$EULA_FILE_NAME" "$build_dir"/
     fi
@@ -272,13 +281,21 @@ function docker_build_marinara {
     local build_dir="$WORK_DIR/marinara_build_dir"
     mkdir -p "$build_dir"
     git clone "https://github.com/microsoft/$MARINARA.git" "$build_dir"
-    pushd "$build_dir"
-    sed -E "s|^FROM mcr\..*installer$|FROM $BASE_IMAGE_NAME as installer|g" -i "dockerfile-$MARINARA"
+
+    if [ -d "$EULA_FILE_PATH" ]; then
+        cp "$EULA_FILE_PATH/$EULA_FILE_NAME" "$build_dir"/
+    fi
+
+    pushd "$build_dir" > /dev/null
+
+    sed -E "s|^FROM mcr\..*installer$|FROM $BASE_BUILDER as installer|g" -i "dockerfile-$MARINARA"
 
     docker build . \
         -t "$MARINARA_IMAGE_NAME" \
         -f dockerfile-$MARINARA \
         --build-arg AZL_VERSION="$AZL_VERSION" \
+        --build-arg INSTALL_DEPENDENCIES=false \
+        --build-arg EULA=$EULA_FILE_NAME \
         --no-cache \
         --progress=plain
 
@@ -327,4 +344,5 @@ function build_images {
 print_inputs
 validate_inputs
 initialization
+build_builder_image
 build_images

.pipelines/containerSourceData/scripts/BuildBaseContainers.sh

@@ -75,6 +75,32 @@ FILE_EXT='.txt'
 # TODO: We may need to update this value for Azure Linux 3.0.
 OS_VERSION_PREFIX="cbl-mariner-"
 DISTRO_IDENTIFIER="cm"
+END_OF_LIFE_1_YEAR=$(date -d "+1 year" "+%Y-%m-%dT%H:%M:%SZ")
+
+# Login to the container registry.
+# Also login ORAS to the container registry.
+# $1: container registry name
+function acr_login {
+    local container_registry=$1
+    local oras_access_token
+
+    echo "+++ az login into Azure ACR $container_registry"
+    oras_access_token=$(az acr login --name "$container_registry" --expose-token --output tsv --query accessToken)
+    oras login "$container_registry.azurecr.io" \
+        --username "00000000-0000-0000-0000-000000000000" \
+        --password "$oras_access_token"
+}
+
+# Attach the end-of-life annotation to the container image.
+# $1: image name
+function oras_attach {
+    local image_name=$1
+
+    oras attach \
+        --artifact-type "application/vnd.microsoft.artifact.lifecycle" \
+        --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$END_OF_LIFE_1_YEAR" \
+        "$image_name"
+}
 
 function create_multi_arch_tags {
     # $1: original container (without '-amd64' or '-arm64' extension in tag)
@@ -168,6 +194,7 @@ function create_multi_arch_tags {
     echo "+++ push $full_multiarch_tag tag"
     docker manifest push "$full_multiarch_tag"
     echo "+++ $full_multiarch_tag tag pushed successfully"
+    oras_attach "$full_multiarch_tag"
 
     # Save the multi-arch tag to a file.
     image_basename=${multiarch_name#*/}
@@ -233,22 +260,21 @@ do
         echo "Image name: $image_name"
         echo
         container_registry="${image_name%%.*}"
-        echo "+++ login into Azure ACR $container_registry"
-        az acr login --name "$container_registry"
+        acr_login "$container_registry"
 
         amd64_image=${image_name%-*}-amd64
         docker pull "$amd64_image"
+        oras_attach "$amd64_image"
 
         # Some container images are only built for AMD64 architecture.
         if [[ $ARCHITECTURE_TO_BUILD == *"ARM64"*  ]]; then
             arm64_image=${image_name%-*}-arm64
             docker pull "$arm64_image"
+            oras_attach "$arm64_image"
         fi
 
         if [[ $container_registry != "$TARGET_ACR" ]]; then
-            echo "+++ login into Azure ACR $TARGET_ACR"
-            az acr login --name "$TARGET_ACR"
-
+            acr_login "$TARGET_ACR"
             echo "Retagging the images to $TARGET_ACR"
             # E.g., If container_registry is azurelinuxdevpreview and TARGET_ACR is azurelinuxpreview, then
             # azurelinuxdevpreview.azurecr.io/base/core:2.0 -> azurelinuxpreview.azurecr.io/base/core:2.0
@@ -258,13 +284,15 @@ do
             docker image tag "$amd64_image" "$amd64_retagged_image_name"
             docker rmi "$amd64_image"
             docker image push "$amd64_retagged_image_name"
+            oras_attach "$amd64_retagged_image_name"
 
             if [[ $ARCHITECTURE_TO_BUILD == *"ARM64"*  ]]; then
                 arm64_retagged_image_name=${arm64_image/"$container_registry"/"$TARGET_ACR"}
                 echo "Retagged arm64 image: $arm64_retagged_image_name"
                 docker image tag "$arm64_image" "$arm64_retagged_image_name"
                 docker rmi "$arm64_image"
                 docker image push "$arm64_retagged_image_name"
+                oras_attach "$arm64_retagged_image_name"
             fi
 
             image_name=$amd64_retagged_image_name