Merge 3.0-dev to 3.0

Author: jslobodzian

.github/workflows/check-circular-deps.yml

@@ -22,10 +22,6 @@ jobs:
       - name: Check for circular dependencies
         run: |
           echo "Checking for circular dependency loops..."
-          # Call this script to sync the toolchain manifests with the LKG daily build.
-          ./toolkit/scripts/setuplkgtoolchain.sh
-          # Determine the LKG daily build ID.
-          LKG_BUILD_ID=$(wget -qO - https://mariner3dailydevrepo.blob.core.windows.net/lkg/lkg-3.0-dev.json | jq -r ".dailybuildid" | tr '\.' '-')
           # Setup the toolchain using the LKG daily build, and then make the full package graph.
           # This will fail if any circular dependency loops are detected in the core SPECs.
-          sudo make -C toolkit -j$(nproc) graph REBUILD_TOOLS=y DAILY_BUILD_ID=${LKG_BUILD_ID}
+          sudo make -C toolkit graph REBUILD_TOOLS=y DAILY_BUILD_ID=lkg -j30

.pipelines/containerSourceData/Dockerfile-Nvidia-Initial

@@ -0,0 +1,30 @@
+ARG BASE_IMAGE \
+\
+FROM $BASE_IMAGE \
+\
+ARG CORE_RPMS_TO_INSTALL \
+ARG NVIDIA_RPMS_TO_INSTALL \
+\
+RUN --mount=type=bind,source=./Stage/,target=/dockerStage/ \\\
+    RPMS_PATH="/dockerStage/RPMS"; \\\
+    LOCAL_REPO_PATH="/localrepo"; \\\
+    NVIDIA_RPM_DOWNLOAD_PATH="/opt/nvidia"; \\\
+    mkdir -p $LOCAL_REPO_PATH; \\\
+    mkdir -p $NVIDIA_RPM_DOWNLOAD_PATH; \\\
+    tdnf install -y createrepo; \\\
+    cp -r ${RPMS_PATH} ${LOCAL_REPO_PATH}; \\\
+    cat /dockerStage/azurelinuxlocal.repo >> /etc/yum.repos.d/local.repo; \\\
+    createrepo --compatibility --database ${LOCAL_REPO_PATH} --workers 10; tdnf makecache; \\\
+    tdnf autoremove -y createrepo; \\\
+    for rpm in "${CORE_RPMS_TO_INSTALL[@]}"; do \\\
+       echo "RPM: $rpm"; \\\
+       tdnf install -y $rpm; \\\
+    done; \\\
+    for rpm in "${NVIDIA_RPMS_TO_INSTALL[@]}"; do \\\
+        echo "RPM: $rpm"; \\\
+        tdnf -y --downloadonly --downloaddir=$NVIDIA_RPM_DOWNLOAD_PATH install $rpm; \\\
+    done; \\\
+    tdnf clean all; \\\
+    rm -f /etc/yum.repos.d/local.repo; \\\
+    rm -rf /var/cache/tdnf; \\\
+    rm -rf ${LOCAL_REPO_PATH};

.pipelines/containerSourceData/cuda/core.pkg

@@ -0,0 +1,3 @@
+util-linux
+ca-certificates
+kernel

.pipelines/containerSourceData/cuda/nvidia.pkg

@@ -0,0 +1,3 @@
+cuda
+nvidia-fabric-manager
+libnvidia-nscq

.pipelines/containerSourceData/scripts/BuildContainerCommonSteps.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+set -e
+
+function initialization {
+    echo "+++ Initialization"
+    if [ "$PUBLISHING_LEVEL" = "preview" ]; then
+        CONTAINER_IMAGE_NAME=${ACR}.azurecr.io/${REPO_PREFIX}/${REPOSITORY}
+    elif [ "$PUBLISHING_LEVEL" = "development" ]; then
+        CONTAINER_IMAGE_NAME=${ACR}.azurecr.io/${REPOSITORY}
+    fi
+
+    BASE_IMAGE_NAME=${BASE_IMAGE_NAME_FULL%:*}  # # mcr.microsoft.com/azurelinux/base/core
+    BASE_IMAGE_TAG=${BASE_IMAGE_NAME_FULL#*:}   # 3.0
+    AZURE_LINUX_VERSION=${BASE_IMAGE_TAG%.*}    # 3.0
+    DISTRO_IDENTIFIER="azl"
+    END_OF_LIFE_1_YEAR=$(date -d "+1 year" "+%Y-%m-%dT%H:%M:%SZ")
+
+    echo "Container Image Name          -> $CONTAINER_IMAGE_NAME"
+    echo "Base ACR Container Name       -> $BASE_IMAGE_NAME"
+    echo "Base ACR Container Tag        -> $BASE_IMAGE_TAG"
+    echo "Azure Linux Version           -> $AZURE_LINUX_VERSION"
+    echo "Distro Identifier             -> $DISTRO_IDENTIFIER"
+    echo "End of Life                   -> $END_OF_LIFE_1_YEAR"
+}
+
+function finalize {
+    echo "+++ Finalize"
+    docker image tag "$CONTAINER_IMAGE_NAME" "$CONTAINER_IMAGE_NAME_FINAL"
+    docker rmi -f "$CONTAINER_IMAGE_NAME"
+    echo "+++ Save container image name to file PublishedContainers-$IMAGE.txt"
+    echo "$CONTAINER_IMAGE_NAME_FINAL" >> "$OUTPUT_DIR/PublishedContainers-$IMAGE.txt"
+
+    # Publish the image to ACR
+    publish_to_acr
+
+    # Generate SBOM
+    generate_image_sbom
+}
+
+function oras_attach {
+    local image_name=$1
+    oras attach \
+        --artifact-type "application/vnd.microsoft.artifact.lifecycle" \
+        --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$END_OF_LIFE_1_YEAR" \
+        "$image_name"
+}
+
+function publish_to_acr {
+    if [[ ! "$PUBLISH_TO_ACR" =~ [Tt]rue ]]; then
+        echo "+++ Skip publishing to ACR"
+        return
+    fi
+    local oras_access_token
+
+    echo "+++ az login into Azure ACR $ACR"
+    oras_access_token=$(az acr login --name "$ACR" --expose-token --output tsv --query accessToken)
+    oras login "$ACR.azurecr.io" \
+        --username "00000000-0000-0000-0000-000000000000" \
+        --password "$oras_access_token"
+
+    echo "+++ Publish container $CONTAINER_IMAGE_NAME_FINAL"
+    docker image push "$CONTAINER_IMAGE_NAME_FINAL"
+    oras_attach "$CONTAINER_IMAGE_NAME_FINAL"
+}
+
+function generate_image_sbom {
+    if [[ ! "$CREATE_SBOM" =~ [Tt]rue ]]; then
+        echo "+++ Skip creating SBOM"
+        return
+    fi
+
+    echo "+++ Generate SBOM for the container image"
+    echo "Sanitized image name has '/' replaced with '-' and ':' replaced with '_'."
+    CONTAINER_IMAGE_NAME_SANITIZED=$(echo "$CONTAINER_IMAGE_NAME_FINAL" | tr '/' '-' | tr ':' '_')
+    echo "CONTAINER_IMAGE_NAME_SANITIZED   -> $CONTAINER_IMAGE_NAME_SANITIZED"
+
+    DOCKER_BUILD_DIR=$(mktemp -d)
+    # SBOM script will create the SBOM at the following path.
+    IMAGE_SBOM_MANIFEST_PATH="$DOCKER_BUILD_DIR/_manifest/spdx_2.2/manifest.spdx.json"
+    /bin/bash "$SBOM_SCRIPT" \
+        "$DOCKER_BUILD_DIR" \
+        "$CONTAINER_IMAGE_NAME_FINAL" \
+        "$SBOM_TOOL_PATH" \
+        "$BASE_IMAGE_NAME-$COMPONENT" \
+        "$COMPONENT_VERSION-$DISTRO_IDENTIFIER$BASE_IMAGE_TAG"
+
+    SBOM_IMAGES_DIR="$OUTPUT_DIR/SBOM_IMAGES"
+    mkdir -p "$SBOM_IMAGES_DIR"
+    cp -v "$IMAGE_SBOM_MANIFEST_PATH" "$SBOM_IMAGES_DIR/$CONTAINER_IMAGE_NAME_SANITIZED.spdx.json"
+    echo "Generated SBOM:'$SBOM_IMAGES_DIR/$CONTAINER_IMAGE_NAME_SANITIZED.spdx.json'"
+    sudo rm -rf "$DOCKER_BUILD_DIR"
+}

.pipelines/containerSourceData/scripts/BuildGoldenContainer.sh

@@ -47,6 +47,7 @@ set -e
 #   │   ├── acrRepoV2.json
 #   ├── scripts
 #   │   ├── BuildGoldenContainer.sh
+#   │   ├── BuildContainerCommonSteps.sh
 #   ├── Dockerfile-Initial
 #   ├── azurelinuxlocal.repo
 
@@ -189,28 +190,6 @@ function validate_inputs {
     fi
 }
 
-function initialization {
-    echo "+++ Initialization"
-    if [ "$PUBLISHING_LEVEL" = "preview" ]; then
-        GOLDEN_IMAGE_NAME=${ACR}.azurecr.io/${REPO_PREFIX}/${REPOSITORY}
-    elif [ "$PUBLISHING_LEVEL" = "development" ]; then
-        GOLDEN_IMAGE_NAME=${ACR}.azurecr.io/${REPOSITORY}
-    fi
-
-    BASE_IMAGE_NAME=${BASE_IMAGE_NAME_FULL%:*}  # mcr.microsoft.com/azurelinux/base/core
-    BASE_IMAGE_TAG=${BASE_IMAGE_NAME_FULL#*:}   # 3.0
-    AZURE_LINUX_VERSION=${BASE_IMAGE_TAG%.*}    # 3.0
-    DISTRO_IDENTIFIER="azl"
-    END_OF_LIFE_1_YEAR=$(date -d "+1 year" "+%Y-%m-%dT%H:%M:%SZ")
-
-    echo "Golden Image Name             -> $GOLDEN_IMAGE_NAME"
-    echo "Base ACR Container Name       -> $BASE_IMAGE_NAME"
-    echo "Base ACR Container Tag        -> $BASE_IMAGE_TAG"
-    echo "Azure Linux Version           -> $AZURE_LINUX_VERSION"
-    echo "Distro Identifier             -> $DISTRO_IDENTIFIER"
-    echo "End of Life                   -> $END_OF_LIFE_1_YEAR"
-}
-
 function get_packages_to_install {
     echo "+++ Get packages to install"
     packagesFilePath="$CONTAINER_SRC_DIR/$IMAGE/$PACKAGE_FILE"
@@ -269,14 +248,14 @@ function docker_build {
     echo "docker buildx build $DOCKER_BUILD_ARGS" \
     "--build-arg BASE_IMAGE=$BASE_IMAGE_NAME_FULL" \
     "--build-arg RPMS_TO_INSTALL=$PACKAGES_TO_INSTALL" \
-    "-t $GOLDEN_IMAGE_NAME --no-cache --progress=plain" \
+    "-t $CONTAINER_IMAGE_NAME --no-cache --progress=plain" \
     "-f $WORK_DIR/Dockerfile ."
 
     echo ""
     docker buildx build $DOCKER_BUILD_ARGS \
         --build-arg BASE_IMAGE="$BASE_IMAGE_NAME_FULL" \
         --build-arg RPMS_TO_INSTALL="$PACKAGES_TO_INSTALL" \
-        -t "$GOLDEN_IMAGE_NAME" --no-cache --progress=plain \
+        -t "$CONTAINER_IMAGE_NAME" --no-cache --progress=plain \
         -f "$WORK_DIR/Dockerfile" .
     popd > /dev/null
 }
@@ -286,7 +265,7 @@ function set_image_tag {
     local containerId
     local installedPackage
 
-    containerId=$(docker run --entrypoint /bin/sh -dt "$GOLDEN_IMAGE_NAME")
+    containerId=$(docker run --entrypoint /bin/sh -dt "$CONTAINER_IMAGE_NAME")
 
     echo "Container ID                  -> $containerId"
 
@@ -312,77 +291,14 @@ function set_image_tag {
     # Rename the image to include package version
     # For HCI Images, do not include "-$DISTRO_IDENTIFIER" in the image tag; Instead use a "."
     if [ "$IS_HCI_IMAGE" = true ]; then
-        # Example: acrafoimages.azurecr.io/base/kubevirt/virt-operator:0.59.0-2.3.0.20240101-amd64
-        GOLDEN_IMAGE_NAME_FINAL="$GOLDEN_IMAGE_NAME:$COMPONENT_VERSION.$BASE_IMAGE_TAG"
+        # Example: acrafoimages.azurecr.io/base/kubevirt/virt-operator:0.59.0-2.2.0.20230607-amd64
+        CONTAINER_IMAGE_NAME_FINAL="$CONTAINER_IMAGE_NAME:$COMPONENT_VERSION.$BASE_IMAGE_TAG"
     else
-        # Example: azurelinuxpreview.azurecr.io/base/nodejs:18.18.2-2-$DISTRO_IDENTIFIER3.0.20240101-amd64
-        GOLDEN_IMAGE_NAME_FINAL="$GOLDEN_IMAGE_NAME:$COMPONENT_VERSION-$DISTRO_IDENTIFIER$BASE_IMAGE_TAG"
+        # Example: azurelinuxpreview.azurecr.io/base/nodejs:16.19.1-2-$DISTRO_IDENTIFIER2.0.20230607-amd64
+        CONTAINER_IMAGE_NAME_FINAL="$CONTAINER_IMAGE_NAME:$COMPONENT_VERSION-$DISTRO_IDENTIFIER$BASE_IMAGE_TAG"
     fi
 }
 
-function finalize {
-    echo "+++ Finalize"
-    docker image tag "$GOLDEN_IMAGE_NAME" "$GOLDEN_IMAGE_NAME_FINAL"
-    docker rmi -f "$GOLDEN_IMAGE_NAME"
-    echo "+++ Save container image name to file PublishedContainers-$IMAGE.txt"
-    echo "$GOLDEN_IMAGE_NAME_FINAL" >> "$OUTPUT_DIR/PublishedContainers-$IMAGE.txt"
-}
-
-function oras_attach {
-    local image_name=$1
-    oras attach \
-        --artifact-type "application/vnd.microsoft.artifact.lifecycle" \
-        --annotation "vnd.microsoft.artifact.lifecycle.end-of-life.date=$END_OF_LIFE_1_YEAR" \
-        "$image_name"
-}
-
-function publish_to_acr {
-    CONTAINER_IMAGE=$1
-    if [[ ! "$PUBLISH_TO_ACR" =~ [Tt]rue ]]; then
-        echo "+++ Skip publishing to ACR"
-        return
-    fi
-    local oras_access_token
-
-    echo "+++ az login into Azure ACR $ACR"
-    oras_access_token=$(az acr login --name "$ACR" --expose-token --output tsv --query accessToken)
-    oras login "$ACR.azurecr.io" \
-        --username "00000000-0000-0000-0000-000000000000" \
-        --password "$oras_access_token"
-
-    echo "+++ Publish container $CONTAINER_IMAGE"
-    docker image push "$CONTAINER_IMAGE"
-    oras_attach "$CONTAINER_IMAGE"
-}
-
-function generate_image_sbom {
-    if [[ ! "$CREATE_SBOM" =~ [Tt]rue ]]; then
-        echo "+++ Skip creating SBOM"
-        return
-    fi
-
-    echo "+++ Generate SBOM for the container image"
-    echo "Sanitized image name has '/' replaced with '-' and ':' replaced with '_'."
-    GOLDEN_IMAGE_NAME_SANITIZED=$(echo "$GOLDEN_IMAGE_NAME_FINAL" | tr '/' '-' | tr ':' '_')
-    echo "GOLDEN_IMAGE_NAME_SANITIZED   -> $GOLDEN_IMAGE_NAME_SANITIZED"
-
-    DOCKER_BUILD_DIR=$(mktemp -d)
-    # SBOM script will create the SBOM at the following path.
-    IMAGE_SBOM_MANIFEST_PATH="$DOCKER_BUILD_DIR/_manifest/spdx_2.2/manifest.spdx.json"
-    /bin/bash "$SBOM_SCRIPT" \
-        "$DOCKER_BUILD_DIR" \
-        "$GOLDEN_IMAGE_NAME_FINAL" \
-        "$SBOM_TOOL_PATH" \
-        "$BASE_IMAGE_NAME-$COMPONENT" \
-        "$COMPONENT_VERSION-$DISTRO_IDENTIFIER$BASE_IMAGE_TAG"
-
-    SBOM_IMAGES_DIR="$OUTPUT_DIR/SBOM_IMAGES"
-    mkdir -p "$SBOM_IMAGES_DIR"
-    cp -v "$IMAGE_SBOM_MANIFEST_PATH" "$SBOM_IMAGES_DIR/$GOLDEN_IMAGE_NAME_SANITIZED.spdx.json"
-    echo "Generated SBOM:'$SBOM_IMAGES_DIR/$GOLDEN_IMAGE_NAME_SANITIZED.spdx.json'"
-    sudo rm -rf "$DOCKER_BUILD_DIR"
-}
-
 function distroless_container {
     if [[ ! "$DISTROLESS" =~ [Tt]rue ]]; then
             echo "+++ Skip creating distroless container"
@@ -394,6 +310,7 @@ function distroless_container {
     create_distroless_container
 }
 
+source "$CONTAINER_SRC_DIR/scripts/BuildContainerCommonSteps.sh"
 print_inputs
 validate_inputs
 initialization
@@ -404,6 +321,4 @@ prepare_docker_directory
 docker_build
 set_image_tag
 finalize
-publish_to_acr "$GOLDEN_IMAGE_NAME_FINAL"
-generate_image_sbom
 distroless_container