diff --git a/Dockerfile b/Dockerfile index 25b6fa327ae2b..a6e2507df4a11 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1255,6 +1255,15 @@ function install_from_sources() { if [[ ${VIRTUAL_ENV=} != "" ]]; then extra_sync_flags="--active" fi + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then + echo + echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + extra_sync_flags="${extra_sync_flags} --no-install-workspace" + fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then set +x @@ -1275,19 +1284,35 @@ function install_from_sources() { ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python else - set +x - echo - echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" - echo # Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving. # --no-binary-package is needed in order to avoid libxml and xmlsec using different version of # libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 + # shellcheck disable=SC2206 # extra_sync_flags holds several flags + local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image + ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec + --no-python-downloads --no-managed-python) + set +x + # Without the workspace sources a lock that does not match the manifests cannot be re-resolved; + # the installation that adds the workspace packages does that later. + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then + echo + echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + echo + echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" + echo set -x - if ! uv sync --all-packages --frozen --group ci-image \ - ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ - --no-python-downloads --no-managed-python; then + if ! "${frozen_sync[@]}"; then set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + echo + echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}" + echo + exit 1 + fi if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}" @@ -1377,6 +1402,10 @@ function install_airflow_when_building_images() { install_from_external_spec fi set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + # Packaging tools and 'pip check' are handled by the installation that adds the workspace packages + return + fi common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" diff --git a/Dockerfile.ci b/Dockerfile.ci index 23b13c180fd1d..4db0ac663b1d0 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -957,6 +957,15 @@ function install_from_sources() { if [[ ${VIRTUAL_ENV=} != "" ]]; then extra_sync_flags="--active" fi + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then + echo + echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + extra_sync_flags="${extra_sync_flags} --no-install-workspace" + fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then set +x @@ -977,19 +986,35 @@ function install_from_sources() { ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python else - set +x - echo - echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" - echo # Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving. # --no-binary-package is needed in order to avoid libxml and xmlsec using different version of # libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 + # shellcheck disable=SC2206 # extra_sync_flags holds several flags + local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image + ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec + --no-python-downloads --no-managed-python) + set +x + # Without the workspace sources a lock that does not match the manifests cannot be re-resolved; + # the installation that adds the workspace packages does that later. + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then + echo + echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + echo + echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" + echo set -x - if ! uv sync --all-packages --frozen --group ci-image \ - ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ - --no-python-downloads --no-managed-python; then + if ! "${frozen_sync[@]}"; then set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + echo + echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}" + echo + exit 1 + fi if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}" @@ -1079,6 +1104,10 @@ function install_airflow_when_building_images() { install_from_external_spec fi set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + # Packaging tools and 'pip check' are handled by the installation that adds the workspace packages + return + fi common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}" @@ -1713,6 +1742,22 @@ fi exec /bin/bash "${@}" EOF +# Extract manifests separately so COPY below depends on their contents, not all source files. +FROM ${BASE_IMAGE} as dependency-manifests +WORKDIR /source +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"] +RUN --mount=type=bind,target=/source \ + manifest_list="$(mktemp)" && \ + trap 'rm -f "${manifest_list}"' EXIT && \ + find . -name pyproject.toml -type f -print0 > "${manifest_list}" && \ + mkdir -p /dependency-manifests && \ + while IFS= read -r -d '' manifest; do \ + relative_path="${manifest#./}"; \ + mkdir -p "/dependency-manifests/$(dirname "${relative_path}")" || exit 1; \ + cp "${manifest}" "/dependency-manifests/${relative_path}" || exit 1; \ + done < "${manifest_list}" && \ + if [ -f uv.lock ]; then cp uv.lock /dependency-manifests/ || exit 1; fi + FROM ${BASE_IMAGE} as main # Nolog bash flag is currently ignored - but you can replace it with other flags (for example @@ -1945,13 +1990,23 @@ RUN bash /scripts/docker/install_packaging_tools.sh COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/ +COPY --from=dependency-manifests /dependency-manifests/ ${AIRFLOW_SOURCES}/ +ARG UPGRADE_RANDOM_INDICATOR_STRING="" +# Only preinstall locked third-party dependencies, with the same uv sync as the full installation. +# A uv.lock that does not match the manifests only skips this warm-up: the full sync after COPY still +# installs workspace packages from the current checkout and retains its resolution fallback. Any other +# failure would fail the full sync as well, so it fails the build here. +RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \ + if [[ -z "${UPGRADE_RANDOM_INDICATOR_STRING}" ]]; then \ + INSTALL_LOCKED_THIRD_PARTY_ONLY="true" bash /scripts/docker/install_airflow_when_building_images.sh; \ + fi + # We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not # copying over stuff that is accidentally generated or that we do not need (such as egg-info) # if you want to add something that is missing and you expect to see it in the image you can # add it with ! in .dockerignore next to the airflow, test etc. directories there COPY . ${AIRFLOW_SOURCES}/ -ARG UPGRADE_RANDOM_INDICATOR_STRING="" ARG VERSION_SUFFIX="" ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \ diff --git a/scripts/docker/install_airflow_when_building_images.sh b/scripts/docker/install_airflow_when_building_images.sh index 772465678a8c4..45e60d5fdbf38 100644 --- a/scripts/docker/install_airflow_when_building_images.sh +++ b/scripts/docker/install_airflow_when_building_images.sh @@ -30,6 +30,11 @@ # UPGRADE_RANDOM_INDICATOR_STRING - if set with random value determines that dependency upgrade should be done # with highest resolution. It is random in order to trigger cache # invalidation when building images +# INSTALL_LOCKED_THIRD_PARTY_ONLY - if "true", only the third-party dependencies locked in uv.lock are installed +# from sources (--no-install-workspace), so only the pyproject.toml files +# and uv.lock are needed. When a frozen dry run shows that uv.lock does not +# match them, or for upgrade builds, nothing is installed; any other +# failure fails. The steps that need the workspace packages are skipped. # # shellcheck shell=bash disable=SC2086 # shellcheck source=scripts/docker/common.sh @@ -41,6 +46,15 @@ function install_from_sources() { if [[ ${VIRTUAL_ENV=} != "" ]]; then extra_sync_flags="--active" fi + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then + echo + echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + extra_sync_flags="${extra_sync_flags} --no-install-workspace" + fi if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then set +x @@ -61,19 +75,35 @@ function install_from_sources() { ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ --no-python-downloads --no-managed-python else - set +x - echo - echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" - echo # Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving. # --no-binary-package is needed in order to avoid libxml and xmlsec using different version of # libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one). # See https://bugs.launchpad.net/lxml/+bug/2110068 + # shellcheck disable=SC2206 # extra_sync_flags holds several flags + local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image + ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec + --no-python-downloads --no-managed-python) + set +x + # Without the workspace sources a lock that does not match the manifests cannot be re-resolved; + # the installation that adds the workspace packages does that later. + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then + echo + echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}" + echo + return + fi + echo + echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}" + echo set -x - if ! uv sync --all-packages --frozen --group ci-image \ - ${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \ - --no-python-downloads --no-managed-python; then + if ! "${frozen_sync[@]}"; then set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + echo + echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}" + echo + exit 1 + fi if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then echo echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}" @@ -163,6 +193,10 @@ function install_airflow_when_building_images() { install_from_external_spec fi set +x + if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then + # Packaging tools and 'pip check' are handled by the installation that adds the workspace packages + return + fi common::install_packaging_tools echo echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"