Skip to content

Reuse locked CI dependencies across workspace source changes #74466

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zozo123 wants to merge 7 commits into
base: main
Choose a base branch
from
Open

Reuse locked CI dependencies across workspace source changes #74466

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
062b745
Reuse locked CI dependencies across workspace source changes
zozo123 Oct 8, 2026
4bb265f
Skip the locked dependency pre-install when uv.lock does not fit
zozo123 Oct 18, 2026
8b6de6b
Merge branch 'main' into ci/ci-image-locked-dependency-layer
zozo123 Oct 18, 2026
ad2ceda
Run the locked dependency pre-install through the install script
zozo123 Oct 19, 2026
9f2b0c8
Skip the locked dependency pre-install only when uv.lock does not fit
zozo123 Oct 19, 2026
e314bcd
Merge branch 'main' into ci/ci-image-locked-dependency-layer
zozo123 Oct 19, 2026
eb931fa
Merge branch 'main' into ci/ci-image-locked-dependency-layer
zozo123 Oct 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 36 additions & 7 deletions Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1255,6 +1255,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
extra_sync_flags="${extra_sync_flags} --no-install-workspace"
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
Expand All @@ -1275,19 +1284,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
# shellcheck disable=SC2206 # extra_sync_flags holds several flags
local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec
--no-python-downloads --no-managed-python)
set +x
# Without the workspace sources a lock that does not match the manifests cannot be re-resolved;
# the installation that adds the workspace packages does that later.
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then
echo
echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
set -x
if ! uv sync --all-packages --frozen --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
if ! "${frozen_sync[@]}"; then
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
echo
echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}"
echo
exit 1
fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand Down Expand Up @@ -1377,6 +1402,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
# Packaging tools and 'pip check' are handled by the installation that adds the workspace packages
return
fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
71 changes: 63 additions & 8 deletions Dockerfile.ci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -957,6 +957,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
extra_sync_flags="${extra_sync_flags} --no-install-workspace"
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
Expand All @@ -977,19 +986,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
# shellcheck disable=SC2206 # extra_sync_flags holds several flags
local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec
--no-python-downloads --no-managed-python)
set +x
# Without the workspace sources a lock that does not match the manifests cannot be re-resolved;
# the installation that adds the workspace packages does that later.
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then
echo
echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
set -x
if ! uv sync --all-packages --frozen --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
if ! "${frozen_sync[@]}"; then
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
echo
echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}"
echo
exit 1
fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand Down Expand Up @@ -1079,6 +1104,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
# Packaging tools and 'pip check' are handled by the installation that adds the workspace packages
return
fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down Expand Up @@ -1713,6 +1742,22 @@ fi
exec /bin/bash "${@}"
EOF

# Extract manifests separately so COPY below depends on their contents, not all source files.
FROM ${BASE_IMAGE} as dependency-manifests
WORKDIR /source
SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-o", "nounset", "-o", "nolog", "-c"]
RUN --mount=type=bind,target=/source \
manifest_list="$(mktemp)" && \
trap 'rm -f "${manifest_list}"' EXIT && \
find . -name pyproject.toml -type f -print0 > "${manifest_list}" && \
mkdir -p /dependency-manifests && \
while IFS= read -r -d '' manifest; do \
relative_path="${manifest#./}"; \
mkdir -p "/dependency-manifests/$(dirname "${relative_path}")" || exit 1; \
cp "${manifest}" "/dependency-manifests/${relative_path}" || exit 1; \
done < "${manifest_list}" && \
if [ -f uv.lock ]; then cp uv.lock /dependency-manifests/ || exit 1; fi

FROM ${BASE_IMAGE} as main

# Nolog bash flag is currently ignored - but you can replace it with other flags (for example
Expand Down Expand Up @@ -1945,13 +1990,23 @@ RUN bash /scripts/docker/install_packaging_tools.sh

COPY --from=scripts install_airflow_when_building_images.sh /scripts/docker/

COPY --from=dependency-manifests /dependency-manifests/ ${AIRFLOW_SOURCES}/
ARG UPGRADE_RANDOM_INDICATOR_STRING=""
# Only preinstall locked third-party dependencies, with the same uv sync as the full installation.
# A uv.lock that does not match the manifests only skips this warm-up: the full sync after COPY still
# installs workspace packages from the current checkout and retains its resolution fallback. Any other
# failure would fail the full sync as well, so it fails the build here.
RUN --mount=type=cache,id=ci-$TARGETARCH-$DEPENDENCY_CACHE_EPOCH,target=/root/.cache/ \
if [[ -z "${UPGRADE_RANDOM_INDICATOR_STRING}" ]]; then \
INSTALL_LOCKED_THIRD_PARTY_ONLY="true" bash /scripts/docker/install_airflow_when_building_images.sh; \
fi

# We can copy everything here. The Context is filtered by dockerignore. This makes sure we are not
# copying over stuff that is accidentally generated or that we do not need (such as egg-info)
# if you want to add something that is missing and you expect to see it in the image you can
# add it with ! in .dockerignore next to the airflow, test etc. directories there
COPY . ${AIRFLOW_SOURCES}/

ARG UPGRADE_RANDOM_INDICATOR_STRING=""
ARG VERSION_SUFFIX=""

ENV UPGRADE_RANDOM_INDICATOR_STRING=${UPGRADE_RANDOM_INDICATOR_STRING} \
Expand Down
48 changes: 41 additions & 7 deletions scripts/docker/install_airflow_when_building_images.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
# UPGRADE_RANDOM_INDICATOR_STRING - if set with random value determines that dependency upgrade should be done
# with highest resolution. It is random in order to trigger cache
# invalidation when building images
# INSTALL_LOCKED_THIRD_PARTY_ONLY - if "true", only the third-party dependencies locked in uv.lock are installed
# from sources (--no-install-workspace), so only the pyproject.toml files
# and uv.lock are needed. When a frozen dry run shows that uv.lock does not
# match them, or for upgrade builds, nothing is installed; any other
# failure fails. The steps that need the workspace packages are skipped.
#
# shellcheck shell=bash disable=SC2086
# shellcheck source=scripts/docker/common.sh
Expand All @@ -41,6 +46,15 @@ function install_from_sources() {
if [[ ${VIRTUAL_ENV=} != "" ]]; then
extra_sync_flags="--active"
fi
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
echo
echo "${COLOR_YELLOW}Upgrade builds resolve dependencies anew; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
extra_sync_flags="${extra_sync_flags} --no-install-workspace"
Comment thread
zozo123 marked this conversation as resolved.
fi
if [[ "${UPGRADE_RANDOM_INDICATOR_STRING=}" != "" ]]; then
if [[ ${PACKAGING_TOOL_CMD} == "pip" ]]; then
set +x
Expand All @@ -61,19 +75,35 @@ function install_from_sources() {
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python
else
set +x
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
# Use uv sync --frozen to install exactly what is pinned in uv.lock without re-resolving.
# --no-binary-package is needed in order to avoid libxml and xmlsec using different version of
# libxml2 (binary lxml embeds its own libxml2, while xmlsec uses system one).
# See https://bugs.launchpad.net/lxml/+bug/2110068
# shellcheck disable=SC2206 # extra_sync_flags holds several flags
local -a frozen_sync=(uv sync --all-packages --frozen --group ci-image
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec
--no-python-downloads --no-managed-python)
set +x
# Without the workspace sources a lock that does not match the manifests cannot be re-resolved;
# the installation that adds the workspace packages does that later.
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]] && ! "${frozen_sync[@]}" --dry-run; then
echo
echo "${COLOR_YELLOW}uv.lock does not match the dependency manifests; skipping the locked third-party installation.${COLOR_RESET}"
echo
return
fi
echo
echo "${COLOR_BLUE}Installing all packages from uv.lock (frozen).${COLOR_RESET}"
echo
set -x
if ! uv sync --all-packages --frozen --group ci-image \
${extra_sync_flags} --no-binary-package lxml --no-binary-package xmlsec \
--no-python-downloads --no-managed-python; then
if ! "${frozen_sync[@]}"; then
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
echo
echo "${COLOR_RED}Failing because the locked third-party installation failed although uv.lock matches the dependency manifests.${COLOR_RESET}"
echo
exit 1
fi
if [[ ${AIRFLOW_FALLBACK_NO_CONSTRAINTS_INSTALLATION} != "true" ]]; then
echo
echo "${COLOR_RED}Failing because frozen uv.lock installation failed and fallback is disabled.${COLOR_RESET}"
Expand Down Expand Up @@ -163,6 +193,10 @@ function install_airflow_when_building_images() {
install_from_external_spec
fi
set +x
if [[ ${INSTALL_LOCKED_THIRD_PARTY_ONLY=} == "true" ]]; then
# Packaging tools and 'pip check' are handled by the installation that adds the workspace packages
return
fi
common::install_packaging_tools
echo
echo "${COLOR_BLUE}Running 'pip check'${COLOR_RESET}"
Expand Down
Loading