diff --git a/backend/Dockerfile b/backend/Dockerfile index a8708c5e3b3..60a31b3eb60 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -143,14 +143,14 @@ RUN mkdir logs VOLUME /app/logs # Setup Python -COPY python/ /app/python/ +COPY --chown=devlake:devlake python/ /app/python/ RUN python3 -m pip install --no-cache --upgrade pip setuptools && \ python3 -m pip install --no-cache -r python/requirements.txt && \ python3 -m pip install --upgrade pip -# Setup Python Poetry package manager -RUN curl -sSL https://install.python-poetry.org | python3 - --version 2.2.1 -ENV PATH="$PATH:/app/.local/bin" +# Setup Python package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL=/app/.local/bin sh +ENV PATH="/app/.local/bin:${PATH}" # Build Python plugins, make sure the scripts has execute permission # RUN find /app/python/ -name "*.sh" | xargs -I{} chmod +x {} @@ -182,4 +182,3 @@ USER devlake ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["lake"] - diff --git a/backend/Makefile b/backend/Makefile index 698f9915ec2..17f166e0bef 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -84,7 +84,7 @@ unit-test-go: scripts/unit-test-go.sh build-pydevlake: - poetry install -C python/pydevlake + sh python/uv.sh sync python/pydevlake unit-test-python: build-pydevlake sh python/build.sh python/test &&\ diff --git a/backend/python/plugins/azuredevops/build.sh b/backend/python/plugins/azuredevops/build.sh index f0db2fed087..43613c9878a 100755 --- a/backend/python/plugins/azuredevops/build.sh +++ b/backend/python/plugins/azuredevops/build.sh @@ -17,4 +17,4 @@ # cd "$(dirname "$0")" -poetry install +sh ../../uv.sh sync . diff --git a/backend/python/plugins/azuredevops/run.sh b/backend/python/plugins/azuredevops/run.sh index e1ea5dd32e8..deae743d954 100755 --- a/backend/python/plugins/azuredevops/run.sh +++ b/backend/python/plugins/azuredevops/run.sh @@ -17,4 +17,4 @@ # cd "$(dirname "$0")" -poetry run python azuredevops/main.py "$@" +sh ../../uv.sh python . azuredevops/main.py "$@" diff --git a/backend/python/run_tests.sh b/backend/python/run_tests.sh index cbd32a73d86..41c5982f092 100755 --- a/backend/python/run_tests.sh +++ b/backend/python/run_tests.sh @@ -15,14 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -cd "${0%/*}" # make sure we're in the correct dir +SCRIPT_DIR=$(CDPATH= cd -- "${0%/*}" && pwd) -for test_dir in $(find . -type f -name "*_test.py" | xargs dirname | sort -u); do +for test_dir in $(find "$SCRIPT_DIR" -path '*/.venv' -prune -o -type f -name "*_test.py" -print | xargs dirname | sort -u); do + project_dir=$(dirname "$test_dir") printf "Running Python tests in $test_dir\n" - cd $test_dir - poetry run pytest + sh "$SCRIPT_DIR/uv.sh" sync "$project_dir" + sh "$SCRIPT_DIR/uv.sh" pytest "$project_dir" "$test_dir" if [ $? != 0 ]; then exit 1 fi - cd - -done \ No newline at end of file +done diff --git a/backend/python/test/fakeplugin/build.sh b/backend/python/test/fakeplugin/build.sh index f0db2fed087..43613c9878a 100755 --- a/backend/python/test/fakeplugin/build.sh +++ b/backend/python/test/fakeplugin/build.sh @@ -17,4 +17,4 @@ # cd "$(dirname "$0")" -poetry install +sh ../../uv.sh sync . diff --git a/backend/python/test/fakeplugin/run.sh b/backend/python/test/fakeplugin/run.sh index e02a7ca2f06..77bcc448fa0 100755 --- a/backend/python/test/fakeplugin/run.sh +++ b/backend/python/test/fakeplugin/run.sh @@ -16,8 +16,5 @@ # limitations under the License. # -echo sys path $PATH >&2 -[ -n "$VIRTUAL_ENV" ] && echo "Using virtualenv: $VIRTUAL_ENV" >&2 && . "$VIRTUAL_ENV/bin/activate" - cd "$(dirname "$0")" -poetry run python fakeplugin/main.py "$@" +sh ../../uv.sh python . fakeplugin/main.py "$@" diff --git a/backend/python/uv.sh b/backend/python/uv.sh new file mode 100755 index 00000000000..9e9d7696f18 --- /dev/null +++ b/backend/python/uv.sh @@ -0,0 +1,113 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -eu + +resolve_path() { + CDPATH= cd -- "$1" && pwd +} + +ensure_uv() { + if command -v uv >/dev/null 2>&1; then + return 0 + fi + + uv_install_dir=${DEVLAKE_UV_INSTALL_DIR:-${HOME:-$(pwd)}/.local/bin} + if [ -x "$uv_install_dir/uv" ]; then + PATH="$uv_install_dir:$PATH" + export PATH + return 0 + fi + mkdir -p "$uv_install_dir" + curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL="$uv_install_dir" sh + PATH="$uv_install_dir:$PATH" + export PATH +} + +sync_project() { + project_dir=$(resolve_path "$1") + ensure_uv + cd "$project_dir" + if [ -d .venv ] && [ ! -x .venv/bin/python ]; then + rm -rf .venv + fi + if [ -x .venv/bin/python ] && ! .venv/bin/python -c "import sys" >/dev/null 2>&1; then + rm -rf .venv + fi + if [ ! -x .venv/bin/python ]; then + uv venv --python "${DEVLAKE_PYTHON_VERSION:-3.9}" .venv + fi + uv pip install --python .venv/bin/python -e . +} + +ensure_project_python() { + project_dir=$(resolve_path "$1") + if [ ! -x "$project_dir/.venv/bin/python" ]; then + sync_project "$project_dir" + else + ensure_uv + fi + printf '%s/.venv/bin/python\n' "$project_dir" +} + +run_python() { + project_dir=$(resolve_path "$1") + shift + python_bin=$(ensure_project_python "$project_dir") + exec "$python_bin" "$@" +} + +run_pytest() { + project_dir=$(resolve_path "$1") + shift + ensure_uv + python_bin=$(ensure_project_python "$project_dir") + uv pip install --python "$python_bin" pytest + exec "$python_bin" -m pytest "$@" +} + +usage() { + echo "Usage: $0 {sync|python|pytest} [args...]" >&2 + exit 1 +} + +command_name=${1:-} +[ -n "$command_name" ] || usage +shift + +case "$command_name" in + sync) + [ $# -eq 1 ] || usage + sync_project "$1" + ;; + python) + [ $# -ge 2 ] || usage + project_dir=$1 + shift + run_python "$project_dir" "$@" + ;; + pytest) + [ $# -ge 1 ] || usage + project_dir=$1 + shift + run_pytest "$project_dir" "$@" + ;; + *) + usage + ;; +esac diff --git a/devops/docker/lake-builder/Dockerfile b/devops/docker/lake-builder/Dockerfile index 61363f0bcff..bcd56208f18 100644 --- a/devops/docker/lake-builder/Dockerfile +++ b/devops/docker/lake-builder/Dockerfile @@ -71,6 +71,5 @@ ENV GOPATH=/go ENV GOROOT= ENV PATH=${GOPATH}/bin:${PATH} -# Python Poetry package manager -RUN curl -sSL https://install.python-poetry.org | python3 - -RUN ln -sf /root/.local/bin/poetry /usr/local/bin \ No newline at end of file +# Python package manager +RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_UNMANAGED_INSTALL=/usr/local/bin sh