build: upgrade to poetry v2 (#248)

Author: jamesgetx

.github/workflows/bk-storages.yml

@@ -16,24 +16,16 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: 3.11
-      - name: Format with isort
-        working-directory: sdks/bk-storages
-        run: |
-          pip install isort==5.12.0
-          isort . --settings-path=pyproject.toml
-      - name: Format with black
-        working-directory: sdks/bk-storages
-        run: |
-          pip install black==23.7.0 click==8.1.6
-          black . --config=pyproject.toml
-      - name: Lint with flake8
-        working-directory: sdks/bk-storages
-        run: |
-          pip install flake8==4.0.1 pyproject-flake8==0.0.1a5
-          pflake8 . --config=pyproject.toml
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+      - name: Install Poetry system-wide
+        # Use uv install to install poetry directly instead github action for simplicity
+        run: uv pip install --system poetry==2.1.3
+      - name: Ruff check
+        run: poetry install && poetry run ruff check ./sdks/bk-storages --config ./pyproject.toml
       - name: Lint with mypy
         working-directory: sdks/bk-storages
         run: |
@@ -43,21 +35,50 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11"]
         os: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v5
+      - name: Install Nox
+        run:  uv pip install --system nox==2025.11.12
+      - name: Show nox version
+        working-directory: sdks/bk-storages
+        run: nox --version
+      - name: Run tests on ${{ matrix.os }}
+        working-directory: sdks/bk-storages
+        run: nox --non-interactive --error-on-missing-interpreters --session "tests(python='${{ matrix.python-version }}')" -- --full-trace
+
+  build:
+    runs-on: macos-latest
+    if: github.event.release && contains(github.event.release.tag_name, 'bk-storages')
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
       - name: Set up Poetry
         uses: abatilo/actions-poetry@v2.3.0
         with:
-          poetry-version: 1.5.1
-      - name: Test bk-storages
+          poetry-version: 2.1.1
+      - name: Build bk-storages
         run: |
           cd sdks/bk-storages
           poetry install
-          poetry run pytest
+          poetry build
+          echo "${{ github.event.release.tag_name }} ${{ github.sha }}" > Release.txt
+          cat Release.txt
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          files: |
+            Release.txt
+            sdks/bk-storages/dist/*
+

sdks/bk-storages/.pre-commit-config.yaml

@@ -106,7 +106,7 @@ def upload_fileobj(self, fh, key: str, allow_overwrite: bool = True, **kwargs):
             logger.exception("Request success, but the server rejects the upload request.")
             raise UploadFailedError(key=key, src=src) from e
         except Exception as e:
-            logger.exception("An unexpected exception occurred, detail: %s", e)
+            logger.exception("An unexpected exception occurred")
             raise UploadFailedError(key=key, src=src) from e
 
     def download_file(self, key: str, filepath: PathLike, *args, **kwargs) -> PathLike:
@@ -146,7 +146,7 @@ def download_fileobj(self, key: str, fh, *args, **kwargs):
                 if chunk:
                     fh.write(chunk)
         except Exception as e:
-            logger.exception("File save failed, detail %s", e)
+            logger.exception("File save failed")
             raise DownloadFailedError(key=key, dest=dest) from e
 
     def delete_file(self, key: str, *args, **kwargs):
@@ -282,7 +282,7 @@ def __init__(self, name, storage: 'BKRepoStorage'):
 
     def _get_file(self):
         if self._file is None:
-            self._file = SpooledTemporaryFile()
+            self._file = SpooledTemporaryFile() # noqa: SIM115
             self._storage.client.download_fileobj(key=self.name, fh=self._file)
             self._file.seek(0)
             self._dirty = False

sdks/bk-storages/bkstorages/backends/bkrepo.py

@@ -116,7 +116,7 @@ def size(self):
 
     def _get_file(self):
         if self._file is None:
-            self._file = SpooledTemporaryFile(
+            self._file = SpooledTemporaryFile( # noqa: SIM115
                 max_size=self._storage.max_memory_size,
                 suffix=".RGWBoto3StorageFile",
                 dir=setting("FILE_UPLOAD_TEMP_DIR", None),
@@ -182,8 +182,7 @@ def close(self):
             # s3boto's behavior.
             parts = [{'ETag': part.e_tag, 'PartNumber': part.part_number} for part in self._multipart.parts.all()]
             self._multipart.complete(MultipartUpload={'Parts': parts})
-        else:
-            if self._multipart is not None:
+        elif self._multipart is not None:
                 self._multipart.abort()
         if self._file is not None:
             self._file.close()
@@ -343,6 +342,7 @@ def lookup_env(names):
                 value = os.environ.get(name)
                 if value:
                     return value
+            return None
 
         access_key = self.access_key or lookup_env(self.access_key_names)
         secret_key = self.secret_key or lookup_env(self.secret_key_names)
@@ -476,19 +476,21 @@ def delete(self, name):
     def exists(self, name):
         if not name:
             try:
-                self.bucket
-                return True
+                _ = self.bucket
             except ImproperlyConfigured:
                 return False
+            else:
+                return True
         name = self._normalize_name(clean_name(name))
         if self.entries:
             return name in self.entries
         obj = self.bucket.Object(self._encode_name(name))
         try:
             obj.load()
-            return True
         except self.connection_response_error:
             return False
+        else:
+            return True
 
     def listdir(self, name):
         name = self._normalize_name(clean_name(name))

sdks/bk-storages/bkstorages/backends/rgw.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+import os
+import tempfile
+
+import nox
+
+ALL_PYTHON = ["3.9", "3.10", "3.11"]
+
+
+# ref: https://stackoverflow.com/questions/59768651/how-to-use-nox-with-poetry
+def install_with_constraints(session, *args: str, **kwargs) -> None:
+    """
+    Install packages constrained by Poetry's lock file.
+
+    This function is a wrapper for nox.sessions.Session.install. It
+    invokes pip to install packages inside of the session's virtualenv.
+    Additionally, pip is passed a constraints file generated from
+    Poetry's lock file, to ensure that the packages are pinned to the
+    versions specified in poetry.lock. This allows you to manage the
+    packages as Poetry development dependencies.
+
+    Arguments:
+        session: The Session object.
+        args: Command-line arguments for pip.
+        kwargs: Additional keyword arguments for Session.install.
+    """
+    req_path = os.path.join(tempfile.gettempdir(), os.urandom(24).hex())
+    session.run(
+        "poetry",
+        "export",
+        "--with=dev",
+        "--format=requirements.txt",
+        f"--output={req_path}",
+        external=True,
+    )
+    session.install(f"--constraint={req_path}", "--use-deprecated=legacy-resolver", *args, **kwargs)
+    os.unlink(req_path)
+
+
+@nox.session(reuse_venv=True)
+@nox.parametrize("python", ALL_PYTHON)
+def tests(session):
+    # Prepare pip and poetry
+    session.run("python", "-m", "ensurepip", "--upgrade")
+    session.install("poetry")
+    session.run("poetry", "self", "add", "poetry-plugin-export@latest")
+
+    # Install dev/test dependencies
+    session.install("-e", ".[all]")
+    install_with_constraints(
+        session,
+        "pytest",
+        "django",
+        "pytest-django",
+        "pytest-mock",
+        "moto[s3]==3.1.3",
+        "requests-mock",
+        "boto3==1.21.32"
+    )
+    session.run("pytest", *session.posargs)