fix: bKGenericRepoClient not support endpoint_url with url path (#247)

Author: jamesgetx

sdks/bk-storages/CHANGE.md

@@ -1,5 +1,9 @@
 ## Change logs
 
+### 2.0.2
+
+- 修复 `BKGenericRepoClient` 不支持 endpoint_url 带子路径的问题 
+
 ### 2.0.1
 
 - 修复 `BKGenericRepoClient.list_dir` 方法在请求 bkrepo 分页查询时，参数名错误的问题

sdks/bk-storages/bkstorages/__init__.py

@@ -8,4 +8,4 @@
  * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
  * specific language governing permissions and limitations under the License.
 """
-__version__ = "2.0.1"
+__version__ = "2.0.2"

sdks/bk-storages/bkstorages/backends/bkrepo.py

@@ -24,7 +24,6 @@
 from django.utils.timezone import localtime
 from requests.adapters import HTTPAdapter
 from requests.auth import HTTPBasicAuth
-from six.moves.urllib_parse import urljoin
 
 from bkstorages.exceptions import DownloadFailedError, ObjectAlreadyExists, RequestError, UploadFailedError
 from bkstorages.utils import clean_name, get_available_overwrite_name, get_setting, safe_join, setting
@@ -92,7 +91,7 @@ def upload_fileobj(self, fh, key: str, allow_overwrite: bool = True, **kwargs):
         :param bool allow_overwrite: 是否覆盖已存在文件
         """
         client = self.get_client()
-        url = urljoin(self.endpoint_url, f'/generic/{self.project}/{self.bucket}/{key}')
+        url = safe_join(self.endpoint_url, f'generic/{self.project}/{self.bucket}/{key}')
         src = getattr(fh, "name", "<memory>")
         headers = {"X-BKREPO-OVERWRITE": str(allow_overwrite)}
 
@@ -127,7 +126,7 @@ def download_fileobj(self, key: str, fh, *args, **kwargs):
         :param IO fh: 文件句柄
         """
         client = self.get_client()
-        url = urljoin(self.endpoint_url, f'/generic/{self.project}/{self.bucket}/{key}')
+        url = safe_join(self.endpoint_url, f'generic/{self.project}/{self.bucket}/{key}')
         dest = getattr(fh, "name", "<memory>")
         try:
             resp = client.get(url, stream=True, timeout=TIMEOUT_THRESHOLD)
@@ -156,14 +155,14 @@ def delete_file(self, key: str, *args, **kwargs):
         :param str key: 文件完整路径
         """
         client = self.get_client()
-        url = urljoin(self.endpoint_url, f'/generic/{self.project}/{self.bucket}/{key}')
+        url = safe_join(self.endpoint_url, f'generic/{self.project}/{self.bucket}/{key}')
         resp = client.delete(url, timeout=TIMEOUT_THRESHOLD)
         self._validate_resp(resp)
 
     def get_file_metadata(self, key: str, *args, **kwargs) -> Dict:
         """具体返回值请看 bk-repo 的文档."""
         client = self.get_client()
-        url = urljoin(self.endpoint_url, f'/generic/{self.project}/{self.bucket}/{key}')
+        url = safe_join(self.endpoint_url, f'generic/{self.project}/{self.bucket}/{key}')
         resp = client.head(url, timeout=TIMEOUT_THRESHOLD)
         if resp.status_code == 200:
             return dict(resp.headers)
@@ -178,7 +177,7 @@ def build_download_url(self, key: str, force_download: bool = False) -> str:
         while key.startswith("/"):
             key = key[1:]
         download = "true" if force_download else "false"
-        url = urljoin(self.endpoint_url, f'/generic/{self.project}/{self.bucket}/{key}?download={download}')
+        url = safe_join(self.endpoint_url, f'generic/{self.project}/{self.bucket}/{key}?download={download}')
         return url
 
     def generate_presigned_url(self, key: str, expires_in: int, token_type: str = "DOWNLOAD", *args, **kwargs) -> str:
@@ -189,7 +188,7 @@ def generate_presigned_url(self, key: str, expires_in: int, token_type: str = "D
         :param str token_type: token类型。UPLOAD:允许上传, DOWNLOAD: 允许下载, ALL: 同时允许上传和下载。
         """
         client = self.get_client()
-        url = urljoin(self.endpoint_url, '/generic/temporary/url/create')
+        url = safe_join(self.endpoint_url, 'generic/temporary/url/create')
 
         resp = client.post(
             url,
@@ -210,7 +209,7 @@ def generate_presigned_url(self, key: str, expires_in: int, token_type: str = "D
             if str(e.code) != '250102':
                 raise
             logger.warning("BKREPO中不存在该文件, 避免报错仅拼接 url ")
-            return urljoin(self.endpoint_url, f'/generic/temporary/token/download/{self.project}/{self.bucket}/{key}')
+            return safe_join(self.endpoint_url, f'generic/temporary/token/download/{self.project}/{self.bucket}/{key}')
 
     def list_dir(self, key_prefix: str) -> Tuple[List, List]:
         """
@@ -234,7 +233,7 @@ def __list_dir(self, key_prefix: str, cur_page: int = 1) -> Tuple[List, List, bo
         """
         directories, files = [], []
         client = self.get_client()
-        url = urljoin(self.endpoint_url, f"/repository/api/node/page/{self.project}/{self.bucket}/{key_prefix}")
+        url = safe_join(self.endpoint_url, f"repository/api/node/page/{self.project}/{self.bucket}/{key_prefix}")
         # NOTE: 按分页查询 bkrepo 的文件数, 1000 是一个经验值, 设置仅可能大的数值是避免发送太多次请求到 bk-repo
         params = {"pageSize": 1000, "pageNumber": cur_page, "includeFolder": True}
         resp = client.get(url, params=params, timeout=TIMEOUT_THRESHOLD)
@@ -250,7 +249,7 @@ def __list_dir(self, key_prefix: str, cur_page: int = 1) -> Tuple[List, List, bo
     def get_bucket_metadata(self) -> Dict:
         """查询仓库信息"""
         client = self.get_client()
-        url = urljoin(self.endpoint_url, f"/repository/api/repo/info/{self.project}/{self.bucket}/GENERIC")
+        url = safe_join(self.endpoint_url, f"repository/api/repo/info/{self.project}/{self.bucket}/GENERIC")
         resp = client.get(url)
         data = self._validate_resp(resp)
         return data

sdks/bk-storages/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bkstorages"
-version = "2.0.1"
+version = "2.0.2"
 description = "File storage backends for blueking PaaS platform"
 readme = "README.md"
 authors = ["blueking <blueking@tencent.com>"]

sdks/bk-storages/tests/test_bkrepo.py

@@ -17,9 +17,9 @@
 import requests_mock
 from django.core.exceptions import SuspiciousFileOperation
 from django.core.files.base import File
-from six.moves.urllib_parse import urljoin
 
 from bkstorages.backends.bkrepo import BKGenericRepoClient, BKRepoFile, BKRepoStorage, parse_gmt_datetime
+from bkstorages.utils import safe_join
 from tests.utils import generate_random_string
 
 
@@ -67,11 +67,11 @@ def fake_upload(request):
             return True
 
         g = adapter.register_uri(
-            "GET", urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{key}"), body=io.BytesIO(content)
+            "GET", safe_join(endpoint, f"generic/dummy-project/dummy-bucket/{key}"), body=io.BytesIO(content)
         )
         adapter.register_uri(
             "PUT",
-            urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{key}"),
+            safe_join(endpoint, f"generic/dummy-project/dummy-bucket/{key}"),
             json={"code": 0, "message": ""},
             additional_matcher=fake_upload,
         )
@@ -131,9 +131,9 @@ def test_list_dir(self, bk_repo_storage, adapter, endpoint, mock_responses, expe
         for idx, records in enumerate(mock_responses):
             adapter.register_uri(
                 "GET",
-                urljoin(
+                safe_join(
                     endpoint,
-                    f"/repository/api/node/page/dummy-project/dummy-bucket/"
+                    f"repository/api/node/page/dummy-project/dummy-bucket/"
                     f"{path}?pageSize=1000&pageNumber={idx + 1}&includeFolder=True",
                 ),
                 json={"code": 0, "message": 0, "data": {"totalPages": len(mock_responses), "records": records}},
@@ -149,7 +149,7 @@ def test_save_without_name(self, bk_repo_storage, adapter, endpoint):
         content = b"aaa"
         fh = File(io.BytesIO(content))
         adapter.register_uri(
-            "PUT", urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{filename}"), json={"code": 0}
+            "PUT", safe_join(endpoint, f"generic/dummy-project/dummy-bucket/{filename}"), json={"code": 0}
         )
         assert bk_repo_storage.save(filename, fh)
         assert adapter.called
@@ -174,7 +174,7 @@ def test_save_with_name(self, root_path, name, has_error, expected_key, bk_repo_
         content = b"import this"
         fh = File(io.BytesIO(content), name="foobar.py")
         adapter.register_uri(
-            "PUT", urljoin(endpoint, f"/generic/dummy-project/dummy-bucket/{expected_key}"), json={"code": 0}
+            "PUT", safe_join(endpoint, f"generic/dummy-project/dummy-bucket/{expected_key}"), json={"code": 0}
         )
         if has_error:
             with pytest.raises(SuspiciousFileOperation):