Storages: do not load image to memory before uploading, remove zone default

Author: Antti Myyrä

upcloud_api/cloud_manager/storage_mixin.py

@@ -1,9 +1,9 @@
 from typing import Optional, Union
+from pathlib import Path
 
 from upcloud_api.api import API
 from upcloud_api.storage import Storage
 from upcloud_api.storage_import import StorageImport
-from upcloud_api.utils import get_raw_data_from_file
 
 
 class StorageManager:
@@ -41,10 +41,10 @@ def get_storage(self, storage: str) -> Storage:
 
     def create_storage(
         self,
+        zone: str,
         size: int = 10,
         tier: str = 'maxiops',
         title: str = 'Storage disk',
-        zone: str = 'fi-hel1',
         backup_rule: Optional[dict] = None,
     ) -> Storage:
         """
@@ -184,33 +184,37 @@ def create_storage_import(
         Creates an import task to import data into an existing storage.
         Source types: http_import or direct_upload.
         """
+        if source not in ["http_import", "direct_upload"]:
+            raise Exception("invalid storage import source: %s", source)
+
         url = f'/storage/{storage}/import'
         body = {'storage_import': {'source': source}}
         if source_location:
             body['storage_import']['source_location'] = source_location
         res = self.api.post_request(url, body)
         return StorageImport(**res['storage_import'])
 
-    def upload_file_for_storage_import(self, storage_import, file):
+    def upload_file_for_storage_import(self, storage_import: StorageImport, file: Path, timeout: int = 600):
         """
         Uploads a file directly to UpCloud's uploader session.
         """
-        # TODO: this should not buffer the entire `file` into memory
-
-        # This is importing and using `requests` directly since there doesn't
-        # seem to be a point in adding a `.api.raw_request()` call to the `API` class.
-        # That could be changed.
 
         import requests
 
-        resp = requests.put(
-            url=storage_import.direct_upload_url,
-            data=get_raw_data_from_file(file),
-            headers={'Content-type': 'application/octet-stream'},
-            timeout=600,
-        )
-        resp.raise_for_status()
-        return resp.json()
+        # This is importing and using `requests` directly since there doesn't
+        # seem to be a point in adding a `.api.raw_request()` call to the `API` class.
+        # That could be changed if there starts to be more of these cases.
+
+        with open(file, 'rb') as f:
+            resp = requests.put(
+                url=storage_import.direct_upload_url,
+                data=f,
+                headers={'Content-type': 'application/octet-stream'},
+                timeout=timeout,
+            )
+
+            resp.raise_for_status()
+            return resp.json()
 
     def get_storage_import_details(self, storage: str) -> StorageImport:
         """

upcloud_api/utils.py

@@ -23,12 +23,3 @@ def try_it_n_times(operation, expected_error_codes, custom_error='operation fail
         if i >= n - 1:
             raise UpCloudClientError(custom_error)
 
-
-def get_raw_data_from_file(file: str) -> bytes:
-    """
-    Helper function to get raw file data for uploading.
-    """
-    with open(file, 'rb') as file:
-        data = file.read()
-    file.close()
-    return data