Merge pull request #70 from TechConsult/imports_and_documentation

Trdls · web-flow · commit dce6459e3366 · 2020-09-28T11:45:09.000+03:00
Imports and documentation
diff --git a/README.md b/README.md
@@ -50,13 +50,6 @@ python setup.py install
 * Manage firewall (on/off and individual rules)
 	* since 0.2: full management of firewall rules
 
-**TODO:**
-* Cloning of storages
-* Full management of special storage types:
-  * CDROMs, custom OS templates
-  * (custom templates can already be cloned to a disk via UUID)
-* Full management of backups (instant and scheduled)
-
 **Changelog:**
 * See the [Releases page](https://github.com/UpCloudLtd/upcloud-python-api/releases)
 
diff --git a/docs/storage-mixin.md b/docs/storage-mixin.md
@@ -62,4 +62,68 @@ def detach_storage(self, server_uuid, address):
 	"""
 	Detach a Storage object to a Server. Return a list of the server's storages.
 	"""
-```
+```
+
+```python
+def load_cd_rom(self, server, address):
+	"""
+	Loads a storage as a CD-ROM in the CD-ROM device of a server. Returns a list of the server's storages.
+	"""
+```
+
+```python
+def eject_cd_rom(self, server):
+	"""
+Ejects the storage from the CD-ROM device of a server. Returns a list of the server's storages.
+	"""
+```
+
+```python
+def create_storage_backup(self, storage, title):
+	"""
+Creates a point-in-time backup of a storage resource. Returns a storage object.
+	"""
+```
+
+```python
+def restore_storage_backup(self, storage):
+	"""
+Restores the origin storage with data from the specified backup storage. Returns a storage object.
+	"""
+```
+
+```python
+def templatize_storage(self, storage, title):
+	"""
+Creates an exact copy of an existing storage resource which can be used as a template for creating new servers. Returns a storage object.
+	"""
+```
+
+```python
+def create_storage_import(self, storage, source, source_location=None):
+	"""
+	Creates an import task to import data into an existing storage and returns a storage import object.
+	Source types: http_import or direct_upload.
+	"""
+```
+
+```python
+def upload_file_for_storage_import(self, storage_import, file):
+	"""
+	Uploads a file directly to UpCloud's uploader session. Returns written bytes, md5sum and sha256sum.
+	"""
+```
+
+```python
+def get_storage_import_details(self, storage):
+	"""
+	Returns detailed information of an ongoing or finished import task within a storage import object.
+	"""
+```
+
+```python
+def cancel_storage_import(self, storage):
+	"""
+	Cancels an ongoing import task. Returns a storage import object.
+	"""
+```
diff --git a/test/conftest.py b/test/conftest.py
@@ -78,13 +78,14 @@ def callback(request):
                                content_type='application/json')
 
     @staticmethod
-    def mock_put(target):
+    def mock_put(target, ignore_data_field=False, empty_payload=False, call_api=True):
         data = json.loads(Mock.read_from_file(target + '.json'))
 
         def callback(request):
-            return Mock.__put_post_callback(request, target, data)
+            return Mock.__put_post_callback(request, target, data, ignore_data_field, empty_payload)
 
-        responses.add_callback(responses.PUT, Mock.base_url + '/' + target,
+        url = Mock.base_url + '/' + target if call_api else target
+        responses.add_callback(responses.PUT, url,
                                callback=callback,
                                content_type='application/json')
 
diff --git a/test/json_data/https:__fi-hel1.img.upcloud.com_uploader_session_07a6c9a3-300e-4d0e-b935-624f3dbdff3f.json b/test/json_data/https:__fi-hel1.img.upcloud.com_uploader_session_07a6c9a3-300e-4d0e-b935-624f3dbdff3f.json
@@ -0,0 +1 @@
+{"written_bytes":909500125,"md5sum":"5cc6f7e7a1c52303ac3137d62410eec5","sha256sum":"bdf14d897406939c11a73d0720ca75c709e756d437f8be9ee26af6b58ede3bd7"}
diff --git a/test/json_data/storage_01d4fcd4-e446-433b-8a9c-551a1284952e_import.json b/test/json_data/storage_01d4fcd4-e446-433b-8a9c-551a1284952e_import.json
@@ -0,0 +1,18 @@
+{
+  "storage_import": {
+    "client_content_length": 0,
+    "client_content_type": "",
+    "completed": "",
+    "created": "2020-06-26T08:51:07Z",
+    "error_code": "",
+    "error_message": "",
+    "md5sum": "",
+    "read_bytes": 0,
+    "sha256sum": "",
+    "source": "http_import",
+    "source_location": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-10.4.0-amd64-netinst.iso",
+    "state": "pending",
+    "uuid": "07a6c9a3-300e-4d0e-b935-624f3dbdff3f",
+    "written_bytes": 0
+  }
+}
diff --git a/test/json_data/storage_01d4fcd4-e446-433b-8a9c-551a1284952e_import_cancel_post.json b/test/json_data/storage_01d4fcd4-e446-433b-8a9c-551a1284952e_import_cancel_post.json
@@ -0,0 +1,18 @@
+{
+  "storage_import": {
+    "client_content_length": 0,
+    "client_content_type": "",
+    "completed": "",
+    "created": "2020-06-26T08:51:07Z",
+    "error_code": "",
+    "error_message": "",
+    "md5sum": "",
+    "read_bytes": 0,
+    "sha256sum": "",
+    "source": "http_import",
+    "source_location": "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-10.4.0-amd64-netinst.iso",
+    "state": "cancelling",
+    "uuid": "07a6c9a3-300e-4d0e-b935-624f3dbdff3f",
+    "written_bytes": 0
+  }
+}
diff --git a/test/json_data/storage_01d4fcd4-e446-433b-8a9c-551a1284952e_import_post.json b/test/json_data/storage_01d4fcd4-e446-433b-8a9c-551a1284952e_import_post.json
@@ -0,0 +1,18 @@
+{
+  "storage_import": {
+    "client_content_length": 0,
+    "client_content_type": "",
+    "completed": "",
+    "created": "2020-06-26T08:51:07Z",
+    "direct_upload_url": "https://fi-hel1.img.upcloud.com/uploader/session/07a6c9a3-300e-4d0e-b935-624f3dbdff3f",
+    "error_code": "",
+    "error_message": "",
+    "md5sum": "",
+    "read_bytes": 0,
+    "sha256sum": "",
+    "source": "direct_upload",
+    "state": "prepared",
+    "uuid": "07a6c9a3-300e-4d0e-b935-624f3dbdff3f",
+    "written_bytes": 0
+  }
+}
diff --git a/test/json_data/test_file.json b/test/json_data/test_file.json
@@ -0,0 +1 @@
+test file for upload test
diff --git a/test/test_storage.py b/test/test_storage.py
@@ -92,6 +92,37 @@ def test_templatize_storage(self, manager):
         assert storage.title == "my server template"
         assert storage.type == "template"
 
+    @responses.activate
+    def test_create_storage_import(self, manager):
+        data = Mock.mock_post("storage/01d4fcd4-e446-433b-8a9c-551a1284952e/import", ignore_data_field=True)
+        storage_import = manager.create_storage_import("01d4fcd4-e446-433b-8a9c-551a1284952e", 'direct_upload')
+        assert storage_import.state == "prepared"
+        assert storage_import.source == "direct_upload"
+
+    @responses.activate
+    def test_upload_file_for_storage_import(self, manager):
+        data = Mock.mock_post("storage/01d4fcd4-e446-433b-8a9c-551a1284952e/import", ignore_data_field=True)
+        storage_import = manager.create_storage_import("01d4fcd4-e446-433b-8a9c-551a1284952e", 'direct_upload')
+        data = Mock.mock_put("https://fi-hel1.img.upcloud.com/uploader/session/07a6c9a3-300e-4d0e-b935-624f3dbdff3f", ignore_data_field=True, empty_payload=True, call_api=False)
+        res = manager.upload_file_for_storage_import(storage_import, 'test/json_data/test_file.json')
+        assert res.get("written_bytes") == 909500125
+        assert res.get("md5sum") == "5cc6f7e7a1c52303ac3137d62410eec5"
+        assert res.get("sha256sum") == "bdf14d897406939c11a73d0720ca75c709e756d437f8be9ee26af6b58ede3bd7"
+
+    @responses.activate
+    def test_get_storage_import_details(self, manager):
+        data = Mock.mock_get("storage/01d4fcd4-e446-433b-8a9c-551a1284952e/import")
+        storage_import = manager.get_storage_import_details("01d4fcd4-e446-433b-8a9c-551a1284952e")
+        assert storage_import.state == "pending"
+        assert storage_import.uuid == "07a6c9a3-300e-4d0e-b935-624f3dbdff3f"
+
+    @responses.activate
+    def test_cancel_storage_import(self, manager):
+        data = Mock.mock_post("storage/01d4fcd4-e446-433b-8a9c-551a1284952e/import/cancel", empty_payload=True, ignore_data_field=True)
+        storage_import = manager.cancel_storage_import("01d4fcd4-e446-433b-8a9c-551a1284952e")
+        assert storage_import.state == "cancelling"
+        assert storage_import.uuid == "07a6c9a3-300e-4d0e-b935-624f3dbdff3f"
+
     @responses.activate
     def test_storage_update(self, manager):
 
diff --git a/upcloud_api/__init__.py b/upcloud_api/__init__.py
@@ -17,6 +17,7 @@
 from upcloud_api.errors import UpCloudClientError, UpCloudAPIError
 from upcloud_api.constants import OperatingSystems, ZONE
 from upcloud_api.storage import Storage
+from upcloud_api.storage_import import StorageImport
 from upcloud_api.ip_address import IPAddress
 from upcloud_api.server import Server, login_user_block
 from upcloud_api.firewall import FirewallRule
diff --git a/upcloud_api/cloud_manager/base.py b/upcloud_api/cloud_manager/base.py
@@ -18,31 +18,34 @@ def __init__(self, token, timeout=None):  # noqa
         self.token = token
         self.timeout = timeout
 
-    def request(self, method, endpoint, body=None, timeout=-1):
+    def request(self, method, endpoint, body=None, timeout=-1, request_to_api=True):
         """
-        Perform a request with a given body to a given endpoint in UpCloud's API.
+        Perform a request with a given body to a given endpoint in UpCloud's API or UpCloud's uploader session.
 
         Handles errors with __error_middleware.
         """
         if method not in set(['GET', 'POST', 'PUT', 'DELETE']):
             raise Exception('Invalid/Forbidden HTTP method')
 
-        url = '/' + self.api_v + endpoint
+        url = 'https://api.upcloud.com/' + self.api_v + endpoint if request_to_api else endpoint
         headers = {
-            'Authorization': self.token,
-            'Content-Type': 'application/json'
+            'Authorization': self.token
         }
 
-        if body:
-            json_body_or_None = json.dumps(body)
+        headers['Content-Type'] = 'application/json' if request_to_api else 'application/octet-stream'
+
+        if body and request_to_api:
+            data = json.dumps(body)
+        elif body and not request_to_api:
+            data = body
         else:
-            json_body_or_None = None
+            data = None
 
         call_timeout = timeout if timeout != -1 else self.timeout
 
         APIcall = getattr(requests, method.lower())
-        res = APIcall('https://api.upcloud.com' + url,
-                      data=json_body_or_None,
+        res = APIcall(url,
+                      data=data,
                       headers=headers,
                       timeout=call_timeout)
 
@@ -65,6 +68,12 @@ def post_request(self, endpoint, body=None, timeout=-1):
         """
         return self.request('POST', endpoint, body, timeout)
 
+    def put_request(self, endpoint, body=None, timeout=-1, request_to_api=True):
+        """
+        Perform a PUT request to a given endpoint in UpCloud's API or UpCloud's uploader session.
+        """
+        return self.request('PUT', endpoint, body, timeout, request_to_api=request_to_api)
+
     def __error_middleware(self, res, res_json):
         """
         Middleware that raises an exception when HTTP statuscode is an error code.
diff --git a/upcloud_api/cloud_manager/storage_mixin.py b/upcloud_api/cloud_manager/storage_mixin.py
@@ -3,7 +3,8 @@
 from __future__ import division
 from __future__ import absolute_import
 
-from upcloud_api import Storage
+from upcloud_api import Storage, StorageImport
+from upcloud_api.utils import get_raw_data_from_file
 
 
 class StorageManager(object):
@@ -150,3 +151,40 @@ def templatize_storage(self, storage, title):
         body = {'storage': {'title': title}}
         res = self.post_request(url, body)
         return Storage(cloud_manager=self, **res['storage'])
+
+    def create_storage_import(self, storage, source, source_location=None):
+        """
+        Creates an import task to import data into an existing storage.
+        Source types: http_import or direct_upload.
+        """
+        url = '/storage/{0}/import'.format(storage)
+        body = {'storage_import': {'source': source}}
+        if source_location:
+            body['storage_import']['source_location'] = source_location
+        res = self.post_request(url, body)
+        return StorageImport(**res['storage_import'])
+
+    def upload_file_for_storage_import(self, storage_import, file):
+        """
+        Uploads a file directly to UpCloud's uploader session.
+        """
+        url = storage_import.direct_upload_url
+        data = get_raw_data_from_file(file)
+        body = {'data': data}
+        return self.put_request(url, body, timeout=600, request_to_api=False)
+
+    def get_storage_import_details(self, storage):
+        """
+        Returns detailed information of an ongoing or finished import task.
+        """
+        url = '/storage/{0}/import'.format(storage)
+        res = self.get_request(url)
+        return StorageImport(**res['storage_import'])
+
+    def cancel_storage_import(self, storage):
+        """
+        Cancels an ongoing import task.
+        """
+        url = '/storage/{0}/import/cancel'.format(storage)
+        res = self.post_request(url)
+        return StorageImport(**res['storage_import'])
diff --git a/upcloud_api/storage_import.py b/upcloud_api/storage_import.py
@@ -0,0 +1,31 @@
+from __future__ import unicode_literals
+from __future__ import print_function
+from __future__ import division
+from __future__ import absolute_import
+
+from upcloud_api import UpCloudResource
+from upcloud_api.utils import assignIfExists
+
+
+class StorageImport(UpCloudResource):
+    """
+    Class representation of UpCloud Storage import.
+    """
+
+    ATTRIBUTES = {
+        'client_content_length': None,
+        'client_content_type': None,
+        'completed': None,
+        'created': None,
+        'error_code': None,
+        'error_message': None,
+        'md5sum': None,
+        'read_bytes': None,
+        'sha256sum': None,
+        'source': None,
+        'source_location': None,
+        'state': None,
+        'uuid': None,
+        'written_bytes': None,
+        'direct_upload_url': None
+    }
diff --git a/upcloud_api/utils.py b/upcloud_api/utils.py
@@ -32,3 +32,13 @@ def try_it_n_times(operation, expected_error_codes, custom_error='operation fail
             sleep(3)
         if i >= n - 1:
             raise UpCloudClientError(custom_error)
+
+
+def get_raw_data_from_file(file):
+    """
+    Helper function to get raw file data for uploading.
+    """
+    with open(file, 'rb') as file:
+        data = file.read()
+    file.close()
+    return data

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{"written_bytes":909500125,"md5sum":"5cc6f7e7a1c52303ac3137d62410eec5","sha256sum":"bdf14d897406939c11a73d0720ca75c709e756d437f8be9ee26af6b58ede3bd7"}`