Repository: lightly-ai/lightly

Issue:
Replace put_request(url, body) to use api_client.requests("PUT", url, body)
Advantages:
- smaller codebase
- less error prone: use generated code for timeout, error handling, ...
- Allows to drop the response library, once the docker endpoints are also refactored to use the new API


Discussion:

Resolving patch:
diff --git a/lightly/api/api_workflow_client.py b/lightly/api/api_workflow_client.py
--- a/lightly/api/api_workflow_client.py
+++ b/lightly/api/api_workflow_client.py
@@ -1,34 +1,36 @@
-import os
+import time
+import random
+import time
 import warnings
+from io import IOBase
 from typing import *
 
-from lightly.__init__ import __version__
-
-from lightly.api.version_checking import get_minimum_compatible_version, version_compare
-from lightly.openapi_generated.swagger_client.models.dataset_data import DatasetData
+import requests
+from requests import Response
 
+from lightly.__init__ import __version__
 from lightly.api.api_workflow_datasets import _DatasetsMixin
-from lightly.openapi_generated.swagger_client.api.datasets_api import DatasetsApi
-
-from lightly.openapi_generated.swagger_client.api.samples_api import SamplesApi
-
-from lightly.api.utils import put_request, getenv
-
-from lightly.api.api_workflow_upload_dataset import _UploadDatasetMixin
-from lightly.api.api_workflow_upload_embeddings import _UploadEmbeddingsMixin
 from lightly.api.api_workflow_download_dataset import _DownloadDatasetMixin
 from lightly.api.api_workflow_sampling import _SamplingMixin
+from lightly.api.api_workflow_upload_dataset import _UploadDatasetMixin
+from lightly.api.api_workflow_upload_embeddings import _UploadEmbeddingsMixin
+from lightly.api.utils import getenv
+from lightly.api.version_checking import get_minimum_compatible_version, version_compare
 from lightly.openapi_generated.swagger_client import TagData, ScoresApi, QuotaApi
+from lightly.openapi_generated.swagger_client.api.datasets_api import DatasetsApi
 from lightly.openapi_generated.swagger_client.api.embeddings_api import EmbeddingsApi
 from lightly.openapi_generated.swagger_client.api.jobs_api import JobsApi
 from lightly.openapi_generated.swagger_client.api.mappings_api import MappingsApi
+from lightly.openapi_generated.swagger_client.api.samples_api import SamplesApi
 from lightly.openapi_generated.swagger_client.api.samplings_api import SamplingsApi
 from lightly.openapi_generated.swagger_client.api.tags_api import TagsApi
 from lightly.openapi_generated.swagger_client.api_client import ApiClient
 from lightly.openapi_generated.swagger_client.configuration import Configuration
+from lightly.openapi_generated.swagger_client.models.dataset_data import DatasetData
 
 
-class ApiWorkflowClient(_UploadEmbeddingsMixin, _SamplingMixin, _UploadDatasetMixin, _DownloadDatasetMixin, _DatasetsMixin):
+class ApiWorkflowClient(_UploadEmbeddingsMixin, _SamplingMixin, _UploadDatasetMixin, _DownloadDatasetMixin,
+                        _DatasetsMixin):
     """Provides a uniform interface to communicate with the api 
     
     The APIWorkflowClient is used to communicaate with the Lightly API. The client
@@ -97,7 +99,7 @@ def dataset_id(self) -> str:
             last_modified_dataset = datasets_sorted[-1]
             self._dataset_id = last_modified_dataset.id
             warnings.warn(UserWarning(f"Dataset has not been specified, "
-                          f"taking the last modified dataset {last_modified_dataset.name} as default dataset."))
+                                      f"taking the last modified dataset {last_modified_dataset.name} as default dataset."))
             return self._dataset_id
 
     def _get_all_tags(self) -> List[TagData]:
@@ -113,7 +115,7 @@ def _order_list_by_filenames(self, filenames_for_list: List[str], list_to_order:
                 Some values belonging to the samples
 
         Returns:
-            The list reorderd. The same reorder applied on the filenames_for_list
+            The list reordered. The same reorder applied on the filenames_for_list
             would put them in the order of the filenames in self.filenames_on_server
 
         """
@@ -130,7 +132,47 @@ def filenames_on_server(self):
                 get_sample_mappings_by_dataset_id(dataset_id=self.dataset_id, field="fileName")
         return self._filenames_on_server
 
-    def upload_file_with_signed_url(self, file, signed_write_url: str):
-        response = put_request(signed_write_url, data=file)
-        file.close()
+    def upload_file_with_signed_url(self, file: IOBase, signed_write_url: str,
+                                    max_backoff: int = 32, max_retries: int = 5) -> Response:
+        """Uploads a file to a url via a put request.
+
+        Args:
+            file:
+                The file to upload.
+            signed_write_url:
+                The url to upload the file to. As no authorization is used,
+                the url must be a signed write url.
+            max_backoff:
+                Maximal backoff before retrying.
+            max_retries:
+                Maximum number of retries before timing out.
+
+        Returns:
+            The response of the put request, usually a 200 for the success case.
+
+        """
+
+        counter = 0
+        backoff = 1. + random.random() * 0.1
+        success = False
+        while not success:
+
+            response = requests.put(signed_write_url, data=file)
+            success = (response.status_code == 200)
+
+            # exponential backoff
+            if response.status_code in [500, 502]:
+                time.sleep(backoff)
+                backoff = 2 * backoff if backoff < max_backoff else backoff
+            # something went wrong
+            elif not success:
+                msg = f'Failed PUT request to {signed_write_url} with status_code '
+                msg += f'{response.status_code}.'
+                raise RuntimeError(msg)
+
+            counter += 1
+            if counter >= max_retries:
+                msg = f'The connection to the server at {signed_write_url} timed out. '
+                raise RuntimeError(msg)
+
         return response
diff --git a/lightly/api/routes/users/service.py b/lightly/api/routes/users/service.py
deleted file mode 100644
--- a/lightly/api/routes/users/service.py
+++ /dev/null
@@ -1,46 +0,0 @@
-""" Users Service """
-
-# Copyright (c) 2020. Lightly AG and its affiliates.
-# All Rights Reserved
-
-import requests
-
-from lightly.api.utils import getenv
-
-
-def _prefix(*args, **kwargs):
-    """Returns the prefix for the users routes.
-
-    All routes through users require authentication via jwt.
-
-    """
-    server_location = getenv(
-        'LIGHTLY_SERVER_LOCATION',
-        'https://api.lightly.ai'
-    )
-    return server_location + '/users'
-
-
-def get_quota(token: str):
-    """Returns a dictionary with the quota for a user.
-
-    The quota defines limitations for the current user.
-
-    Args:
-        token:
-            A token to identify the user.
-
-    Returns:
-        The quota for the user and the status code of the response.
-    """
-    dst_url = _prefix() + '/quota'
-    payload = {
-        'token': token
-    }
-
-    response = requests.get(dst_url, params=payload)
-    status_code = response.status_code
-    if status_code == 200:
-        return response.json()['maxDatasetSize'], status_code
-    else:
-        return LIGHTLY_MAXIMUM_DATASET_SIZE, status_code
diff --git a/lightly/api/utils.py b/lightly/api/utils.py
--- a/lightly/api/utils.py
+++ b/lightly/api/utils.py
@@ -5,17 +5,13 @@
 
 import io
 import os
-import time
-import random
 
 import numpy as np
-import requests
-import warnings
 from PIL import Image, ImageFilter
-
 # the following two lines are needed because
 # PIL misidentifies certain jpeg images as MPOs
 from PIL import JpegImagePlugin
+
 JpegImagePlugin._getmp = lambda: None
 
 LEGAL_IMAGE_FORMATS = ['jpg', 'jpeg', 'png', 'tiff', 'bmp']
@@ -239,56 +235,3 @@ def check_image(image):
 
     metadata['is_corrupted'] = is_corrupted
     return metadata
-
-def put_request(dst_url, data=None, params=None, json=None,
-                max_backoff=32, max_retries=5):
-    """Makes a PUT request with random offset retries.
-
-    Args:
-        dst_url:
-            Url the PUT request is made to.
-        data:
-            PUT data.
-        params:
-            PUT parameters.
-        json:
-            PUT json.
-        max_backoff:
-            Maximal backoff before throwing timing out.
-        max_retries:
-            Maximum number of retries before timing out.
-
-    Returns:
-        The server response.
-
-    Raises:
-        RuntimeError if the request fails.
-
-    """
-    counter = 0
-    backoff = 1. + random.random() * 0.1
-    success = False
-    while not success:
-
-        response = requests.put(dst_url, data=data, json=json, params=params)
-        success = (response.status_code == 200)
-
-        # exponential backoff
-        if response.status_code in [500, 502]:
-            time.sleep(backoff)
-            backoff = 2*backoff if backoff < max_backoff else backoff
-        # something went wrong
-        elif not success:
-            msg = f'Failed PUT request to {dst_url} with status_code '
-            msg += f'{response.status_code}.'
-            raise RuntimeError(msg)
-
-        counter += 1
-        if counter >= max_retries:
-            break
-
-    if not success:
-        msg = f'The connection to the server at {dst_url} timed out. '
-        raise RuntimeError(msg)
-
-    return response


Test patch:
diff --git a/tests/api/test_Put.py b/tests/api/test_Put.py
deleted file mode 100644
--- a/tests/api/test_Put.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import unittest
-import json
-import os
-
-import random
-import responses
-import pytest
-
-
-from lightly.api.utils import put_request
-
-N = 10
-
-@pytest.mark.slow
-class TestPut(unittest.TestCase):
-
-    def setup(self, psuccess=1.):
-
-        self.psuccess = psuccess
-
-        # set up url
-        self.dst_url = os.getenvb(b'LIGHTLY_SERVER_LOCATION',
-                                  b'https://api.lightly.ai').decode()
-        # route
-        self.dst_url += '/sample/route/to/put'
-
-    def callback(self, request):
-        self.assertEqual('sample=data', request.body)
-        if random.random() < self.psuccess:
-            return (200, [], json.dumps({}))
-        else:
-            return (500, [], json.dumps({}))
-
-    @responses.activate
-    def test_put_all_success(self):
-        '''Make sure everything works in no-error scenario.
-
-        '''
-
-        self.setup(psuccess=1.)
-
-        responses.add_callback(
-            responses.PUT, self.dst_url,
-            callback=self.callback,
-            content_type='application/json'
-        )
-
-        for _ in range(N):
-            data = {'sample': 'data'}
-            put_request(self.dst_url, data)
-
-    @responses.activate
-    def test_put_some_success(self):
-        '''Make sure everything works in no-error scenario.
-
-        '''
-
-        self.setup(psuccess=.9)
-
-        responses.add_callback(
-            responses.PUT, self.dst_url,
-            callback=self.callback,
-            content_type='application/json'
-        )
-
-        for _ in range(N):
-            data = {'sample': 'data'}
-            put_request(self.dst_url, data)
-
-    @responses.activate
-    def test_put_no_success(self):
-        '''Make sure everything works in no-error scenario.
-
-        '''
-
-        self.setup(psuccess=0.)
-
-        responses.add_callback(
-            responses.PUT, self.dst_url,
-            callback=self.callback,
-            content_type='application/json'
-        )
-
-        with self.assertRaises(RuntimeError):
-            data = {'sample': 'data'}
-            put_request(self.dst_url, data)
\ No newline at end of file
diff --git a/tests/api_workflow/mocked_api_workflow_client.py b/tests/api_workflow/mocked_api_workflow_client.py
--- a/tests/api_workflow/mocked_api_workflow_client.py
+++ b/tests/api_workflow/mocked_api_workflow_client.py
@@ -1,4 +1,7 @@
 import unittest
+from io import IOBase
+
+from requests import Response
 
 from lightly.openapi_generated.swagger_client.models.dataset_create_request import DatasetCreateRequest
 
@@ -10,7 +13,6 @@
 
 from lightly.api.api_workflow_client import ApiWorkflowClient
 
-from io import BufferedReader
 from typing import *
 
 from lightly.openapi_generated.swagger_client import ScoresApi, CreateEntityResponse, SamplesApi, SampleCreateRequest, \
@@ -180,16 +182,12 @@ class MockedQuotaApi(QuotaApi):
     def get_quota_maximum_dataset_size(self, **kwargs):
         return "60000"
 
-def mocked_upload_file_with_signed_url(file: str, url: str, mocked_return_value=True) -> bool:
-    assert isinstance(file, BufferedReader)
-    assert isinstance(url, str)
-    return mocked_return_value
-
-
-def mocked_put_request(dst_url, data=None, params=None, json=None, max_backoff=32, max_retries=5) -> bool:
+def mocked_request_put(dst_url: str, data=IOBase) -> Response:
     assert isinstance(dst_url, str)
-    success = True
-    return success
+    assert isinstance(data, IOBase)
+    response_ = Response()
+    response_.status_code = 200
+    return response_
 
 
 class MockedApiClient(ApiClient):
@@ -223,7 +221,7 @@ def __init__(self, *args, **kwargs):
         self.datasets_api = MockedDatasetsApi(api_client=self.api_client)
         self.quota_api = MockedQuotaApi(api_client=self.api_client)
 
-        lightly.api.api_workflow_client.put_request = mocked_put_request
+        lightly.api.api_workflow_client.requests.put = mocked_request_put
 
         self.wait_time_till_next_poll = 0.001  # for api_workflow_sampling