Repository: lightly-ai/lightly

Issue:
Remove embeddings.csv and rename embeddings_sorted.csv in lightly_outputs
When running lightly-cli we create two files in the output folder `embeddings.csv` and `embeddings_sorted.csv`. Having two files is just confusing. We only use the `embeddings_sorted.csv` for upload to the API. So I'd suggest the following:

Don't save the `embeddings_sorted.csv` on the disk, but only in memory or delete it.


Discussion:

Resolving patch:
diff --git a/lightly/api/api_workflow_client.py b/lightly/api/api_workflow_client.py
--- a/lightly/api/api_workflow_client.py
+++ b/lightly/api/api_workflow_client.py
@@ -109,7 +109,10 @@ def dataset_id(self) -> str:
     def _get_all_tags(self) -> List[TagData]:
         return self.tags_api.get_tags_by_dataset_id(self.dataset_id)
 
-    def _order_list_by_filenames(self, filenames_for_list: List[str], list_to_order: List[object]) -> List[object]:
+    def _order_list_by_filenames(
+            self, filenames_for_list: List[str],
+            list_to_order: List[object]
+    ) -> List[object]:
         """Orders a list such that it is in the order of the filenames specified on the server.
 
         Args:
@@ -119,14 +122,23 @@ def _order_list_by_filenames(self, filenames_for_list: List[str], list_to_order:
                 Some values belonging to the samples
 
         Returns:
-            The list reordered. The same reorder applied on the filenames_for_list
-            would put them in the order of the filenames in self.filenames_on_server
+            The list reordered.
+            The same reorder applied on the filenames_for_list would put them
+            in the order of the filenames in self.filenames_on_server.
+            every filename in self.filenames_on_server must be in the
+            filenames_for_list.
 
         """
-        assert len(filenames_for_list) == len(list_to_order)
+        if len(filenames_for_list) != len(list_to_order) or \
+                len(filenames_for_list) != len(self.filenames_on_server):
+            raise ValueError(
+                f"All inputs (filenames_for_list,  list_to_order and "
+                f"self.filenames_on_server) must have the same length, "
+                f"but their lengths are: ({len(filenames_for_list)},"
+                f"{len(list_to_order)} and {len(self.filenames_on_server)})."
+            )
         dict_by_filenames = dict(zip(filenames_for_list, list_to_order))
-        list_ordered = [dict_by_filenames[filename] for filename in self.filenames_on_server
-                        if filename in filenames_for_list]
+        list_ordered = [dict_by_filenames[filename] for filename in self.filenames_on_server]
         return list_ordered
 
     @property
diff --git a/lightly/api/api_workflow_upload_embeddings.py b/lightly/api/api_workflow_upload_embeddings.py
--- a/lightly/api/api_workflow_upload_embeddings.py
+++ b/lightly/api/api_workflow_upload_embeddings.py
@@ -1,4 +1,5 @@
 import csv
+import tempfile
 from typing import List
 
 from lightly.openapi_generated.swagger_client.models.dataset_embedding_data \
@@ -58,7 +59,7 @@ def upload_embeddings(self, path_to_embeddings_csv: str, name: str):
             return
 
         # create a new csv with the filenames in the desired order
-        path_to_ordered_embeddings_csv = self._order_csv_by_filenames(
+        rows_csv = self._order_csv_by_filenames(
             path_to_embeddings_csv=path_to_embeddings_csv)
 
         # get the URL to upload the csv to
@@ -67,11 +68,21 @@ def upload_embeddings(self, path_to_embeddings_csv: str, name: str):
         self.embedding_id = response.embedding_id
         signed_write_url = response.signed_write_url
 
-        # upload the csv to the URL
-        with open(path_to_ordered_embeddings_csv, 'rb') as file_ordered_embeddings_csv:
-            self.upload_file_with_signed_url(file=file_ordered_embeddings_csv, signed_write_url=signed_write_url)
+        # save the csv rows in a temporary in-memory string file
+        # using a csv writer and then read them as bytes
+        with tempfile.SpooledTemporaryFile(mode="rw") as f:
+            writer = csv.writer(f)
+            writer.writerows(rows_csv)
+            f.seek(0)
+            embeddings_csv_as_bytes = f.read().encode('utf-8')
+
+        # write the bytes to a temporary in-memory byte file
+        with tempfile.SpooledTemporaryFile(mode='r+b') as f_bytes:
+            f_bytes.write(embeddings_csv_as_bytes)
+            f_bytes.seek(0)
+            self.upload_file_with_signed_url(file=f_bytes, signed_write_url=signed_write_url)
 
-    def _order_csv_by_filenames(self, path_to_embeddings_csv: str) -> str:
+    def _order_csv_by_filenames(self, path_to_embeddings_csv: str) -> List[str]:
         """Orders the rows in a csv according to the order specified on the server and saves it as a new file.
 
         Args:
@@ -102,12 +113,7 @@ def _order_csv_by_filenames(self, path_to_embeddings_csv: str) -> str:
             rows_without_header_ordered = \
                 self._order_list_by_filenames(filenames, rows_without_header)
 
-            rows_to_write = [header_row]
-            rows_to_write += rows_without_header_ordered
-
-        path_to_ordered_embeddings_csv = path_to_embeddings_csv.replace('.csv', '_sorted.csv')
-        with open(path_to_ordered_embeddings_csv, 'w') as f:
-            writer = csv.writer(f)
-            writer.writerows(rows_to_write)
+            rows_csv = [header_row]
+            rows_csv += rows_without_header_ordered
 
-        return path_to_ordered_embeddings_csv
+        return rows_csv


Test patch:
diff --git a/tests/UNMOCKED_end2end_tests/test_api.py b/tests/UNMOCKED_end2end_tests/test_api.py
--- a/tests/UNMOCKED_end2end_tests/test_api.py
+++ b/tests/UNMOCKED_end2end_tests/test_api.py
@@ -92,7 +92,9 @@ def create_new_dataset_with_embeddings(path_to_dataset: str,
         print("Finished save of embeddings")
 
     # upload the embeddings
+    print("Starting embedding upload")
     api_workflow_client.upload_embeddings(path_to_embeddings_csv=path_to_embeddings_csv, name="embedding_1")
+    print("Finished embedding upload")
 
     return api_workflow_client
 
diff --git a/tests/api_workflow/mocked_api_workflow_client.py b/tests/api_workflow/mocked_api_workflow_client.py
--- a/tests/api_workflow/mocked_api_workflow_client.py
+++ b/tests/api_workflow/mocked_api_workflow_client.py
@@ -1,3 +1,4 @@
+import tempfile
 import unittest
 from io import IOBase
 
@@ -214,7 +215,9 @@ def get_quota_maximum_dataset_size(self, **kwargs):
 
 def mocked_request_put(dst_url: str, data=IOBase) -> Response:
     assert isinstance(dst_url, str)
-    assert isinstance(data, IOBase)
+    content_bytes: bytes = data.read()
+    content_str: str = content_bytes.decode('utf-8')
+    assert content_str.startswith('filenames')
     response_ = Response()
     response_.status_code = 200
     return response_
@@ -255,6 +258,13 @@ def __init__(self, *args, **kwargs):
 
         self.wait_time_till_next_poll = 0.001  # for api_workflow_sampling
 
+    def upload_file_with_signed_url(
+            self, file: IOBase, signed_write_url: str,
+            max_backoff: int = 32, max_retries: int = 5
+    ) -> Response:
+        res = Response()
+        return res
+
 
 class MockedApiWorkflowSetup(unittest.TestCase):
     def setUp(self, token="token_xyz",  dataset_id="dataset_id_xyz") -> None:
diff --git a/tests/api_workflow/test_api_workflow.py b/tests/api_workflow/test_api_workflow.py
--- a/tests/api_workflow/test_api_workflow.py
+++ b/tests/api_workflow/test_api_workflow.py
@@ -47,7 +47,8 @@ def test_reorder_random(self):
             api_workflow_client = MockedApiWorkflowClient(token="token_xyz", dataset_id="dataset_id_xyz")
             api_workflow_client.mappings_api.sample_names = filenames_on_server
 
-            numbers_in_tag = list(np.random.choice(numbers_to_choose_from, 50))
+            numbers_in_tag = np.copy(numbers_all)
+            np.random.shuffle(numbers_in_tag)
             filenames_for_list = [f"img_{i}" for i in numbers_in_tag]
 
             list_ordered = api_workflow_client._order_list_by_filenames(filenames_for_list,
@@ -59,12 +60,38 @@ def test_reorder_manual(self):
         filenames_on_server = ['a', 'b', 'c']
         api_workflow_client = MockedApiWorkflowClient(token="token_xyz", dataset_id="dataset_id_xyz")
         api_workflow_client.mappings_api.sample_names = filenames_on_server
-        filenames_for_list = ['c', 'a']
-        list_to_order = ['cccc', 'aaaa']
+        filenames_for_list = ['c', 'a', 'b']
+        list_to_order = ['cccc', 'aaaa', 'bbbb']
         list_ordered = api_workflow_client._order_list_by_filenames(filenames_for_list, list_to_order=list_to_order)
-        list_desired_order = ['aaaa', 'cccc']
+        list_desired_order = ['aaaa', 'bbbb', 'cccc']
         assert list_ordered == list_desired_order
 
+    def test_reorder_wrong_lengths(self):
+        filenames_on_server = ['a', 'b', 'c']
+        api_workflow_client = MockedApiWorkflowClient(
+            token="token_xyz", dataset_id="dataset_id_xyz"
+        )
+        api_workflow_client.mappings_api.sample_names = filenames_on_server
+        filenames_for_list = ['c', 'a', 'b']
+        list_to_order = ['cccc', 'aaaa', 'bbbb']
+
+        with self.subTest("filenames_for_list wrong length"):
+            with self.assertRaises(ValueError):
+                api_workflow_client._order_list_by_filenames(
+                    filenames_for_list[:-1], list_to_order)
+
+        with self.subTest("list_to_order wrong length"):
+            with self.assertRaises(ValueError):
+                api_workflow_client._order_list_by_filenames(
+                    filenames_for_list, list_to_order[:-1])
+
+        with self.subTest("filenames_for_list and list_to_order wrong length"):
+            with self.assertRaises(ValueError):
+                api_workflow_client._order_list_by_filenames(
+                    filenames_for_list[:-1], list_to_order[:-1])
+
+
+
 
 
 
diff --git a/tests/api_workflow/test_api_workflow_upload_embeddings.py b/tests/api_workflow/test_api_workflow_upload_embeddings.py
--- a/tests/api_workflow/test_api_workflow_upload_embeddings.py
+++ b/tests/api_workflow/test_api_workflow_upload_embeddings.py
@@ -8,15 +8,15 @@
 from tests.api_workflow.mocked_api_workflow_client import MockedApiWorkflowSetup
 
 
-class TestApiWorkflowUploadEmbeddigns(MockedApiWorkflowSetup):
+class TestApiWorkflowUploadEmbeddings(MockedApiWorkflowSetup):
     def t_ester_upload_embedding(self,
                                  n_data,
                                  special_name_first_sample: bool = False,
                                  special_char_in_first_filename: str = None):
         # create fake embeddings
-        folder_path = tempfile.mkdtemp()
+        self.folder_path = tempfile.mkdtemp()
         path_to_embeddings = os.path.join(
-            folder_path,
+            self.folder_path,
             'embeddings.csv'
         )
         sample_names = [f'img_{i}.jpg' for i in range(n_data)]
@@ -39,6 +39,8 @@ def t_ester_upload_embedding(self,
     def test_upload_success(self):
         n_data = len(self.api_workflow_client.mappings_api.sample_names)
         self.t_ester_upload_embedding(n_data=n_data)
+        filepath_embeddings_sorted = os.path.join(self.folder_path, "embeddings_sorted.csv")
+        self.assertFalse(os.path.isfile(filepath_embeddings_sorted))
 
     def test_upload_wrong_lenght(self):
         n_data = 42 + len(self.api_workflow_client.mappings_api.sample_names)
@@ -70,3 +72,13 @@ def test_set_embedding_id_failure(self):
 
     def test_set_embedding_id_default(self):
         self.api_workflow_client.set_embedding_id_by_name()
+
+    def tearDown(self) -> None:
+        for filename in ["embeddings.csv", "embeddings_sorted.csv"]:
+            if hasattr(self, 'folder_path'):
+                try:
+                    filepath = os.path.join(self.folder_path, filename)
+                    os.remove(filepath)
+                except FileNotFoundError:
+                    pass
+