Repository: lightly-ai/lightly

Issue:
Codecoverage fluctuates due to version checking failing only sometimes
## Example problem:
https://github.com/lightly-ai/lightly/pull/312 shows a change in the code coverage
<img width="557" alt="image" src="https://user-images.githubusercontent.com/20324507/115432652-eaef8000-a206-11eb-8d81-3fcaff209ac1.png">
Same for https://github.com/lightly-ai/lightly/pull/303:
<img width="672" alt="image" src="https://user-images.githubusercontent.com/20324507/115432748-035f9a80-a207-11eb-86dd-7c851d274fcb.png">

Both PRs did not do anything which should change code coverage, thus this should not happen. Other PRs, on the other hand, have the opposite effect, e.g. https://github.com/lightly-ai/lightly/pull/308
<img width="586" alt="image" src="https://user-images.githubusercontent.com/20324507/115433398-d1026d00-a207-11eb-90d1-d69dea7cd43d.png">


## Solution:
https://github.com/lightly-ai/lightly/blob/bd982c9513effd75c11d9b2671de09a4be6ce155/.github/workflows/test.yml#L30-L32
change it, perhaps `export LIGHTLY_SERVER_LOCATION="localhost:-1"`works.


Discussion:

Resolving patch:
diff --git a/lightly/cli/lightly_cli.py b/lightly/cli/lightly_cli.py
--- a/lightly/cli/lightly_cli.py
+++ b/lightly/cli/lightly_cli.py
@@ -31,7 +31,7 @@ def _lightly_cli(cfg, is_cli_call=True):
     embeddings = _embed_cli(cfg, is_cli_call)
     cfg['embeddings'] = embeddings
 
-    if cfg['token'] and cfg['dataset_id']:
+    if cfg['token'] and (cfg['dataset_id'] or cfg['new_dataset_name']):
         _upload_cli(cfg)   
 
 
diff --git a/lightly/cli/upload_cli.py b/lightly/cli/upload_cli.py
--- a/lightly/cli/upload_cli.py
+++ b/lightly/cli/upload_cli.py
@@ -7,6 +7,7 @@
 
 # Copyright (c) 2020. Lightly AG and its affiliates.
 # All Rights Reserved
+import warnings
 
 import hydra
 
@@ -20,7 +21,6 @@
 
 
 def _upload_cli(cfg, is_cli_call=True):
-
     input_dir = cfg['input_dir']
     if input_dir and is_cli_call:
         input_dir = fix_input_path(input_dir)
@@ -31,6 +31,23 @@ def _upload_cli(cfg, is_cli_call=True):
 
     dataset_id = cfg['dataset_id']
     token = cfg['token']
+    new_dataset_name = cfg['new_dataset_name']
+
+    if not token:
+        warnings.warn('Please specify your access token. For help, try: lightly-upload --help')
+        return
+
+    dataset_id_ok = dataset_id and len(dataset_id) > 0
+    new_dataset_name_ok = new_dataset_name and len(new_dataset_name) > 0
+    if new_dataset_name_ok and not dataset_id_ok:
+        api_workflow_client = ApiWorkflowClient(token=token)
+        api_workflow_client.create_dataset(dataset_name=new_dataset_name)
+    elif dataset_id_ok and not new_dataset_name_ok:
+        api_workflow_client = ApiWorkflowClient(token=token, dataset_id=dataset_id)
+    else:
+        warnings.warn('Please specify either the dataset_id of an existing dataset or a new_dataset_name. '
+                      'For help, try: lightly-upload --help')
+        return
 
     size = cfg['resize']
     if not isinstance(size, int):
@@ -39,15 +56,6 @@ def _upload_cli(cfg, is_cli_call=True):
     if isinstance(size, tuple) or size > 0:
         transform = torchvision.transforms.Resize(size)
 
-    if not token or not dataset_id:
-        print('Please specify your access token and dataset id.')
-        print('For help, try: lightly-upload --help')
-        return
-
-    api_workflow_client = ApiWorkflowClient(
-        token=token, dataset_id=dataset_id
-    )
-
     if input_dir:
         mode = cfg['upload']
         dataset = LightlyDataset(input_dir=input_dir, transform=transform)
@@ -79,13 +87,16 @@ def upload_cli(cfg):
             Path to the csv file storing the embeddings generated by
             lightly.
         token:
-            User access token to the Lightly platform. If dataset_id
-            and token are specified, the images and embeddings are 
-            uploaded to the platform.
+            User access token to the Lightly platform. If needs to be
+            specified to upload the images and embeddings to the platform.
         dataset_id:
-            Identifier of the dataset on the Lightly platform. If 
-            dataset_id and token are specified, the images and 
-            embeddings are uploaded to the platform.
+            Identifier of the dataset on the Lightly platform.
+            Either the dataset_id or the new_dataset_name need to be
+            specified.
+        new_dataset_name:
+            The name of the new dataset to create on the Lightly platform.
+            Either the dataset_id or the new_dataset_name need to be
+            specified.
         upload:
             String to determine whether to upload the full images, 
             thumbnails only, or metadata only.
@@ -102,11 +113,14 @@ def upload_cli(cfg):
             to (size * height / width, size).
 
     Examples:
-        >>> # upload thumbnails to the Lightly platform
+        >>> # create a new dataset on the Lightly platform and upload thumbnails to it
+        >>> lightly-upload input_dir=data/ token='123' new_dataset_name='new_dataset_name_xyz'
+        >>>
+        >>> # upload thumbnails to the Lightly platform to an existing dataset
         >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ'
         >>> 
-        >>> # upload full images to the Lightly platform
-        >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='full'
+        >>> # create a new dataset on the Lightly platform and upload full images to it
+        >>> lightly-upload input_dir=data/ token='123' new_dataset_name='new_dataset_name_xyz' upload='full'
         >>>
         >>> # upload metadata to the Lightly platform
         >>> lightly-upload input_dir=data/ token='123' dataset_id='XYZ' upload='metadata'
@@ -117,6 +131,9 @@ def upload_cli(cfg):
         >>> # upload both, images and embeddings in a single command
         >>> lightly-upload input_dir=data/ embeddings=embeddings.csv upload='full' \\
         >>>     token='123' dataset_id='XYZ'
+        >>> # create a new dataset on the Lightly platform and upload both, images and embeddings
+        >>> lightly-upload input_dir=data/ embeddings=embeddings.csv upload='full' \\
+        >>>     token='123' new_dataset_name='new_dataset_name_xyz'
 
     """
     _upload_cli(cfg)


Test patch:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -27,7 +27,7 @@ jobs:
       run: pip install -e '.[all]'
     - name: Run Pytest
       run: |
-        LIGHTLY_SERVER_LOCATION="localhost:-1"
+        export LIGHTLY_SERVER_LOCATION="localhost:-1"
         pip install pytest-cov
         python -m pytest -s -v --runslow --cov=./lightly --cov-report=xml --ignore=./lightly/openapi_generated/
     - name: Upload coverage to Codecov
diff --git a/tests/cli/test_cli_download.py b/tests/cli/test_cli_download.py
--- a/tests/cli/test_cli_download.py
+++ b/tests/cli/test_cli_download.py
@@ -16,10 +16,11 @@ class TestCLIDownload(MockedApiWorkflowSetup):
     @classmethod
     def setUpClass(cls) -> None:
         sys.modules["lightly.cli.download_cli"].ApiWorkflowClient = MockedApiWorkflowClient
-        initialize(config_path="../../lightly/cli/config", job_name="test_app")
+
 
     def setUp(self):
-        self.cfg = compose(config_name="config", overrides=["token='123'", "dataset_id='XYZ'"])
+        with initialize(config_path="../../lightly/cli/config", job_name="test_app"):
+            self.cfg = compose(config_name="config", overrides=["token='123'", "dataset_id='XYZ'"])
 
 
     def parse_cli_string(self, cli_words: str):
diff --git a/tests/cli/test_cli_magic.py b/tests/cli/test_cli_magic.py
new file mode 100644
--- /dev/null
+++ b/tests/cli/test_cli_magic.py
@@ -0,0 +1,76 @@
+import os
+import re
+import sys
+import tempfile
+
+import torchvision
+from hydra.experimental import compose, initialize
+
+import lightly
+from tests.api_workflow.mocked_api_workflow_client import MockedApiWorkflowSetup, MockedApiWorkflowClient
+
+
+class TestCLIMagic(MockedApiWorkflowSetup):
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        sys.modules["lightly.cli.upload_cli"].ApiWorkflowClient = MockedApiWorkflowClient
+
+    def setUp(self):
+        MockedApiWorkflowSetup.setUp(self)
+        self.create_fake_dataset()
+        with initialize(config_path="../../lightly/cli/config", job_name="test_app"):
+            self.cfg = compose(config_name="config", overrides=[
+                "token='123'",
+                f"input_dir={self.folder_path}",
+                "trainer.max_epochs=0"
+            ])
+
+    def create_fake_dataset(self):
+        n_data = len(self.api_workflow_client.filenames_on_server)
+        self.dataset = torchvision.datasets.FakeData(size=n_data, image_size=(3, 32, 32))
+
+        self.folder_path = tempfile.mkdtemp()
+        sample_names = [f'img_{i}.jpg' for i in range(n_data)]
+        self.sample_names = sample_names
+        for sample_idx in range(n_data):
+            data = self.dataset[sample_idx]
+            path = os.path.join(self.folder_path, sample_names[sample_idx])
+            data[0].save(path)
+
+    def parse_cli_string(self, cli_words: str):
+        cli_words = cli_words.replace("lightly-magic ", "")
+        cli_words = re.split("=| ", cli_words)
+        assert len(cli_words) % 2 == 0
+        dict_keys = cli_words[0::2]
+        dict_values = cli_words[1::2]
+        for key, value in zip(dict_keys, dict_values):
+            value = value.strip('\"')
+            value = value.strip('\'')
+            self.cfg[key] = value
+
+    def test_parse_cli_string(self):
+        cli_string = "lightly-magic dataset_id='XYZ' upload='thumbnails'"
+        self.parse_cli_string(cli_string)
+        assert self.cfg["dataset_id"] == 'XYZ'
+        assert self.cfg["upload"] == 'thumbnails'
+
+    def test_magic_new_dataset_name(self):
+        cli_string = "lightly-magic new_dataset_name='new_dataset_name_xyz'"
+        self.parse_cli_string(cli_string)
+        lightly.cli.lightly_cli(self.cfg)
+
+    def test_magic_new_dataset_id(self):
+        cli_string = "lightly-magic dataset_id='xyz'"
+        self.parse_cli_string(cli_string)
+        lightly.cli.lightly_cli(self.cfg)
+
+    def tearDown(self) -> None:
+        for filename in ["embeddings.csv", "embeddings_sorted.csv"]:
+            try:
+                os.remove(filename)
+            except FileNotFoundError:
+                pass
+
+
+
diff --git a/tests/cli/test_cli_upload.py b/tests/cli/test_cli_upload.py
new file mode 100644
--- /dev/null
+++ b/tests/cli/test_cli_upload.py
@@ -0,0 +1,81 @@
+import os
+import re
+import sys
+import tempfile
+
+import torchvision
+from hydra.experimental import compose, initialize
+
+import lightly
+from tests.api_workflow.mocked_api_workflow_client import MockedApiWorkflowSetup, MockedApiWorkflowClient
+
+
+class TestCLIUpload(MockedApiWorkflowSetup):
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        sys.modules["lightly.cli.upload_cli"].ApiWorkflowClient = MockedApiWorkflowClient
+
+    def setUp(self):
+        self.create_fake_dataset()
+        with initialize(config_path="../../lightly/cli/config", job_name="test_app"):
+            self.cfg = compose(config_name="config", overrides=["token='123'", f"input_dir={self.folder_path}"])
+
+    def create_fake_dataset(self, n_data: int=5):
+        self.dataset = torchvision.datasets.FakeData(size=n_data,
+                                                     image_size=(3, 32, 32))
+
+        self.folder_path = tempfile.mkdtemp()
+        sample_names = [f'img_{i}.jpg' for i in range(n_data)]
+        self.sample_names = sample_names
+        for sample_idx in range(n_data):
+            data = self.dataset[sample_idx]
+            path = os.path.join(self.folder_path, sample_names[sample_idx])
+            data[0].save(path)
+
+    def parse_cli_string(self, cli_words: str):
+        cli_words = cli_words.replace("lightly-upload ", "")
+        cli_words = re.split("=| ", cli_words)
+        assert len(cli_words) % 2 == 0
+        dict_keys = cli_words[0::2]
+        dict_values = cli_words[1::2]
+        for key, value in zip(dict_keys, dict_values):
+            value = value.strip('\"')
+            value = value.strip('\'')
+            self.cfg[key] = value
+
+    def test_parse_cli_string(self):
+        cli_string = "lightly-upload dataset_id='XYZ' upload='thumbnails'"
+        self.parse_cli_string(cli_string)
+        assert self.cfg["dataset_id"] == 'XYZ'
+        assert self.cfg["upload"] == 'thumbnails'
+
+    def test_upload_no_token(self):
+        self.cfg['token']=''
+        with self.assertWarns(UserWarning):
+            lightly.cli.upload_cli(self.cfg)
+
+    def test_upload_new_dataset_name(self):
+        cli_string = "lightly-upload new_dataset_name='new_dataset_name_xyz'"
+        self.parse_cli_string(cli_string)
+        lightly.cli.upload_cli(self.cfg)
+
+    def test_upload_new_dataset_id(self):
+        cli_string = "lightly-upload dataset_id='xyz'"
+        self.parse_cli_string(cli_string)
+        lightly.cli.upload_cli(self.cfg)
+
+    def test_upload_no_dataset(self):
+        cli_string = "lightly-upload input_dir=data/ token='123'"
+        self.parse_cli_string(cli_string)
+        with self.assertWarns(UserWarning):
+            lightly.cli.upload_cli(self.cfg)
+
+    def test_upload_both_dataset(self):
+        cli_string = "lightly-upload new_dataset_name='new_dataset_name_xyz' dataset_id='xyz'"
+        self.parse_cli_string(cli_string)
+        with self.assertWarns(UserWarning):
+            lightly.cli.upload_cli(self.cfg)
+
+
+