Repository: lightly-ai/lightly

Issue:
Allow easy way to upload active learning scores without triggering a sampling
AFAIK there is currently no way to upload active learning scores to the cloud platform without triggering a sampling. It would be great if we would support something like this

```python
al_agent.upload_scores(scorer)
```


Discussion:

Resolving patch:
diff --git a/lightly/active_learning/agents/agent.py b/lightly/active_learning/agents/agent.py
--- a/lightly/active_learning/agents/agent.py
+++ b/lightly/active_learning/agents/agent.py
@@ -161,11 +161,36 @@ def added_set(self):
         )
 
 
+    def upload_scores(self, al_scorer: Scorer):
+        """Computes and uploads active learning scores to the Lightly webapp.
+
+        Args:
+            al_scorer:
+                An instance of a class inheriting from Scorer, e.g. a ClassificationScorer.
+
+        """
+        # calculate active learning scores
+        al_scores_dict = al_scorer.calculate_scores()
+
+        # Check if the length of the query_set and each of the scores are the same
+        no_query_samples = len(self.query_set)
+        for score in al_scores_dict.values():
+            no_query_samples_with_scores = len(score)
+            if no_query_samples != no_query_samples_with_scores:
+                raise ValueError(
+                    f'Number of query samples ({no_query_samples}) must match '
+                    f'the number of predictions ({no_query_samples_with_scores})!'
+                )
+        self.api_workflow_client.upload_scores(al_scores_dict, self._query_tag_id)
+
+
     def query(self,
               sampler_config: SamplerConfig,
-              al_scorer: Scorer = None) -> Tuple[List[str], List[str]]:
+              al_scorer: Scorer = None):
         """Performs an active learning query.
 
+        First the active learning scores are computed and uploaded,
+        then the sampling query is performed.
         After the query, the labeled set is updated to contain all selected samples,
         the added set is recalculated as (new labeled set - old labeled set), and
         the query set stays the same.
@@ -187,26 +212,12 @@ def query(self,
             )
             return
 
-        # calculate active learning scores
-        scores_dict = None
-        if al_scorer is not None:
-            scores_dict = al_scorer.calculate_scores()
-
-            # Check if the length of the query_set and each of the scores are the same
-            no_query_samples = len(self.query_set)
-            for score in scores_dict.values():
-                no_query_samples_with_scores = len(score)
-                if no_query_samples != no_query_samples_with_scores:
-                    raise ValueError(
-                        f'Number of query samples ({no_query_samples}) must match '
-                        f'the number of predictions ({no_query_samples_with_scores})!'
-                    )
-
+        if al_scorer:
+            self.upload_scores(al_scorer)
 
         # perform the sampling
         new_tag_data = self.api_workflow_client.sampling(
             sampler_config=sampler_config,
-            al_scores=scores_dict,
             preselected_tag_id=self._preselected_tag_id,
             query_tag_id=self._query_tag_id
         )
diff --git a/lightly/api/api_workflow_sampling.py b/lightly/api/api_workflow_sampling.py
--- a/lightly/api/api_workflow_sampling.py
+++ b/lightly/api/api_workflow_sampling.py
@@ -1,5 +1,5 @@
 import time
-from typing import Dict, List
+from typing import Dict, List, Union
 
 import numpy as np
 
@@ -10,11 +10,11 @@
 from lightly.openapi_generated.swagger_client.models.tag_data import TagData
 from lightly.openapi_generated.swagger_client.models.sampling_config import SamplingConfig
 from lightly.openapi_generated.swagger_client.models.sampling_create_request import SamplingCreateRequest
-from lightly.openapi_generated.swagger_client.models.sampling_config_stopping_condition import SamplingConfigStoppingCondition
+from lightly.openapi_generated.swagger_client.models.sampling_config_stopping_condition import \
+    SamplingConfigStoppingCondition
 
 
-
-def _parse_active_learning_scores(scores):
+def _parse_active_learning_scores(scores: Union[np.ndarray, List]):
     """Makes list/np.array of active learning scores serializable.
 
     """
@@ -26,11 +26,34 @@ def _parse_active_learning_scores(scores):
     return list(scores)
 
 
-
 class _SamplingMixin:
 
-    def sampling(self, sampler_config: SamplerConfig, al_scores: Dict[str, List[np.ndarray]] = None,
-                 preselected_tag_id: str = None, query_tag_id: str = None) -> TagData:
+    def upload_scores(self, al_scores: Dict[str, np.ndarray], query_tag_id: str = None):
+
+        tags = self._get_all_tags()
+
+        # upload the active learning scores to the api
+        # change @20210422: we store the active learning scores with the query
+        # tag. policy is that if there's no explicit query tag, the whole dataset
+        # will be the query tag (i.e. query_tag = initial-tag)
+        # set the query tag to the initial-tag if necessary
+        if query_tag_id is None:
+            query_tag = next(t for t in tags if t.name == 'initial-tag')
+            query_tag_id = query_tag.id
+        # iterate over all available score types and upload them
+        for score_type, score_values in al_scores.items():
+            body = ActiveLearningScoreCreateRequest(
+                score_type=score_type,
+                scores=_parse_active_learning_scores(score_values)
+            )
+            self.scores_api.create_or_update_active_learning_score_by_tag_id(
+                body,
+                dataset_id=self.dataset_id,
+                tag_id=query_tag_id,
+            )
+
+    def sampling(self, sampler_config: SamplerConfig, preselected_tag_id: str = None, query_tag_id: str = None) \
+            -> TagData:
         """Performs a sampling given the arguments.
 
         Args:
@@ -64,28 +87,6 @@ def sampling(self, sampler_config: SamplerConfig, al_scores: Dict[str, List[np.n
         except AttributeError:
             self.set_embedding_id_by_name()
 
-
-        # upload the active learning scores to the api
-        # change @20210422: we store the active learning scores with the query
-        # tag. policy is that if there's no explicit query tag, the whole dataset
-        # will be the query tag (i.e. query_tag = initial-tag)
-        if al_scores is not None:
-            # set the query tag to the initial-tag if necessary
-            if query_tag_id is None:
-                query_tag = next(t for t in tags if t.name == 'initial-tag')
-                query_tag_id = query_tag.id
-            # iterate over all available score types and upload them
-            for score_type, score_values in al_scores.items():
-                body = ActiveLearningScoreCreateRequest(
-                    score_type=score_type,
-                    scores=_parse_active_learning_scores(score_values)
-                )
-                self.scores_api.create_or_update_active_learning_score_by_tag_id(
-                    body,
-                    dataset_id=self.dataset_id,
-                    tag_id=query_tag_id,
-                )
-
         # trigger the sampling
         payload = self._create_sampling_create_request(sampler_config, preselected_tag_id, query_tag_id)
         payload.row_count = self._get_all_tags()[0].tot_size
@@ -98,9 +99,9 @@ def sampling(self, sampler_config: SamplerConfig, al_scores: Dict[str, List[np.n
 
         wait_time_till_next_poll = getattr(self, "wait_time_till_next_poll", 1)
         while job_status_data is None \
-            or job_status_data.status == JobState.RUNNING \
+                or job_status_data.status == JobState.RUNNING \
                 or job_status_data.status == JobState.WAITING \
-                    or job_status_data.status == JobState.UNKNOWN:
+                or job_status_data.status == JobState.UNKNOWN:
             # sleep before polling again
             time.sleep(wait_time_till_next_poll)
             # try to read the sleep time until the next poll from the status data
@@ -148,4 +149,3 @@ def _create_sampling_create_request(self, sampler_config: SamplerConfig, presele
                                                         preselected_tag_id=preselected_tag_id,
                                                         query_tag_id=query_tag_id)
         return sampling_create_request
-


Test patch:
diff --git a/tests/active_learning/test_active_learning_agent.py b/tests/active_learning/test_active_learning_agent.py
--- a/tests/active_learning/test_active_learning_agent.py
+++ b/tests/active_learning/test_active_learning_agent.py
@@ -110,3 +110,19 @@ def test_agent_query_too_few(self):
         )
 
         agent.query(sampler_config)
+
+    def test_agent_only_upload_scores(self):
+        self.api_workflow_client.embedding_id = "embedding_id_xyz"
+        agent = ActiveLearningAgent(
+            self.api_workflow_client,
+            preselected_tag_name="preselected_tag_name_xyz",
+        )
+
+        n_predictions = len(agent.query_set)
+        predictions = np.random.rand(n_predictions, 10).astype(np.float32)
+        predictions_normalized = predictions / np.sum(predictions, axis=1)[:, np.newaxis]
+        al_scorer = ScorerClassification(predictions_normalized)
+
+        agent.upload_scores(al_scorer)
+
+