Repository: voxel51/fiftyone

Issue:
[FR] Return info for unexpected labels
At the moment I am using
```
dataset.load_annotations(run, skip_unexpected=True)
```
to load annotations from CVAT into fiftyone. This logs a warning in the console when unexpected labels occur, otherwise it silently ignores the labels. For me, the code is running on a server where no interactive prompt is desired as this would halt the server. Therefore, my only option for `skip_unexpected` is `True`. I think it would be really nice, if there was some return value which indicates whether unexpected labels occured. Or even better, the ability to tag samples with unexpected labels. This way I can exclude them from my dataset later or relabel them afterward.
What do you think?



Discussion:

Resolving patch:
diff --git a/fiftyone/core/collections.py b/fiftyone/core/collections.py
--- a/fiftyone/core/collections.py
+++ b/fiftyone/core/collections.py
@@ -6263,7 +6263,7 @@ def load_annotation_view(self, anno_key, select_fields=False):
         )
 
     def load_annotations(
-        self, anno_key, skip_unexpected=False, cleanup=False, **kwargs
+        self, anno_key, unexpected="prompt", cleanup=False, **kwargs
     ):
         """Downloads the labels from the given annotation run from the
         annotation backend and merges them into this collection.
@@ -6274,21 +6274,26 @@ def load_annotations(
 
         Args:
             anno_key: an annotation key
-            skip_unexpected (False): whether to skip any unexpected labels that
-                don't match the run's label schema when merging. If False and
-                unexpected labels are encountered, you will be presented an
-                interactive prompt to deal with them
+            unexpected ("prompt"): how to deal with any unexpected labels that
+                don't match the run's label schema when importing. The
+                supported values are:
+
+                -   ``"prompt"``: present an interactive prompt to
+                    direct/discard unexpected labels
+                -   ``"ignore"``: automatically ignore any unexpected labels
+                -   ``"return"``: return a dict containing all unexpected
+                    labels, or ``None`` if there aren't any
             cleanup (False): whether to delete any informtation regarding this
                 run from the annotation backend after loading the annotations
             **kwargs: optional keyword arguments for
                 :meth:`fiftyone.utils.annotations.AnnotationResults.load_credentials`
+
+        Returns:
+            ``None``, unless ``unexpected=="return"`` and unexpected labels are
+            found, in which case a dict containing the extra labels is returned
         """
-        foua.load_annotations(
-            self,
-            anno_key,
-            skip_unexpected=skip_unexpected,
-            cleanup=cleanup,
-            **kwargs,
+        return foua.load_annotations(
+            self, anno_key, unexpected=unexpected, cleanup=cleanup, **kwargs,
         )
 
     def delete_annotation_run(self, anno_key):
diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py
--- a/fiftyone/utils/annotations.py
+++ b/fiftyone/utils/annotations.py
@@ -981,7 +981,7 @@ def _format_attributes(backend, attributes):
 
 
 def load_annotations(
-    samples, anno_key, skip_unexpected=False, cleanup=False, **kwargs
+    samples, anno_key, unexpected="prompt", cleanup=False, **kwargs
 ):
     """Downloads the labels from the given annotation run from the annotation
     backend and merges them into the collection.
@@ -993,19 +993,30 @@ def load_annotations(
     Args:
         samples: a :class:`fiftyone.core.collections.SampleCollection`
         anno_key: an annotation key
-        skip_unexpected (False): whether to skip any unexpected labels that
-            don't match the run's label schema when merging. If False and
-            unexpected labels are encountered, you will be presented an
-            interactive prompt to deal with them
+        unexpected ("prompt"): how to deal with any unexpected labels that
+            don't match the run's label schema when importing. The supported
+            values are:
+
+            -   ``"prompt"``: present an interactive prompt to direct/discard
+                unexpected labels
+            -   ``"ignore"``: automatically ignore any unexpected labels
+            -   ``"return"``: return a dict containing all unexpected labels,
+                or ``None`` if there aren't any
         cleanup (False): whether to delete any informtation regarding this run
             from the annotation backend after loading the annotations
         **kwargs: optional keyword arguments for
             :meth:`AnnotationResults.load_credentials`
+
+    Returns:
+        ``None``, unless ``unexpected=="return"`` and unexpected labels are
+        found, in which case a dict containing the extra labels is returned
     """
     results = samples.load_annotation_results(anno_key, **kwargs)
     label_schema = results.config.label_schema
     annotations = results.backend.download_annotations(results)
 
+    unexpected_annos = defaultdict(dict)
+
     for label_field, label_info in label_schema.items():
         label_type = label_info.get("type", None)
         global_attrs, class_attrs = _parse_attributes(label_info)
@@ -1041,12 +1052,12 @@ def load_annotations(
                     )
             else:
                 # Unexpected labels
-                if skip_unexpected or not allow_additions:
-                    new_field = None
-                else:
+                if unexpected == "prompt" and allow_additions:
                     new_field = _prompt_field(
                         samples, anno_type, label_field, label_schema
                     )
+                else:
+                    new_field = None
 
                 if new_field:
                     if anno_type == "scalar":
@@ -1063,11 +1074,17 @@ def load_annotations(
                         label_field,
                     )
 
+                    if unexpected == "return":
+                        unexpected_annos[label_field][anno_type] = annos
+
     results.backend.save_run_results(samples, anno_key, results)
 
     if cleanup:
         results.cleanup()
 
+    if unexpected == "return":
+        return dict(unexpected_annos) if unexpected_annos else None
+
 
 def _parse_attributes(label_info):
     classes = label_info.get("classes", [])


Test patch: