Repository: voxel51/fiftyone

Issue:
[FR] Prevent users from specifying sample-level spatial fields when annotating videos
Currently the annotation API lets users define sample-level spatial fields like `detections` for a video annotation task. However, importing such data will raise an error.

The issue is that the correct syntax is to prefix frame-level fields with `frames.`, but this is currently hard to self-debug.

We should do one or both of the following things to make this self-diagnosable:
- Raise an error at schema creation time alerting the user that they cannot annotate sample-level spatial fields for video
- Update the import logic so that frame-level labels are treated as new, unexpected annotations rather than treating them as sample-level labels (they are not)

```py
import fiftyone as fo
import fiftyone.zoo as foz

dataset = foz.load_zoo_dataset("quickstart-video").limit(1).select_fields().clone()

dataset.annotate(
    "test",
    label_field="ground_truth",
    label_type="detections",  # this should instead be "frames.detections"
    classes=["vehicle"],
    launch_editor=True,
)

# Create a video track in CVAT...

dataset.load_annotations("test")
```

```
Downloading labels from CVAT...
Download complete
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-4-07cea798f021> in <module>
----> 1 dataset.load_annotations("test")

~/dev/fiftyone/fiftyone/core/collections.py in load_annotations(self, anno_key, skip_unexpected, cleanup, **kwargs)
   6291             skip_unexpected=skip_unexpected,
   6292             cleanup=cleanup,
-> 6293             **kwargs,
   6294         )
   6295 

~/dev/fiftyone/fiftyone/utils/annotations.py in load_annotations(samples, anno_key, skip_unexpected, cleanup, **kwargs)
   1027                         label_info=label_info,
   1028                         global_attrs=global_attrs,
-> 1029                         class_attrs=class_attrs,
   1030                     )
   1031             else:

~/dev/fiftyone/fiftyone/utils/annotations.py in _merge_labels(samples, anno_dict, results, label_field, label_type, label_info, global_attrs, class_attrs)
   1282             only_keyframes = True
   1283 
-> 1284         _update_tracks(samples, label_field, anno_dict, only_keyframes)
   1285 
   1286     id_map = results.id_map.get(label_field, {})

~/dev/fiftyone/fiftyone/utils/annotations.py in _update_tracks(samples, label_field, anno_dict, only_keyframes)
   1558 
   1559         for _frame_id, _label_ids, _indexes in zip(
-> 1560             _frame_ids, _frame_lids, _frame_inds
   1561         ):
   1562             _label_ids = _to_list(_label_ids)

TypeError: zip argument #2 must support iteration
```



Discussion:

Resolving patch:
diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py
--- a/fiftyone/utils/annotations.py
+++ b/fiftyone/utils/annotations.py
@@ -407,13 +407,21 @@ def _build_label_schema(
         _return_type = _RETURN_TYPES_MAP[_label_type]
         _is_trackable = _is_frame_field and _return_type in _TRACKABLE_TYPES
 
-        if is_video and _return_type in _SPATIAL_TYPES and not _is_frame_field:
-            raise ValueError(
-                "Invalid label field '%s'. Spatial labels of type '%s' being "
-                "annotated on a video must be stored in a frame-level field, "
-                "i.e., one that starts with 'frames.'"
-                % (_label_field, _label_type)
-            )
+        if is_video and not _is_frame_field:
+            if _return_type in _SPATIAL_TYPES:
+                raise ValueError(
+                    "Invalid label field '%s'. Spatial labels of type '%s' "
+                    "being annotated on a video must be stored in a "
+                    "frame-level field, i.e., one that starts with 'frames.'"
+                    % (_label_field, _label_type)
+                )
+            elif not backend.supports_video_sample_fields:
+                raise ValueError(
+                    "Invalid label field '%s'. Backend '%s' does not support "
+                    "annotating video fields at a sample-level. Labels must be "
+                    "stored in a frame-level field, i.e., one that starts with "
+                    "'frames.'" % (_label_field, backend.config.name)
+                )
 
         # We found an existing field with multiple label types, so we must
         # select only the relevant labels
@@ -1783,6 +1791,15 @@ def supports_keyframes(self):
         """
         raise NotImplementedError("subclass must implement supports_keyframes")
 
+    @property
+    def supports_video_sample_fields(self):
+        """Whether this backend supports annotating video labels at a
+        sample-level.
+        """
+        raise NotImplementedError(
+            "subclass must implement supports_video_sample_fields"
+        )
+
     @property
     def requires_label_schema(self):
         """Whether this backend requires a pre-defined label schema for its
diff --git a/fiftyone/utils/cvat.py b/fiftyone/utils/cvat.py
--- a/fiftyone/utils/cvat.py
+++ b/fiftyone/utils/cvat.py
@@ -3129,6 +3129,10 @@ def supported_attr_types(self):
     def supports_keyframes(self):
         return True
 
+    @property
+    def supports_video_sample_fields(self):
+        return False
+
     @property
     def requires_label_schema(self):
         return False  # schemas can be inferred from existing CVAT projects
diff --git a/fiftyone/utils/labelbox.py b/fiftyone/utils/labelbox.py
--- a/fiftyone/utils/labelbox.py
+++ b/fiftyone/utils/labelbox.py
@@ -142,6 +142,10 @@ def supported_attr_types(self):
     def supports_keyframes(self):
         return False
 
+    @property
+    def supports_video_sample_fields(self):
+        return False  # @todo resolve FiftyOne bug to allow this to be True
+
     def recommend_attr_tool(self, name, value):
         if isinstance(value, bool):
             return {"type": "radio", "values": [True, False]}


Test patch: