Repository: voxel51/fiftyone

Issue:
[FR] Improve Label Studio support to work with multiple fields
### Proposal Summary

Currently Label Studio (LS) backend only supports passing single field to be labeled. ([Enforced here](https://github.com/voxel51/fiftyone/blame/26866c5a37e2dff83fee0f18bbde9f7153ff0e99/fiftyone/utils/labelstudio.py#L226)). I propose to fix this and allow more complex workflows. 

### Motivation

-   **What is the use case for this feature?** Examples when user need to label more than one thing. Or needs to pass some existing predictions to be classified.
-   **Why is this use case valuable to support for FiftyOne users in general?** LS has smoother UI than CVAT and is being developed rapidly, would be good to better support it. 
-   **Why is it currently difficult to achieve this use case?** Current code doesn't support this functionality

### What areas of FiftyOne does this feature affect?

-   [ ] App: FiftyOne application
-   [x] Core: Core `fiftyone` Python library
-   [ ] Server: FiftyOne server

### Details

In my opinion the implementation is straightforward:
1. create a more complex labeling config for LS
2. correctly parse it back to 51

I've already patched this to work for me locally, and it only requires minor modifications. 

### Willingness to contribute

-   [x] Yes. I can contribute this feature independently
-   [ ] Yes. I would be willing to contribute this feature with guidance from the FiftyOne community
-   [ ] No. I cannot contribute this feature at this time



Discussion:
hi @bonlime this would be a fantastic addition. I see that you're willing to contribute this feature, thank you! 🙌

If you run into any questions along the way, feel free to reach out and we'll be happy to assist.

The CVAT and Labelbox backend implementations may be good points of reference for you:
- https://github.com/voxel51/fiftyone/blob/develop/fiftyone/utils/cvat.py
- https://github.com/voxel51/fiftyone/blob/develop/fiftyone/utils/labelbox.py

Resolving patch:
diff --git a/fiftyone/utils/labelstudio.py b/fiftyone/utils/labelstudio.py
--- a/fiftyone/utils/labelstudio.py
+++ b/fiftyone/utils/labelstudio.py
@@ -8,6 +8,7 @@
 """
 from copy import deepcopy
 from datetime import datetime
+from collections import defaultdict
 import itertools
 import json
 import logging
@@ -223,12 +224,8 @@ def _init_project(self, config, samples):
                 break
 
         # generate label config
-        assert len(label_schema) == 1
-        _, label_info = label_schema.popitem()
         label_config = generate_labeling_config(
-            media=samples.media_type,
-            label_type=label_info["type"],
-            labels=label_info["classes"],
+            label_schema, samples.media_type
         )
 
         project = self._client.start_project(
@@ -264,7 +261,7 @@ def _prepare_tasks(self, samples, label_schema, media_field):
                         smp[label_field],
                         label_type=label_type,
                         full_result={
-                            "from_name": "label",
+                            "from_name": label_field,
                             "to_name": "image",
                             "original_width": smp.metadata["width"],
                             "original_height": smp.metadata["height"],
@@ -294,9 +291,7 @@ def _upload_tasks(self, project, tasks, predictions=None):
         files = [
             (
                 one["source_id"],
-                (
-                    open(one[one["media_type"]], "rb")
-                ),
+                (open(one[one["media_type"]], "rb")),
             )
             for one in tasks
         ]
@@ -351,7 +346,7 @@ def task_filter(x):
 
         return list(filter(task_filter, matched_tasks))
 
-    def _import_annotations(self, tasks, task_map, label_type):
+    def _import_annotations(self, tasks, task_map):
         results = {}
         for t in tasks:
             # convert latest annotation results
@@ -365,25 +360,27 @@ def _import_annotations(self, tasks, task_map, label_type):
                 if len(annotations) == 0
                 else sorted(annotations, key=lambda x: x["updated_at"])[-1]
             )
-            if label_type == "keypoints":
-                labels = import_label_studio_annotation(
-                    latest_annotation["result"]
-                )
-            else:
-                labels = [
-                    import_label_studio_annotation(r, label_type=label_type)
-                    for r in latest_annotation.get("result", [])
-                ]
+            latest_result = latest_annotation.get("result", [])
+            # we need logic below to group keypoints with same label together
+            result_kps = [
+                r for r in latest_result if _result_type(r) == "keypointlabels"
+            ]
+            result_not_kps = [
+                r for r in latest_result if _result_type(r) != "keypointlabels"
+            ]
+            labels = [
+                import_label_studio_annotation(r) for r in result_not_kps
+            ]
+            if result_kps:
+                labels += [import_label_studio_annotation(result_kps)]
+
+            if not labels:
+                continue
 
             # add to dict
-            if labels:
-                label_ids = (
-                    {l.id: l for l in labels}
-                    if not isinstance(labels[0], fol.Regression)
-                    else labels[0]
-                )
-                sample_id = task_map[t["id"]]
-                results[sample_id] = label_ids
+            sample_id = task_map[t["id"]]
+            # we save and pass both id and the name of the label field
+            results[sample_id] = {l.id: (ln, l) for (ln, l) in labels}
 
         return results
 
@@ -447,14 +444,34 @@ def download_annotations(self, results):
             project, list(results.uploaded_tasks.keys())
         )
         annotations = {}
+        all_labels = self._import_annotations(
+            labeled_tasks, results.uploaded_tasks
+        )
+        annotations = defaultdict(lambda: defaultdict(dict))
+        for task_id, task_labels in all_labels.items():
+            for label_id, (label_field, label) in task_labels.items():
+                annotations[label_field][task_id][label_id] = label
+
+        # convert to expected format
+        final_annotations = {}
         for label_field, label_info in results.config.label_schema.items():
             return_type = foua._RETURN_TYPES_MAP[label_info["type"]]
-            labels = self._import_annotations(
-                labeled_tasks, results.uploaded_tasks, return_type
-            )
-            annotations.update({label_field: {return_type: labels}})
+            # need to convert masks to detections
+            if label_info["type"] == "instances":
+                new_label_field = defaultdict(dict)
+                for task_id, task_labels in annotations[label_field].items():
+                    for label_id, label in task_labels.items():
+                        detections = label.to_detections(
+                            {255: label.label}
+                        ).detections
+                        detections = detections[0] if detections else None
+                        new_label_field[task_id][label_id] = detections
+                annotations[label_field] = new_label_field
+            final_annotations[label_field] = {
+                return_type: annotations[label_field]
+            }
 
-        return annotations
+        return final_annotations
 
     def upload_predictions(self, project, tasks, sample_labels, label_type):
         """Uploads the given predictions to an existing Label Studio project.
@@ -538,63 +555,67 @@ def _from_dict(cls, d, samples, config, anno_key):
         )
 
 
-def generate_labeling_config(media, label_type, labels=None):
+def generate_labeling_config(label_schema, media):
     """Generates a labeling config for a Label Studio project.
 
     Args:
+        label_shema: dict defining each label type
         media: The media type to label
-        label_type: The type of labels to use
-        labels (None): the labels to use
 
     Returns:
         a labeling config
     """
     assert media in ["image", "video"]
-    assert (
-        label_type in _LABEL_TYPES.keys()
-        or label_type in foua._RETURN_TYPES_MAP.keys()
+    possible_types = list(_LABEL_TYPES.keys()) + list(
+        foua._RETURN_TYPES_MAP.keys()
     )
 
     # root view and media view
     root = etree.Element("View")
     etree.SubElement(root, media.capitalize(), name=media, value=f"${media}")
 
-    # labels view
-    parent_tag, child_tag, tag_kwargs = _ls_tags_from_type(label_type)
-    # parent_name = child_tag.lower() if child_tag else parent_tag.lower()
-    label_view = etree.SubElement(
-        root, parent_tag, name="label", toName=media, **tag_kwargs
-    )
-    if labels:
-        for one in labels:
-            etree.SubElement(label_view, child_tag, value=one)
+    for label_field, label_info in label_schema.items():
+        assert label_info["type"] in possible_types
+        parent_tag, child_tag, tag_kwargs = _ls_tags_from_type(
+            label_info["type"]
+        )
+        label_view = etree.SubElement(
+            root, parent_tag, name=label_field, toName=media, **tag_kwargs
+        )
+        if "classes" in label_info and label_info["classes"]:
+            for one in label_info["classes"]:
+                etree.SubElement(label_view, child_tag, value=one)
 
     config_str = etree.tostring(root, pretty_print=True).decode()
     return config_str
 
 
-def import_label_studio_annotation(result, label_type=None):
+def _result_type(result):
+    if isinstance(result, dict):
+        return result["type"]
+    elif isinstance(result, list):
+        return result[0]["type"]
+    else:
+        raise TypeError("Result type %s is not understood" % type(result))
+
+
+def import_label_studio_annotation(result):
     """Imports an annotation from Label Studio.
 
     Args:
         result: the annotation result from Label Studio
-        label_type (None): the label type to use when importing the annotation.
-            This argument is only used when importing brush labels. By default,
-            these labels are imported as semantic segmentations, but you can
-            pass ``label_type="instances"`` to import them as instance
-            segmentations instead
 
     Returns:
         a :class:`fiftyone.core.labels.Label`
     """
     # TODO link keypoints by parent id
     # TODO handle multiple classes for segmentation
+    ls_type = _result_type(result)
+    # we return this to match with label field later
     if isinstance(result, dict):
-        ls_type = result["type"]
+        from_name = result.get("from_name", "from_name")
     elif isinstance(result, list):
-        ls_type = result[0]["type"]
-    else:
-        raise TypeError("Result type %s is not understood" % type(result))
+        from_name = result[0].get("from_name", "from_name")
 
     if ls_type == "choices":
         label = _from_choices(result)
@@ -605,7 +626,7 @@ def import_label_studio_annotation(result, label_type=None):
     elif ls_type == "keypointlabels":
         label = _from_keypointlabels(result)
     elif ls_type == "brushlabels":
-        label = _from_brushlabels(result, label_type=label_type)
+        label = _from_brushlabels(result)
     elif ls_type == "number":
         label = _from_number(result)
     else:
@@ -617,8 +638,7 @@ def import_label_studio_annotation(result, label_type=None):
         label.id = label_id
     except:
         pass
-
-    return label
+    return from_name, label
 
 
 def _update_dict(src_dict, update_dict):
@@ -865,10 +885,13 @@ def _from_keypointlabels(result):
         points = _normalize_values(points)
         keypoints.append(fol.Keypoint(label=key, points=points))
 
-    return keypoints
+    if len(keypoints) == 1:
+        return keypoints[0]
+    else:
+        return fol.Keypoints(keypoints=keypoints)
 
 
-def _from_brushlabels(result, label_type=None):
+def _from_brushlabels(result):
     label_values = result["value"]["brushlabels"]
     img = brush.decode_rle(result["value"]["rle"])
     shape = (result["original_height"], result["original_width"], 4)
@@ -876,11 +899,6 @@ def _from_brushlabels(result, label_type=None):
 
     label = label_values[0]
     segmentation = fol.Segmentation(label=label, mask=mask)
-
-    if label_type == "instances":
-        detections = segmentation.to_detections({255: label}).detections
-        return detections[0] if detections else None
-
     return segmentation
 
 


Test patch:
diff --git a/tests/intensive/labelstudio_tests.py b/tests/intensive/labelstudio_tests.py
--- a/tests/intensive/labelstudio_tests.py
+++ b/tests/intensive/labelstudio_tests.py
@@ -38,12 +38,16 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "classification",
-                "labels": ["Adult content", "Weapons", "Violence"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "classification",
+                        "classes": ["Adult content", "Weapons", "Violence"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <Choices name="label" toName="image" choice="single-radio">
+  <Choices name="some_random_field" toName="image" choice="single-radio">
     <Choice value="Adult content"/>
     <Choice value="Weapons"/>
     <Choice value="Violence"/>
@@ -53,12 +57,16 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "classifications",
-                "labels": ["Boeing", "Airbus"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "classifications",
+                        "classes": ["Boeing", "Airbus"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <Choices name="label" toName="image" choice="multiple">
+  <Choices name="some_random_field" toName="image" choice="multiple">
     <Choice value="Boeing"/>
     <Choice value="Airbus"/>
   </Choices>
@@ -67,12 +75,16 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "detections",
-                "labels": ["Airplane", "Car"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "detections",
+                        "classes": ["Airplane", "Car"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <RectangleLabels name="label" toName="image">
+  <RectangleLabels name="some_random_field" toName="image">
     <Label value="Airplane"/>
     <Label value="Car"/>
   </RectangleLabels>
@@ -81,26 +93,34 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "instances",
-                "labels": ["Airplane", "Car"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "instances",
+                        "classes": ["Airplane", "Car"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <RectangleLabels name="label" toName="image">
+  <BrushLabels name="some_random_field" toName="image">
     <Label value="Airplane"/>
     <Label value="Car"/>
-  </RectangleLabels>
+  </BrushLabels>
 </View>""",
         },
         {
             "input": {
                 "media": "image",
-                "label_type": "polylines",
-                "labels": ["Airplane", "Car"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "polylines",
+                        "classes": ["Airplane", "Car"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <PolygonLabels name="label" toName="image">
+  <PolygonLabels name="some_random_field" toName="image">
     <Label value="Airplane"/>
     <Label value="Car"/>
   </PolygonLabels>
@@ -109,12 +129,16 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "polygons",
-                "labels": ["Airplane", "Car"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "polygons",
+                        "classes": ["Airplane", "Car"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <PolygonLabels name="label" toName="image">
+  <PolygonLabels name="some_random_field" toName="image">
     <Label value="Airplane"/>
     <Label value="Car"/>
   </PolygonLabels>
@@ -123,12 +147,16 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "keypoints",
-                "labels": ["Engine", "Tail"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "keypoints",
+                        "classes": ["Engine", "Tail"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <KeyPointLabels name="label" toName="image">
+  <KeyPointLabels name="some_random_field" toName="image">
     <Label value="Engine"/>
     <Label value="Tail"/>
   </KeyPointLabels>
@@ -137,12 +165,16 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "segmentation",
-                "labels": ["Planet", "Moonwalker"],
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "segmentation",
+                        "classes": ["Planet", "Moonwalker"],
+                    }
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <BrushLabels name="label" toName="image">
+  <BrushLabels name="some_random_field" toName="image">
     <Label value="Planet"/>
     <Label value="Moonwalker"/>
   </BrushLabels>
@@ -151,11 +183,58 @@ def label_configs():
         {
             "input": {
                 "media": "image",
-                "label_type": "scalar",
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "scalar",
+                    }
+                },
+            },
+            "output": """<View>
+  <Image name="image" value="$image"/>
+  <Number name="some_random_field" toName="image"/>
+</View>""",
+        },
+        # Multiple fields example. Also check that can have multiple of the same type
+        {
+            "input": {
+                "media": "image",
+                "label_schema": {
+                    "some_random_field": {
+                        "type": "segmentation",
+                        "classes": ["Planet", "Moonwalker"],
+                    },
+                    "some_random_field2": {
+                        "type": "classification",
+                        "classes": ["PlanetClass", "MoonwalkerClass"],
+                    },
+                    "some_random_field3": {
+                        "type": "classification",
+                        "classes": ["PlanetClass2", "MoonwalkerClass2"],
+                    },
+                    "some_random_field4": {
+                        "type": "detection",
+                        "classes": ["PlanetDet", "MoonwalkerDet"],
+                    },
+                },
             },
             "output": """<View>
   <Image name="image" value="$image"/>
-  <Number name="label" toName="image"/>
+  <BrushLabels name="some_random_field" toName="image">
+    <Label value="Planet"/>
+    <Label value="Moonwalker"/>
+  </BrushLabels>
+  <Choices name="some_random_field2" toName="image" choice="single-radio">
+    <Choice value="PlanetClass"/>
+    <Choice value="MoonwalkerClass"/>
+  </Choices>
+  <Choices name="some_random_field3" toName="image" choice="single-radio">
+    <Choice value="PlanetClass2"/>
+    <Choice value="MoonwalkerClass2"/>
+  </Choices>
+  <RectangleLabels name="some_random_field4" toName="image">
+    <Label value="PlanetDet"/>
+    <Label value="MoonwalkerDet"/>
+  </RectangleLabels>
 </View>""",
         },
     ]
@@ -298,17 +377,89 @@ def label_mappings():
                     "type": "keypointlabels",
                 },
             ],
-            "fiftyone": [
-                fol.Keypoint(
-                    label="Tail",
-                    points=[
-                        (0.39, 0.81),
-                        (0.333, 0.771),
-                        (0.424, 0.894),
-                        (0.424, 0.745),
-                    ],
-                ),
+            "fiftyone": fol.Keypoint(
+                label="Tail",
+                points=[
+                    (0.39, 0.81),
+                    (0.333, 0.771),
+                    (0.424, 0.894),
+                    (0.424, 0.745),
+                ],
+            ),
+        },
+        {
+            "labelstudio": [
+                {
+                    "original_width": 600,
+                    "original_height": 403,
+                    "image_rotation": 0,
+                    "value": {
+                        "x": 39.0,
+                        "y": 81.0,
+                        "width": 0.346,
+                        "keypointlabels": ["LEye"],
+                    },
+                    "from_name": "label",
+                    "type": "keypointlabels",
+                },
+                {
+                    "original_width": 600,
+                    "original_height": 403,
+                    "image_rotation": 0,
+                    "value": {
+                        "x": 33.3,
+                        "y": 77.1,
+                        "width": 0.346,
+                        "keypointlabels": ["LEye"],
+                    },
+                    "from_name": "label",
+                    "type": "keypointlabels",
+                },
+                {
+                    "original_width": 600,
+                    "original_height": 403,
+                    "image_rotation": 0,
+                    "value": {
+                        "x": 42.4,
+                        "y": 89.4,
+                        "width": 0.346,
+                        "keypointlabels": ["REye"],
+                    },
+                    "from_name": "label",
+                    "type": "keypointlabels",
+                },
+                {
+                    "original_width": 600,
+                    "original_height": 403,
+                    "image_rotation": 0,
+                    "value": {
+                        "x": 42.4,
+                        "y": 74.5,
+                        "width": 0.346,
+                        "keypointlabels": ["REye"],
+                    },
+                    "from_name": "label",
+                    "type": "keypointlabels",
+                },
             ],
+            "fiftyone": fol.Keypoints(
+                keypoints=[
+                    fol.Keypoint(
+                        label="LEye",
+                        points=[
+                            (0.39, 0.81),
+                            (0.333, 0.771),
+                        ],
+                    ),
+                    fol.Keypoint(
+                        label="REye",
+                        points=[
+                            (0.424, 0.894),
+                            (0.424, 0.745),
+                        ],
+                    ),
+                ]
+            ),
         },
         {
             "labelstudio": {
@@ -324,7 +475,7 @@ def label_mappings():
 
 def test_import_labels(label_mappings):
     for case in label_mappings:
-        label = fouls.import_label_studio_annotation(case["labelstudio"])
+        label = fouls.import_label_studio_annotation(case["labelstudio"])[1]
         expected = case["fiftyone"]
 
         if isinstance(expected, (list, tuple)):
@@ -389,6 +540,81 @@ def test_annotate_simple(backend_config, setup, label_type):
     assert len(labelled) == len(dummy_predictions)
 
 
+def test_annotate_not_all(backend_config, setup):
+    # check that we can load partially annotated data
+    label_type = "classification"
+    dataset, anno_key, label_field, labels = setup
+    label_field += f"-{label_type}"
+    dataset.annotate(
+        anno_key,
+        label_field=label_field,
+        project_name=f"labelstudio_{label_type}_tests",
+        label_type=label_type,
+        classes=labels if label_type != "scalar" else None,
+        backend="labelstudio",
+        **backend_config,
+    )
+
+    # create dummy prediction and convert to annotations
+    dummy_predictions = _generate_dummy_predictions(
+        label_type, labels, len(dataset) - 1
+    )
+    _add_dummy_annotations(
+        dataset, anno_key, dummy_predictions, backend_config
+    )
+
+    dataset.load_annotations(anno_key, cleanup=False, **backend_config)
+
+    labelled = dataset.exists(label_field)
+    assert len(labelled) == len(dummy_predictions)
+    assert len(labelled) == len(dataset) - 1
+
+
+def test_annotate_multiple_fields(backend_config, setup):
+    # check that we can load partially annotated data
+    dataset, anno_key, label_field, labels = setup
+    label_schema = {
+        "cls_field": {
+            "type": "classification",
+            "labels": ["yes", "no"],
+            "classes": ["yes", "no"],
+        },
+        "det_field": {
+            "type": "detections",
+            "classes": ["obj"],
+        },
+    }
+
+    dataset.annotate(
+        anno_key,
+        label_schema=label_schema,
+        project_name=f"labelstudio_multi-field_tests",
+        backend="labelstudio",
+        **backend_config,
+    )
+
+    # create dummy prediction and convert to annotations
+    dummy_predictions_cls = _generate_dummy_predictions(
+        "classification", labels, len(dataset)
+    )
+    dummy_predictions_det = _generate_dummy_predictions(
+        "detections", labels, len(dataset)
+    )
+    _add_dummy_annotations(
+        dataset, anno_key, dummy_predictions_det, backend_config
+    )
+    dataset.load_annotations(anno_key, cleanup=False, **backend_config)
+    labelled = dataset.exists("det_field")
+    assert len(labelled) == len(dataset)
+    # can't load them together, so do it one by one
+    _add_dummy_annotations(
+        dataset, anno_key, dummy_predictions_cls, backend_config
+    )
+    dataset.load_annotations(anno_key, cleanup=False, **backend_config)
+    labelled = dataset.exists("cls_field")
+    assert len(labelled) == len(dataset)
+
+
 @pytest.mark.parametrize(
     "label_type", ["classification", "detections", "polylines", "keypoints"]
 )
@@ -442,6 +668,9 @@ def _assert_labels_equal(converted, expected):
         np.testing.assert_allclose(
             converted.points, expected.points, atol=0.01, rtol=0.01
         )
+    elif expected._cls == "Keypoints":
+        for pair in zip(converted.keypoints, expected.keypoints):
+            _assert_labels_equal(*pair)
     elif expected._cls == "Regression":
         assert expected.value == converted.value
     else:
@@ -509,7 +738,7 @@ def generate_keypoint():
             return [
                 {
                     "type": "keypointlabels",
-                    "from_name": "label",
+                    "from_name": "new_field-keypoints",
                     "to_name": "image",
                     "value": {
                         "x": random.randint(0, 50),