Repository: voxel51/fiftyone

Issue:
[BUG] Evaluations method(s) do not support embedded label fields
In at least the coco detection evaluation method, schema related code prevents evaluation of labels in dynamic embedded documents

```python
import fiftyone as fo
import fiftyone.zoo as foz

dataset = foz.load_zoo_dataset("quickstart", max_samples=1)

for sample in dataset:
    sample["embedded"] = fo.DynamicEmbeddedDocument(
        ground_truth=sample.ground_truth, predictions=sample.predictions
    )
    sample.save()

dataset.delete_sample_field("ground_truth")
dataset.delete_sample_field("predictions")

dataset.add_dynamic_sample_fields()

dataset.evaluate_detections(
    "embedded.predictions", gt_field="embedded.ground_truth"
)
``` 

Out:
```python
Traceback (most recent call last):
  File "/Users/ben/code/fiftyone/fiftyone/core/document.py", line 69, in __getitem__
    return self.get_field(field_name)
  File "/Users/ben/code/fiftyone/fiftyone/core/sample.py", line 99, in get_field
    return super().get_field(field_name)
  File "/Users/ben/code/fiftyone/fiftyone/core/document.py", line 753, in get_field
    raise AttributeError(
AttributeError: Field 'embedded.ground_truth' was not selected on this SampleView

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/ben/code/scratch/eval.py", line 18, in <module>
    dataset.evaluate_detections(
  File "/Users/ben/code/fiftyone/fiftyone/core/collections.py", line 3255, in evaluate_detections
    return foue.evaluate_detections(
  File "/Users/ben/code/fiftyone/fiftyone/utils/eval/detection.py", line 195, in evaluate_detections
    doc_matches = eval_method.evaluate(doc, eval_key=eval_key)
  File "/Users/ben/code/fiftyone/fiftyone/utils/eval/coco.py", line 157, in evaluate
    gts = sample_or_frame[self.gt_field]
  File "/Users/ben/code/fiftyone/fiftyone/core/sample.py", line 60, in __getitem__
    return super().__getitem__(field_name)
  File "/Users/ben/code/fiftyone/fiftyone/core/document.py", line 71, in __getitem__
    raise KeyError(e.args[0])
KeyError: "Field 'embedded.ground_truth' was not selected on this SampleView"
``` 


Discussion:

Resolving patch:
diff --git a/fiftyone/core/document.py b/fiftyone/core/document.py
--- a/fiftyone/core/document.py
+++ b/fiftyone/core/document.py
@@ -654,10 +654,6 @@ def __init__(
         excluded_fields=None,
         filtered_fields=None,
     ):
-        if selected_fields is not None and excluded_fields is not None:
-            selected_fields = selected_fields.difference(excluded_fields)
-            excluded_fields = None
-
         self._view = view
         self._selected_fields = selected_fields
         self._excluded_fields = excluded_fields
@@ -733,7 +729,7 @@ def has_field(self, field_name):
             return False
 
         sf = self._selected_fields
-        if sf is not None and field_name not in sf:
+        if sf is not None and field_name.split(".", 1)[0] not in sf:
             return False
 
         return super().has_field(field_name)
@@ -746,16 +742,14 @@ def get_field(self, field_name):
                 % (field_name, self.__class__.__name__)
             )
 
-        value = super().get_field(field_name)
-
         sf = self._selected_fields
-        if sf is not None and field_name not in sf:
+        if sf is not None and field_name.split(".", 1)[0] not in sf:
             raise AttributeError(
                 "Field '%s' was not selected on this %s"
                 % (field_name, self.__class__.__name__)
             )
 
-        return value
+        return super().get_field(field_name)
 
     def set_field(
         self,
@@ -785,12 +779,13 @@ def set_field(
                 dynamic=dynamic,
             )
 
+            if self._selected_fields is not None:
+                root = field_name.split(".", 1)[0]
+                self._selected_fields.add(root)
+
             if self._excluded_fields is not None:
                 self._excluded_fields.discard(field_name)
 
-            if self._selected_fields is not None:
-                self._selected_fields.add(field_name)
-
     def clear_field(self, field_name):
         # Ensures field exists
         _ = self.get_field(field_name)
@@ -800,7 +795,10 @@ def clear_field(self, field_name):
     def to_dict(self, include_private=False):
         d = super().to_dict(include_private=include_private)
 
-        if self._selected_fields or self._excluded_fields:
+        if (
+            self._selected_fields is not None
+            or self._excluded_fields is not None
+        ):
             field_names = set(
                 self._get_field_names(
                     include_private=include_private,
@@ -815,7 +813,10 @@ def to_dict(self, include_private=False):
     def to_mongo_dict(self, include_id=False):
         d = super().to_mongo_dict(include_id=include_id)
 
-        if self._selected_fields or self._excluded_fields:
+        if (
+            self._selected_fields is not None
+            or self._excluded_fields is not None
+        ):
             field_names = set(
                 self._get_field_names(include_private=True, use_db_fields=True)
             )
diff --git a/fiftyone/core/odm/document.py b/fiftyone/core/odm/document.py
--- a/fiftyone/core/odm/document.py
+++ b/fiftyone/core/odm/document.py
@@ -306,9 +306,12 @@ def has_field(self, field_name):
     def get_field(self, field_name):
         chunks = field_name.split(".", 1)
         if len(chunks) > 1:
-            value = getattr(self, chunks[0])
-            if value is not None:
-                return value.get_field(chunks[1])
+            try:
+                value = getattr(self, chunks[0])
+                if value is not None:
+                    return value.get_field(chunks[1])
+            except AttributeError:
+                pass
 
             if self.has_field(field_name):
                 return None
diff --git a/fiftyone/core/validation.py b/fiftyone/core/validation.py
--- a/fiftyone/core/validation.py
+++ b/fiftyone/core/validation.py
@@ -216,22 +216,21 @@ def _validate_fields(
     same_type,
     frames=False,
 ):
-    if frames:
-        schema = sample_collection.get_frame_field_schema()
-    else:
-        schema = sample_collection.get_field_schema()
-
     label_types = {}
     for field_name in field_names:
-        if field_name not in schema:
+        path = field_name
+        if frames:
+            path = sample_collection._FRAMES_PREFIX + path
+
+        field = sample_collection.get_field(path)
+
+        if field is None:
             ftype = "frame field" if frames else "sample field"
             raise ValueError(
                 "%s has no %s '%s'"
                 % (sample_collection.__class__.__name__, ftype, field_name)
             )
 
-        field = schema[field_name]
-
         try:
             label_type = field.document_type
         except:


Test patch:
diff --git a/tests/unittests/evaluation_tests.py b/tests/unittests/evaluation_tests.py
--- a/tests/unittests/evaluation_tests.py
+++ b/tests/unittests/evaluation_tests.py
@@ -135,6 +135,28 @@ def test_evaluate_regressions_simple(self):
         self.assertNotIn("eval2", dataset.list_evaluations())
         self.assertNotIn("eval2", dataset.get_field_schema())
 
+    @drop_datasets
+    def test_evaluate_regressions_embedded_fields(self):
+        dataset = self._make_regression_dataset()
+
+        dataset.add_sample_field(
+            "embedded",
+            fo.EmbeddedDocumentField,
+            embedded_doc_type=fo.DynamicEmbeddedDocument,
+        )
+
+        dataset.rename_sample_field("predictions", "embedded.predictions")
+        dataset.rename_sample_field("ground_truth", "embedded.ground_truth")
+
+        results = dataset.evaluate_regressions(
+            "embedded.predictions",
+            gt_field="embedded.ground_truth",
+            eval_key="eval",
+            method="simple",
+        )
+
+        results.print_metrics()
+
     def test_custom_regression_evaluation(self):
         dataset = self._make_regression_dataset()
 
@@ -603,6 +625,29 @@ def test_evaluate_classifications_binary(self):
         self.assertNotIn("eval2", dataset.list_evaluations())
         self.assertNotIn("eval2", dataset.get_field_schema())
 
+    @drop_datasets
+    def test_evaluate_classifications_embedded_fields(self):
+        dataset = self._make_classification_dataset()
+
+        dataset.add_sample_field(
+            "embedded",
+            fo.EmbeddedDocumentField,
+            embedded_doc_type=fo.DynamicEmbeddedDocument,
+        )
+
+        dataset.rename_sample_field("predictions", "embedded.predictions")
+        dataset.rename_sample_field("ground_truth", "embedded.ground_truth")
+
+        results = dataset.evaluate_classifications(
+            "embedded.predictions",
+            gt_field="embedded.ground_truth",
+            eval_key="eval",
+            method="simple",
+        )
+
+        results.report()
+        results.print_report()
+
     def test_custom_classification_evaluation(self):
         dataset = self._make_classification_dataset()
 
@@ -1732,6 +1777,44 @@ def test_load_evaluation_view_select_fields(self):
         with self.assertRaises(KeyError):
             detection["eval2"]
 
+    @drop_datasets
+    def test_evaluate_detections_embedded_fields(self):
+        dataset = self._make_detections_dataset()
+
+        dataset.add_sample_field(
+            "embedded",
+            fo.EmbeddedDocumentField,
+            embedded_doc_type=fo.DynamicEmbeddedDocument,
+        )
+
+        dataset.rename_sample_field("predictions", "embedded.predictions")
+        dataset.rename_sample_field("ground_truth", "embedded.ground_truth")
+
+        _, gt_eval_field = dataset._get_label_field_path(
+            "embedded.ground_truth", "eval"
+        )
+        _, pred_eval_field = dataset._get_label_field_path(
+            "embedded.predictions", "eval"
+        )
+
+        results = dataset.evaluate_detections(
+            "embedded.predictions",
+            gt_field="embedded.ground_truth",
+            eval_key="eval",
+            method="coco",
+        )
+
+        schema = dataset.get_field_schema(flat=True)
+        self.assertIn("eval_tp", schema)
+        self.assertIn("eval_fp", schema)
+        self.assertIn("eval_fn", schema)
+        self.assertIn(gt_eval_field, schema)
+        self.assertIn(gt_eval_field + "_id", schema)
+        self.assertIn(gt_eval_field + "_iou", schema)
+        self.assertIn(pred_eval_field, schema)
+        self.assertIn(pred_eval_field + "_id", schema)
+        self.assertIn(pred_eval_field + "_iou", schema)
+
     def test_custom_detection_evaluation(self):
         dataset = self._make_detections_dataset()
 
@@ -2769,6 +2852,33 @@ def test_evaluate_segmentations_rgb(self):
         self.assertNotIn("eval2_precision", dataset.get_field_schema())
         self.assertNotIn("eval2_recall", dataset.get_field_schema())
 
+    @drop_datasets
+    def test_evaluate_segmentations_embedded_fields(self):
+        dataset = self._make_segmentation_dataset()
+
+        dataset.add_sample_field(
+            "embedded",
+            fo.EmbeddedDocumentField,
+            embedded_doc_type=fo.DynamicEmbeddedDocument,
+        )
+
+        dataset.rename_sample_field("predictions", "embedded.predictions")
+        dataset.rename_sample_field("ground_truth", "embedded.ground_truth")
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore")  # suppress missing masks warning
+
+            results = dataset.evaluate_segmentations(
+                "embedded.predictions",
+                gt_field="embedded.ground_truth",
+                eval_key="eval",
+                method="simple",
+                mask_targets={0: "background", 1: "cat", 2: "dog"},
+            )
+
+        results.report()
+        results.print_report()
+
     def test_custom_segmentation_evaluation(self):
         dataset = self._make_segmentation_dataset()
 
diff --git a/tests/unittests/operators/decorators_tests.py b/tests/unittests/operators/decorators_tests.py
--- a/tests/unittests/operators/decorators_tests.py
+++ b/tests/unittests/operators/decorators_tests.py
@@ -56,7 +56,7 @@ def test_rgrs_dir_state_change_with_delete(self):
         with tempfile.TemporaryDirectory() as tmpdirname:
             initial_dir_state = dir_state(tmpdirname)
             for p in plugin_paths:
-                time.sleep(0.01)
+                time.sleep(0.1)
                 os.makedirs(os.path.join(tmpdirname, p))
 
             # verify that max time is greater after adding files
@@ -69,7 +69,7 @@ def test_rgrs_dir_state_change_with_delete(self):
             )
             dir_state2 = dir_state(tmpdirname)
             self.assertGreaterEqual(dir_state2, dir_state1)
-            time.sleep(0.01)
+            time.sleep(0.1)
 
             shutil.rmtree(
                 os.path.join(tmpdirname, plugin_paths[1].rsplit("/", 1)[0])
@@ -82,11 +82,11 @@ def test_rgrs_dir_state_change_with_rename(self):
         with tempfile.TemporaryDirectory() as tmpdirname:
             initial_dir_state = dir_state(tmpdirname)
             for p in plugin_paths:
-                time.sleep(0.01)
+                time.sleep(0.1)
                 os.makedirs(os.path.join(tmpdirname, p))
 
             # add wait for test to pass on older systems/python versions
-            time.sleep(0.01)
+            time.sleep(0.1)
 
             # verify that max time is greater after adding files
             dir_state1 = dir_state(tmpdirname)