Repository: voxel51/fiftyone

Issue:
[BUG] merge_samples() fails on video datasets
```py
import fiftyone as fo
import fiftyone.zoo as foz

dataset = foz.load_zoo_dataset("quickstart-video")

# Should work but currently fails
dataset2 = dataset[:3].clone()
dataset2.merge_samples(dataset[3:6])
dataset2.merge_samples(dataset[6:])

# This works though
dataset3 = dataset[:3].clone()
dataset3.add_collection(dataset[3:6], new_ids=True)
dataset3.add_collection(dataset[6:], new_ids=True)
```

```
DuplicateKeyError: Index build failed: 5d5bd97f-24a5-495b-8097-b43f0d187d38: Collection fiftyone.frames.samples.647a9273f5959fd1b57b17ab ( 2f9c7bc2-e25b-43b4-8cb9-14d4ad458710 ) :: caused by :: E11000 duplicate key error collection: fiftyone.frames.samples.647a9273f5959fd1b57b17ab index: _merge_key_1_frame_number_1 dup key: { _merge_key: null, frame_number: 1 }, full error: {'ok': 0.0, 'errmsg': 'Index build failed: 5d5bd97f-24a5-495b-8097-b43f0d187d38: Collection fiftyone.frames.samples.647a9273f5959fd1b57b17ab ( 2f9c7bc2-e25b-43b4-8cb9-14d4ad458710 ) :: caused by :: E11000 duplicate key error collection: fiftyone.frames.samples.647a9273f5959fd1b57b17ab index: _merge_key_1_frame_number_1 dup key: { _merge_key: null, frame_number: 1 }', 'code': 11000, 'codeName': 'DuplicateKey', 'keyPattern': {'_merge_key': 1, 'frame_number': 1}, 'keyValue': {'_merge_key': None, 'frame_number': 1}}
```



Discussion:

Resolving patch:
diff --git a/fiftyone/core/dataset.py b/fiftyone/core/dataset.py
--- a/fiftyone/core/dataset.py
+++ b/fiftyone/core/dataset.py
@@ -7694,8 +7694,8 @@ def _merge_samples_pipeline(
         )
 
         if contains_videos:
-            _index_frames(dst_videos, key_field, frame_key_field)
-            _index_frames(src_videos, key_field, frame_key_field)
+            _index_frames(dst_dataset, key_field, frame_key_field)
+            _index_frames(src_dataset, key_field, frame_key_field)
 
             # Create unique index on frame merge key
             frame_index_spec = [(frame_key_field, 1), ("frame_number", 1)]
@@ -7979,8 +7979,13 @@ def _merge_label_list_field(doc, elem_field, overwrite=False):
     }
 
 
-def _index_frames(sample_collection, key_field, frame_key_field):
-    ids, keys, all_sample_ids = sample_collection.values(
+def _index_frames(dataset, key_field, frame_key_field):
+    if dataset.media_type == fom.GROUP:
+        dst_videos = dataset.select_group_slices(media_type=fom.VIDEO)
+    else:
+        dst_videos = dataset
+
+    ids, keys, all_sample_ids = dst_videos.values(
         ["_id", key_field, "frames._sample_id"]
     )
     keys_map = {k: v for k, v in zip(ids, keys)}
@@ -7994,7 +7999,7 @@ def _index_frames(sample_collection, key_field, frame_key_field):
 
         frame_keys.append(sample_keys)
 
-    sample_collection.set_values(
+    dst_videos.set_values(
         "frames." + frame_key_field,
         frame_keys,
         expand_schema=False,


Test patch:
diff --git a/tests/unittests/video_tests.py b/tests/unittests/video_tests.py
--- a/tests/unittests/video_tests.py
+++ b/tests/unittests/video_tests.py
@@ -913,6 +913,32 @@ def test_video_frames_merge(self):
         self.assertEqual(frame4["field1"], "a")
         self.assertEqual(frame4["field2"], "b")
 
+    @drop_datasets
+    def test_video_frames_view_merge(self):
+        sample1 = fo.Sample(filepath="video1.mp4")
+        sample1.frames[1] = fo.Frame(field1="a", field2="b")
+        sample1.frames[2] = fo.Frame()
+
+        sample2 = fo.Sample(filepath="video2.mp4")
+        sample2.frames[1] = fo.Frame(field1="c", field2="d")
+
+        sample3 = fo.Sample(filepath="video3.mp4")
+        sample3.frames[1] = fo.Frame(field1="e", field2="f")
+        sample3.frames[2] = fo.Frame()
+        sample3.frames[3] = fo.Frame()
+
+        dataset1 = fo.Dataset()
+        dataset1.add_samples([sample1, sample2, sample3])
+
+        dataset2 = fo.Dataset()
+        dataset2.media_type = "video"
+
+        dataset2.merge_samples(dataset1[:1])
+        dataset2.merge_samples(dataset1[1:])
+
+        self.assertEqual(dataset2.count(), 3)
+        self.assertEqual(dataset2.count("frames"), 6)
+
     @drop_datasets
     def test_merge_video_samples_and_labels(self):
         sample11 = fo.Sample(filepath="video1.mp4")