Repository: voxel51/fiftyone

Issue:
[BUG] Unable to import FiftyOneDataset that was exported (both through web interface and code).
### Describe the problem

I am unable to import samples from an exported FiftyOneDataset. I will include the logs in the last section. I built a dataset and exported it into a FiftyOneDataset. This was done both using the web interface where you press `\` ` and type `Export Samples -> Media and Labels ->  FiftyOne Dataset` as well as through code like below:

### Code to reproduce issue

```
final = dataset.load_saved_view("finalview")
final.export(
    export_dir='/shared/export/',
    dataset_type=fo.types.FiftyOneDataset,
    export_media=True,
    progress=True
)
```

### System information

-   **OS Platform and Distribution** (e.g., Linux Ubuntu 22.04): WSL2 - Ubuntu 22.04
-   **Python version** (`python --version`): 3.11
-   **FiftyOne version** (`fiftyone --version`): 1.3.1
-   **FiftyOne installed from** (pip or source): Using docker image `voxel51/fiftyone:1.3.1`

### Other info/logs

```
2025-03-11 21:09:48 [2025-03-12 04:09:48 +0000] [8] [ERROR] Error in ASGI Framework
2025-03-11 21:09:48 Traceback (most recent call last):
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 270, in __call__
2025-03-11 21:09:48     await wrap(partial(self.listen_for_disconnect, receive))
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 266, in wrap
2025-03-11 21:09:48     await func()
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 234, in listen_for_disconnect
2025-03-11 21:09:48     message = await receive()
2025-03-11 21:09:48               ^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 113, in receive_or_disconnect
2025-03-11 21:09:48     async with anyio.create_task_group() as task_group:
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 771, in __aexit__
2025-03-11 21:09:48     raise exc_val
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 121, in receive_or_disconnect
2025-03-11 21:09:48     message = await wrap(wrapped_receive)
2025-03-11 21:09:48               ^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 116, in wrap
2025-03-11 21:09:48     result = await func()
2025-03-11 21:09:48              ^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 50, in wrapped_receive
2025-03-11 21:09:48     msg = await self.receive()
2025-03-11 21:09:48           ^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/usr/local/lib/python3.11/asyncio/queues.py", line 158, in get
2025-03-11 21:09:48     await getter
2025-03-11 21:09:48 asyncio.exceptions.CancelledError: Cancelled by cancel scope 7fdf8fb167d0
2025-03-11 21:09:48 
2025-03-11 21:09:48 During handling of the above exception, another exception occurred:
2025-03-11 21:09:48 
2025-03-11 21:09:48   + Exception Group Traceback (most recent call last):
2025-03-11 21:09:48   |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/_utils.py", line 76, in collapse_excgroups
2025-03-11 21:09:48   |     yield
2025-03-11 21:09:48   |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 263, in __call__
2025-03-11 21:09:48   |     async with anyio.create_task_group() as task_group:
2025-03-11 21:09:48   |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 767, in __aexit__
2025-03-11 21:09:48   |     raise BaseExceptionGroup(
2025-03-11 21:09:48   | ExceptionGroup: unhandled errors in a TaskGroup (1 sub-exception)
2025-03-11 21:09:48   +-+---------------- 1 ----------------
2025-03-11 21:09:48     | Traceback (most recent call last):
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/hypercorn/asyncio/task_group.py", line 27, in _handle
2025-03-11 21:09:48     |     await app(scope, receive, send, sync_spawn, call_soon)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/hypercorn/app_wrappers.py", line 34, in __call__
2025-03-11 21:09:48     |     await self.app(scope, receive, send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/applications.py", line 112, in __call__
2025-03-11 21:09:48     |     await self.middleware_stack(scope, receive, send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 187, in __call__
2025-03-11 21:09:48     |     raise exc
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 165, in __call__
2025-03-11 21:09:48     |     await self.app(scope, receive, _send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/cors.py", line 93, in __call__
2025-03-11 21:09:48     |     await self.simple_response(scope, receive, send, request_headers=headers)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/cors.py", line 144, in simple_response
2025-03-11 21:09:48     |     await self.app(scope, receive, send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 181, in __call__
2025-03-11 21:09:48     |     raise app_exc
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 140, in coro
2025-03-11 21:09:48     |     await self.app(scope, receive_or_disconnect, send_no_error)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
2025-03-11 21:09:48     |     await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
2025-03-11 21:09:48     |     raise exc
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
2025-03-11 21:09:48     |     await app(scope, receive, sender)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/routing.py", line 714, in __call__
2025-03-11 21:09:48     |     await self.middleware_stack(scope, receive, send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/routing.py", line 734, in app
2025-03-11 21:09:48     |     await route.handle(scope, receive, send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/routing.py", line 288, in handle
2025-03-11 21:09:48     |     await self.app(scope, receive, send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/endpoints.py", line 41, in dispatch
2025-03-11 21:09:48     |     await response(self.scope, self.receive, self.send)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 262, in __call__
2025-03-11 21:09:48     |     with collapse_excgroups():
2025-03-11 21:09:48     |   File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
2025-03-11 21:09:48     |     self.gen.throw(typ, value, traceback)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
2025-03-11 21:09:48     |     raise exc
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 266, in wrap
2025-03-11 21:09:48     |     await func()
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 246, in stream_response
2025-03-11 21:09:48     |     async for chunk in self.body_iterator:
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/concurrency.py", line 60, in iterate_in_threadpool
2025-03-11 21:09:48     |     yield await anyio.to_thread.run_sync(_next, as_iterator)
2025-03-11 21:09:48     |           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/to_thread.py", line 56, in run_sync
2025-03-11 21:09:48     |     return await get_async_backend().run_sync_in_worker_thread(
2025-03-11 21:09:48     |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 2461, in run_sync_in_worker_thread
2025-03-11 21:09:48     |     return await future
2025-03-11 21:09:48     |            ^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 962, in run
2025-03-11 21:09:48     |     result = context.run(func, *args)
2025-03-11 21:09:48     |              ^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/concurrency.py", line 49, in _next
2025-03-11 21:09:48     |     return next(iterator)
2025-03-11 21:09:48     |            ^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/operators/server.py", line 114, in create_response_generator
2025-03-11 21:09:48     |     for generated_message in generator:
2025-03-11 21:09:48     |   File "/fiftyone/default/__plugins__/@voxel51/io/__init__.py", line 183, in execute
2025-03-11 21:09:48     |     for update in _import_media_and_labels(ctx):
2025-03-11 21:09:48     |   File "/fiftyone/default/__plugins__/@voxel51/io/__init__.py", line 862, in _import_media_and_labels
2025-03-11 21:09:48     |     ctx.dataset.add_dir(
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 5326, in add_dir
2025-03-11 21:09:48     |     return self.add_importer(
2025-03-11 21:09:48     |            ^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 5922, in add_importer
2025-03-11 21:09:48     |     return foud.import_samples(
2025-03-11 21:09:48     |            ^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/utils/data/importers.py", line 123, in import_samples
2025-03-11 21:09:48     |     return dataset_importer.import_samples(
2025-03-11 21:09:48     |            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/utils/data/importers.py", line 1812, in import_samples
2025-03-11 21:09:48     |     return self._import_samples(
2025-03-11 21:09:48     |            ^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/utils/data/importers.py", line 1890, in _import_samples
2025-03-11 21:09:48     |     dataset._merge_doc(new_doc)
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 3505, in _merge_doc
2025-03-11 21:09:48     |     _merge_dataset_doc(
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 8903, in _merge_dataset_doc
2025-03-11 21:09:48     |     dataset._sample_doc_cls.merge_field_schema(
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/odm/mixins.py", line 312, in merge_field_schema
2025-03-11 21:09:48     |     validate_field_name(
2025-03-11 21:09:48     |   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/odm/utils.py", line 146, in validate_field_name
2025-03-11 21:09:48     |     raise ValueError(
2025-03-11 21:09:48     | ValueError: Invalid field name: '_dataset_id'. Field names cannot start with '_'
2025-03-11 21:09:48     +------------------------------------
2025-03-11 21:09:48 
2025-03-11 21:09:48 During handling of the above exception, another exception occurred:
2025-03-11 21:09:48 
2025-03-11 21:09:48 Traceback (most recent call last):
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/hypercorn/asyncio/task_group.py", line 27, in _handle
2025-03-11 21:09:48     await app(scope, receive, send, sync_spawn, call_soon)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/hypercorn/app_wrappers.py", line 34, in __call__
2025-03-11 21:09:48     await self.app(scope, receive, send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/applications.py", line 112, in __call__
2025-03-11 21:09:48     await self.middleware_stack(scope, receive, send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 187, in __call__
2025-03-11 21:09:48     raise exc
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/errors.py", line 165, in __call__
2025-03-11 21:09:48     await self.app(scope, receive, _send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/cors.py", line 93, in __call__
2025-03-11 21:09:48     await self.simple_response(scope, receive, send, request_headers=headers)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/cors.py", line 144, in simple_response
2025-03-11 21:09:48     await self.app(scope, receive, send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 181, in __call__
2025-03-11 21:09:48     raise app_exc
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/base.py", line 140, in coro
2025-03-11 21:09:48     await self.app(scope, receive_or_disconnect, send_no_error)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
2025-03-11 21:09:48     await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
2025-03-11 21:09:48     raise exc
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
2025-03-11 21:09:48     await app(scope, receive, sender)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/routing.py", line 714, in __call__
2025-03-11 21:09:48     await self.middleware_stack(scope, receive, send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/routing.py", line 734, in app
2025-03-11 21:09:48     await route.handle(scope, receive, send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/routing.py", line 288, in handle
2025-03-11 21:09:48     await self.app(scope, receive, send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/endpoints.py", line 41, in dispatch
2025-03-11 21:09:48     await response(self.scope, self.receive, self.send)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 262, in __call__
2025-03-11 21:09:48     with collapse_excgroups():
2025-03-11 21:09:48   File "/usr/local/lib/python3.11/contextlib.py", line 158, in __exit__
2025-03-11 21:09:48     self.gen.throw(typ, value, traceback)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/_utils.py", line 82, in collapse_excgroups
2025-03-11 21:09:48     raise exc
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 266, in wrap
2025-03-11 21:09:48     await func()
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/responses.py", line 246, in stream_response
2025-03-11 21:09:48     async for chunk in self.body_iterator:
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/concurrency.py", line 60, in iterate_in_threadpool
2025-03-11 21:09:48     yield await anyio.to_thread.run_sync(_next, as_iterator)
2025-03-11 21:09:48           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/to_thread.py", line 56, in run_sync
2025-03-11 21:09:48     return await get_async_backend().run_sync_in_worker_thread(
2025-03-11 21:09:48            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 2461, in run_sync_in_worker_thread
2025-03-11 21:09:48     return await future
2025-03-11 21:09:48            ^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/anyio/_backends/_asyncio.py", line 962, in run
2025-03-11 21:09:48     result = context.run(func, *args)
2025-03-11 21:09:48              ^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/starlette/concurrency.py", line 49, in _next
2025-03-11 21:09:48     return next(iterator)
2025-03-11 21:09:48            ^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/operators/server.py", line 114, in create_response_generator
2025-03-11 21:09:48     for generated_message in generator:
2025-03-11 21:09:48   File "/fiftyone/default/__plugins__/@voxel51/io/__init__.py", line 183, in execute
2025-03-11 21:09:48     for update in _import_media_and_labels(ctx):
2025-03-11 21:09:48   File "/fiftyone/default/__plugins__/@voxel51/io/__init__.py", line 862, in _import_media_and_labels
2025-03-11 21:09:48     ctx.dataset.add_dir(
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 5326, in add_dir
2025-03-11 21:09:48     return self.add_importer(
2025-03-11 21:09:48            ^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 5922, in add_importer
2025-03-11 21:09:48     return foud.import_samples(
2025-03-11 21:09:48            ^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/utils/data/importers.py", line 123, in import_samples
2025-03-11 21:09:48     return dataset_importer.import_samples(
2025-03-11 21:09:48            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/utils/data/importers.py", line 1812, in import_samples
2025-03-11 21:09:48     return self._import_samples(
2025-03-11 21:09:48            ^^^^^^^^^^^^^^^^^^^^^
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/utils/data/importers.py", line 1890, in _import_samples
2025-03-11 21:09:48     dataset._merge_doc(new_doc)
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 3505, in _merge_doc
2025-03-11 21:09:48     _merge_dataset_doc(
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/dataset.py", line 8903, in _merge_dataset_doc
2025-03-11 21:09:48     dataset._sample_doc_cls.merge_field_schema(
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/odm/mixins.py", line 312, in merge_field_schema
2025-03-11 21:09:48     validate_field_name(
2025-03-11 21:09:48   File "/opt/.fiftyone-venv/lib/python3.11/site-packages/fiftyone/core/odm/utils.py", line 146, in validate_field_name
2025-03-11 21:09:48     raise ValueError(
2025-03-11 21:09:48 ValueError: Invalid field name: '_dataset_id'. Field names cannot start with '_'
```

### Willingness to contribute

The FiftyOne Community encourages bug fix contributions. Would you or another
member of your organization be willing to contribute a fix for this bug to the
FiftyOne codebase?

-   [ ] Yes. I can contribute a fix for this bug independently
-   [ ] Yes. I would be willing to contribute a fix for this bug with guidance
        from the FiftyOne community
-   [X] No. I cannot contribute a bug fix at this time

Discussion:
@brimoor: Hi @bhoo-git 👋 

It looks like you're using the [`@voxel51/io`](https://github.com/voxel51/fiftyone-plugins/tree/main/plugins/io) to export and import data in `FiftyOneDataset` format.

Could you provide more information about the data you're trying to export? Ideally a reproducible example of the dataset as a zip file?

As the video below shows, I tried the workflow with the `quickstart` dataset from our zoo and everything worked as expected:

https://github.com/user-attachments/assets/faad5480-d0f3-4b7b-8a2d-43c12b834327
@bhoo-git: I think the main difference for me was that I was trying to import the exported dataset into a different dataset that already has samples. I'm not sure if it's intended to be this way or if I am doing something wrong, but I am looking to merge samples from multiple datasets into one. below is a recording of the workflow I was attempting.

https://github.com/user-attachments/assets/6e40b3a2-b463-437c-bd06-896f5c1f3fb4@brimoor: Thanks for the reproducible example! I've fixed the bug in https://github.com/voxel51/fiftyone/pull/5586 which will be included in `fiftyone==1.4.0`, scheduled for release March 21

Solution patch:
diff --git a/app/packages/embeddings/src/EmbeddingsPlot.tsx b/app/packages/embeddings/src/EmbeddingsPlot.tsx
index 457fa5623..4eb7f3693 100644
--- a/app/packages/embeddings/src/EmbeddingsPlot.tsx
+++ b/app/packages/embeddings/src/EmbeddingsPlot.tsx
@@ -92,10 +92,10 @@ export function EmbeddingsPlot({
               result[p.fullData.name].push(p.id);
               pointIds.push(p.id);
             }
-            handleSelected(pointIds);
+            handleSelected(pointIds, selected.lassoPoints);
           }}
           onDeselect={() => {
-            handleSelected(null);
+            handleSelected(null, null);
           }}
           config={{
             scrollZoom: true,
diff --git a/app/packages/embeddings/src/tracesToData.tsx b/app/packages/embeddings/src/tracesToData.tsx
index 120cab532..be6ba0756 100644
--- a/app/packages/embeddings/src/tracesToData.tsx
+++ b/app/packages/embeddings/src/tracesToData.tsx
@@ -20,41 +20,63 @@ export function tracesToData(
   const isCategorical = style === "categorical";
   const isContinuous = style === "continuous";
   const isUncolored = style === "uncolored";
+
+  const hasSelection = Array.isArray(plotSelection) && plotSelection.length > 0;
+  const selectionSet = hasSelection ? new Set(plotSelection) : null;
+
+  const mappedColorscale = colorscale.map(([r, g, b], idx) => {
+    const color = Color.fromCSSRGBValues(r, g, b);
+    return [idx / (colorscale.length - 1), color.toCSSRGBString()];
+  });
+
   return Object.entries(traces)
     .sort((a, b) => sortStringsAlphabetically(a[0], b[0]))
     .map(addLineBreaks)
     .map(([key, trace]) => {
-      const selectedpoints = plotSelection?.length
-        ? plotSelection
-            .map((id) => getPointIndex(trace, id))
-            .filter((p) => p !== null)
-        : null;
       const color =
         getLabelColor(key, setting) ??
         getConvertedColor(getColor(key)) ??
         new Color(255, 165, 0);
 
-      const mappedColorscale = colorscale.map(
-        (c: [number, number, number], idx) => {
-          const color = Color.fromCSSRGBValues(...c);
-          return [idx / (colorscale.length - 1), color.toCSSRGBString()];
+      const x = new Array(trace.length);
+      const y = new Array(trace.length);
+      const ids = new Array(trace.length);
+      const labelsForColors =
+        isContinuous && !isUncolored ? new Array(trace.length) : null;
+
+      const selectedpoints = [];
+
+      for (let i = 0; i < trace.length; i++) {
+        const { points, id, sample_id, label } = trace[i];
+        x[i] = points[0];
+        y[i] = points[1];
+        ids[i] = id;
+
+        if (labelsForColors) {
+          labelsForColors[i] = label;
+        }
+
+        if (hasSelection && selectionSet) {
+          if (selectionSet.has(id) || selectionSet.has(sample_id)) {
+            selectedpoints.push(i);
+          }
         }
-      );
+      }
 
       return {
-        x: trace.map((d) => d.points[0]),
-        y: trace.map((d) => d.points[1]),
-        ids: trace.map((d) => d.id),
+        x,
+        y,
+        ids,
         type: "scattergl",
         mode: "markers",
         marker: {
-          autocolorscale: !isContinuous, // isCategorical || isUncolored,
+          autocolorscale: !isContinuous,
           colorscale: mappedColorscale,
           color: isCategorical
             ? color.toCSSRGBString()
             : isUncolored
             ? null
-            : trace.map((d) => d.label),
+            : labelsForColors,
           size: 6,
           colorbar:
             isCategorical || isUncolored
@@ -66,7 +88,7 @@ export function tracesToData(
                 },
         },
         name: key,
-        selectedpoints,
+        selectedpoints: selectedpoints.length ? selectedpoints : null,
         selected: {
           marker: {
             opacity: 1,
diff --git a/app/packages/embeddings/src/useBrainResult.tsx b/app/packages/embeddings/src/useBrainResult.tsx
index 784ea7e5b..ed7b6504f 100644
--- a/app/packages/embeddings/src/useBrainResult.tsx
+++ b/app/packages/embeddings/src/useBrainResult.tsx
@@ -6,6 +6,7 @@ import { useColorByField } from "./useLabelSelector";
 // a react hook that fetches a list of brain results
 // and has a loading state and an error state
 export const useBrainResult = () => usePanelStatePartial("brainResult", null);
+export const usePointsField = () => usePanelStatePartial("pointsField", null);
 export function useBrainResultsSelector() {
   const [selected, setSelected] = useBrainResult();
   const dataset = useRecoilValue(fos.dataset);
diff --git a/app/packages/embeddings/src/useExtendedStageEffect.tsx b/app/packages/embeddings/src/useExtendedStageEffect.tsx
index 0602a8691..6b4868f9a 100644
--- a/app/packages/embeddings/src/useExtendedStageEffect.tsx
+++ b/app/packages/embeddings/src/useExtendedStageEffect.tsx
@@ -3,6 +3,8 @@ import { useRecoilCallback, useRecoilValue, useSetRecoilState } from "recoil";
 import * as fos from "@fiftyone/state";
 import { usePanelStatePartial } from "@fiftyone/spaces";
 import { fetchExtendedStage } from "./fetch";
+import { atoms as selectionAtoms } from "./usePlotSelection";
+import { usePointsField } from "./useBrainResult";
 
 export default function useExtendedStageEffect() {
   const datasetName = useRecoilValue(fos.datasetName);
@@ -16,6 +18,8 @@ export default function useExtendedStageEffect() {
     return snapshot.getPromise(fos.datasetName);
   });
   const slices = useRecoilValue(fos.currentSlices(false));
+  const lassoPoints = useRecoilValue(selectionAtoms.lassoPoints);
+  const [pointsField] = usePointsField();
 
   useEffect(() => {
     if (loadedPlot && Array.isArray(selection)) {
@@ -25,6 +29,8 @@ export default function useExtendedStageEffect() {
         patchesField: loadedPlot.patches_field,
         selection,
         slices,
+        lassoPoints,
+        pointsField,
       }).then(async (res) => {
         const currentDataset = await getCurrentDataset();
         if (currentDataset !== datasetName) return;
@@ -33,5 +39,12 @@ export default function useExtendedStageEffect() {
         });
       });
     }
-  }, [datasetName, loadedPlot?.patches_field, view, selection]);
+  }, [
+    datasetName,
+    loadedPlot?.patches_field,
+    view,
+    selection,
+    pointsField,
+    lassoPoints,
+  ]);
 }
diff --git a/app/packages/embeddings/src/usePlotSelection.tsx b/app/packages/embeddings/src/usePlotSelection.tsx
index 8ace6685a..1f6dd2299 100644
--- a/app/packages/embeddings/src/usePlotSelection.tsx
+++ b/app/packages/embeddings/src/usePlotSelection.tsx
@@ -1,10 +1,22 @@
-import { useRecoilState, useRecoilValue, useSetRecoilState } from "recoil";
+import {
+  atom,
+  useRecoilState,
+  useRecoilValue,
+  useSetRecoilState,
+} from "recoil";
 import * as fos from "@fiftyone/state";
 import { usePanelStatePartial } from "@fiftyone/spaces";
 import { useBrainResultInfo } from "./useBrainResultInfo";
 import { SELECTION_SCOPE } from "./constants";
 import { useResetExtendedSelection } from "@fiftyone/state";
 
+export const atoms = {
+  lassoPoints: atom({
+    key: "lassoPoints",
+    default: { x: [], y: [] },
+  }),
+};
+
 export function usePlotSelection() {
   const brainResultInfo = useBrainResultInfo();
   const patchesField = brainResultInfo?.config?.patchesField;
@@ -20,14 +32,22 @@ export function usePlotSelection() {
     [],
     true
   );
+  const [lassoPoints, setLassoPoints] = useRecoilState(atoms.lassoPoints);
   const selectedPatchIds = useRecoilValue(fos.selectedPatchIds(patchesField));
   const selectedPatchSampleIds = useRecoilValue(fos.selectedPatchSamples);
-  function handleSelected(selectedResults) {
+  function handleSelected(
+    selectedResults,
+    lassoPoints: { x: number[]; y: number[] }
+  ) {
     setSelectedSamples(new Set());
     setExtendedSelection({
       selection: selectedResults,
       scope: SELECTION_SCOPE,
     });
+    if (lassoPoints) {
+      // TODO: this handleSelected is called without lassoPoints in some unkown cases
+      setLassoPoints(lassoPoints);
+    }
     if (selectedResults === null) {
       clearSelection();
     }
diff --git a/app/packages/embeddings/src/useViewChangeEffect.tsx b/app/packages/embeddings/src/useViewChangeEffect.tsx
index 19f91c010..0d677d096 100644
--- a/app/packages/embeddings/src/useViewChangeEffect.tsx
+++ b/app/packages/embeddings/src/useViewChangeEffect.tsx
@@ -3,7 +3,7 @@ import * as fos from "@fiftyone/state";
 import { useEffect } from "react";
 import { useRecoilValue, useSetRecoilState } from "recoil";
 import { fetchPlot } from "./fetch";
-import { useBrainResult } from "./useBrainResult";
+import { useBrainResult, usePointsField } from "./useBrainResult";
 import { useColorByField } from "./useLabelSelector";
 import { useWarnings } from "./useWarnings";
 
@@ -11,6 +11,7 @@ export function useViewChangeEffect() {
   const colorSeed = useRecoilValue(fos.colorSeed);
   const datasetName = useRecoilValue(fos.datasetName);
   const [brainKey, setBrainKey] = useBrainResult();
+  const [pointsField, setPointsField] = usePointsField();
   const [labelField] = useColorByField();
   const view = useRecoilValue(fos.view);
   const slices = useRecoilValue(fos.currentSlices(false));
@@ -72,6 +73,7 @@ export function useViewChangeEffect() {
 
         setLoadingPlotError(null);
         setLoadedPlot(res);
+        setPointsField(res.points_field);
       })
       .finally(() => setLoadingPlot(false));
   }, [datasetName, brainKey, labelField, view, colorSeed, slices, filters]);
diff --git a/docs/source/brain.rst b/docs/source/brain.rst
index 23adb9890..a22e2ca1b 100644
--- a/docs/source/brain.rst
+++ b/docs/source/brain.rst
@@ -337,7 +337,7 @@ visualization run by passing the `method` parameter to
 .. code:: python
     :linenos:
 
-    index = fob.compute_visualization(..., method="<method>", ...)
+    results = fob.compute_visualization(..., method="<method>", ...)
 
 Alternatively, you can change your default dimensionality reduction method for
 an entire session by setting the `FIFTYONE_BRAIN_DEFAULT_VISUALIZATION_METHOD`
@@ -383,7 +383,7 @@ time you call
 .. code:: python
     :linenos:
 
-    index = fob.compute_visualization(
+    results = fob.compute_visualization(
         ...
         method="umap",
         min_dist=0.2,
@@ -392,6 +392,81 @@ time you call
 Alternatively, you can more permanently configure your dimensionality reduction
 method(s) via your :ref:`brain config <brain-config>`.
 
+Optimizing lassoing performance
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can pass ``create_index=True`` to
+:meth:`compute_visualization() <fiftyone.brain.compute_visualization>` to store
+a spatial index of the computed points in a field of your dataset's samples.
+
+This is highly recommended for large datasets as it enables efficient querying
+when lassoing points in the :ref:`Embeddings panel <app-embeddings-panel>`.
+
+.. tabs::
+
+  .. group-tab:: Image embeddings
+
+    .. code-block:: python
+        :linenos:
+
+        import fiftyone as fo
+        import fiftyone.brain as fob
+        import fiftyone.zoo as foz
+
+        dataset = foz.load_zoo_dataset("quickstart")
+
+        # Generate a visualization with a spatial index
+        results = fob.compute_visualization(
+            dataset,
+            brain_key="img_viz",
+            create_index=True,
+        )
+
+  .. group-tab:: Object embeddings
+
+    .. code-block:: python
+        :linenos:
+
+        import fiftyone as fo
+        import fiftyone.brain as fob
+        import fiftyone.zoo as foz
+
+        dataset = foz.load_zoo_dataset("quickstart")
+
+        # Generate a patch visualization with a spatial index
+        results = fob.compute_visualization(
+            dataset,
+            patches_field="ground_truth",
+            brain_key="gt_viz",
+            create_index=True,
+        )
+
+.. note::
+
+    By default, spatial indexes are created in a field/attribute with name
+    ``brain_key``, but you can customize this by passing the ``points_field``
+    parameter to
+    :meth:`compute_visualization() <fiftyone.brain.compute_visualization>`.
+
+You can check whether an existing visualization result has a spatial index
+via
+:meth:`has_spatial_index <fiftyone.brain.visualization.VisualizationResults.has_spatial_index>`,
+and you can add or remove spatial indexes via
+:meth:`index_points() <fiftyone.brain.visualization.VisualizationResults.index_points>` and
+:meth:`remove_index() <fiftyone.brain.visualization.VisualizationResults.remove_index>`:
+
+.. code-block:: python
+    :linenos:
+
+    print(results.has_spatial_index)
+    # True/False
+
+    # Add a spatial index to existing visualization results
+    results.index_points()
+
+    # Remove the spatial index from existing visualization results
+    results.remove_index()
+
 .. _brain-similarity:
 
 Similarity
diff --git a/fiftyone/core/odm/mixins.py b/fiftyone/core/odm/mixins.py
index 80e6c3b79..a81e5863f 100644
--- a/fiftyone/core/odm/mixins.py
+++ b/fiftyone/core/odm/mixins.py
@@ -1130,9 +1130,6 @@ class DatasetMixin(object):
                     )
                 )
 
-        if isinstance(field, fof.ObjectIdField) and field_name.startswith("_"):
-            field_name = field_name[1:]
-
         if field_name in doc._fields:
             existing_field = doc._fields[field_name]
 
diff --git a/fiftyone/core/stages.py b/fiftyone/core/stages.py
index f6734e27c..4e1cbf814 100644
--- a/fiftyone/core/stages.py
+++ b/fiftyone/core/stages.py
@@ -3323,7 +3323,8 @@ class GeoWithin(_GeoStage):
         strict (True): whether a sample's location data must strictly fall
             within boundary (True) in order to match, or whether any
             intersection suffices (False)
-
+        create_index (True): whether to create the required spherical index,
+            if necessary
     """
 
     def __init__(
diff --git a/fiftyone/server/routes/embeddings.py b/fiftyone/server/routes/embeddings.py
index 8325bd30e..b9814a357 100644
--- a/fiftyone/server/routes/embeddings.py
+++ b/fiftyone/server/routes/embeddings.py
@@ -76,6 +76,7 @@ class OnPlotLoad(HTTPEndpoint):
 
         patches_field = results.config.patches_field
         is_patches_plot = patches_field is not None
+        points_field = results.config.points_field
 
         # Determines which points from `results` are in `view`, which are the
         # only points we want to display in the embeddings plot
@@ -151,6 +152,7 @@ class OnPlotLoad(HTTPEndpoint):
             "available_count": available_count,
             "missing_count": missing_count,
             "patches_field": patches_field,
+            "points_field": points_field,
         }
 
 
@@ -240,6 +242,8 @@ class EmbeddingsExtendedStage(HTTPEndpoint):
         patches_field = data["patchesField"]  # patches field of plot, or None
         selected_ids = data["selection"]  # selected IDs in plot
         slices = data["slices"]
+        lasso_points = data.get("lassoPoints", None)
+        points_field = data.get("pointsField", None)
 
         view = fosv.get_view(
             dataset_name,
@@ -250,7 +254,39 @@ class EmbeddingsExtendedStage(HTTPEndpoint):
         is_patches_view = view._is_patches
         is_patches_plot = patches_field is not None
 
-        if not is_patches_view and not is_patches_plot:
+        if (
+            lasso_points is not None
+            and points_field is not None
+            and (patches_field is None or is_patches_view)
+        ):
+            # Lasso points via spatial index
+            # Unfortunately we can't use $geoWithin to filter nested arrays.
+            # The user must have switched to a patches view for this
+            if patches_field is not None:
+                _, points_field = view._get_label_field_path(
+                    patches_field, points_field
+                )
+
+            stage = fos.Mongo(
+                [
+                    {
+                        "$match": {
+                            points_field: {
+                                "$geoWithin": {
+                                    "$polygon": [
+                                        [x, y]
+                                        for x, y in zip(
+                                            lasso_points["x"],
+                                            lasso_points["y"],
+                                        )
+                                    ]
+                                }
+                            }
+                        }
+                    }
+                ]
+            )
+        elif not is_patches_view and not is_patches_plot:
             # Samples plot and samples view
             stage = fos.Select(selected_ids)
         elif is_patches_view and is_patches_plot:


Test patch:
diff --git a/tests/unittests/import_export_tests.py b/tests/unittests/import_export_tests.py
index a4a06fd75..bbef2e534 100644
--- a/tests/unittests/import_export_tests.py
+++ b/tests/unittests/import_export_tests.py
@@ -3479,6 +3479,35 @@ class MultitaskImageDatasetTests(ImageDatasetTests):
             )
         )
 
+        # Test full schema merge
+
+        export_dir = self._new_dir()
+
+        sample1 = fo.Sample(filepath="video1.mp4")
+        sample1.frames[1] = fo.Frame()
+
+        dataset1 = fo.Dataset()
+        dataset1.add_sample(sample1)
+
+        sample2 = fo.Sample(filepath="video2.mp4")
+        sample2.frames[1] = fo.Frame()
+
+        dataset2 = fo.Dataset()
+        dataset2.add_sample(sample2)
+
+        dataset2.export(
+            export_dir=export_dir,
+            dataset_type=fo.types.FiftyOneDataset,
+            export_media=False,
+        )
+
+        dataset1.add_dir(
+            dataset_dir=export_dir,
+            dataset_type=fo.types.FiftyOneDataset,
+        )
+
+        self.assertEqual(len(dataset1), 2)
+
     @skipwindows
     @drop_datasets
     def test_legacy_fiftyone_dataset(self):
@@ -3843,6 +3872,35 @@ class MultitaskImageDatasetTests(ImageDatasetTests):
             )
         )
 
+        # Test full schema merge
+
+        export_dir = self._new_dir()
+
+        sample1 = fo.Sample(filepath="video1.mp4")
+        sample1.frames[1] = fo.Frame()
+
+        dataset1 = fo.Dataset()
+        dataset1.add_sample(sample1)
+
+        sample2 = fo.Sample(filepath="video2.mp4")
+        sample2.frames[1] = fo.Frame()
+
+        dataset2 = fo.Dataset()
+        dataset2.add_sample(sample2)
+
+        dataset2.export(
+            export_dir=export_dir,
+            dataset_type=fo.types.LegacyFiftyOneDataset,
+            export_media=False,
+        )
+
+        dataset1.add_dir(
+            dataset_dir=export_dir,
+            dataset_type=fo.types.LegacyFiftyOneDataset,
+        )
+
+        self.assertEqual(len(dataset1), 2)
+
 
 class OpenLABELImageDatasetTests(ImageDatasetTests):
     @drop_datasets