Repository: voxel51/fiftyone

Issue:
[FR] Allow Label Studio local files import
### Proposal Summary

Currently when uploading image from voxel51 -> Label studio for annotation we open and pass all images as bytes. This
* is slow when trying to import large numbers of files (10k+) 
* consumes a lot of RAM
* leads to data being duplicated in LS cache folder

Instead we can leverage LS local storage, it's not enabled by default, but it's possible to read env var and detect if it's enabled, in such case pass images as filepath, not bytes. This wouldn't change anything for the end users.

In my use case this improved speed of sending large batches of images for annotations by 20x

### What areas of FiftyOne does this feature affect?

-   [ ] App: FiftyOne application
-   [x] Core: Core `fiftyone` Python library
-   [ ] Server: FiftyOne server

### Details

Draft implementation looks as follows:
```python
def _upload_tasks(self, project, tasks, predictions=None):
    """Uploads files to Label Studio and registers them as tasks.

    Args:
        project: a ``label_studio_sdk.Project``
        tasks: a list of task dicts
        predictions (None): optional predictions to upload

    Returns:
        a dict mapping ``task_id`` to ``sample_id``
    """
    ls_root = os.getenv("LABEL_STUDIO_LOCAL_FILES_DOCUMENT_ROOT", None)
    ls_local_enabled = os.getenv("LABEL_STUDIO_LOCAL_FILES_SERVING_ENABLED", None)
    local_storage_enabled = not (ls_root is None or ls_local_enabled is None)
    # if files are not inside local storage root, we can't use local storage
    common_prefix = os.path.commonprefix([one[one["media_type"]] for one in tasks]).rstrip("/")
    if local_storage_enabled and ls_root in common_prefix:
        project.connect_local_import_storage(common_prefix)
        def _get_file_path(file_path):
            return "/data/local-files?d=" + os.path.relpath(file_path, ls_root)
        files_data = [{one["media_type"]: _get_file_path(one[one["media_type"]])} for one in tasks]
        
        # Make the request to import tasks using URLs
        upload_resp = self._client.make_request(
            "POST",
            f"/api/projects/{project.id}/import",
            json=files_data,
        )

    else:
        files = [
            (one["source_id"], open(one[one["media_type"]], "rb")) for one in tasks
        ]
        # upload files first and get their upload ids
        upload_resp = self._client.make_request(
            "POST",
            f"/api/projects/{project.id}/import",
            params={"commit_to_project": True},
            files=files,
        )

        # create tasks out of the uploaded files
        payload = json.dumps(
            {
                "file_upload_ids": upload_resp.json()["file_upload_ids"],
                "files_as_tasks_list": False,
            }
        )
        self._client.headers.update({"Content-Type": "application/json"})
        self._client.make_request(
            "POST", f"/api/projects/{project.id}/reimport", data=payload
        )



    # get uploaded task ids
    uploaded_ids = project.get_tasks(only_ids=True)[-len(tasks) :]
    uploaded_tasks = {
        i: t["source_id"] for i, t in zip(uploaded_ids, tasks)
    }

    # upload predictions if given
    if predictions:
        source2task = {v: k for k, v in uploaded_tasks.items()}
        for _, label_predictions in predictions.items():
            ls_predictions = [
                {
                    "task": source2task[smp_id],
                    "result": pred,
                }
                for smp_id, pred in label_predictions.items()
            ]
            project.create_predictions(ls_predictions)

    return uploaded_tasks
```

### Willingness to contribute

The FiftyOne Community welcomes contributions! Would you or another member of your organization be willing to contribute an implementation of this feature?

-   [x] Yes. I can contribute this feature independently
-   [ ] Yes. I would be willing to contribute this feature with guidance from the FiftyOne community
-   [ ] No. I cannot contribute this feature at this time



Discussion:
Hi @bonlime 👋 this looks like a great upgrade. A contribution would be most welcome! 🙇 
@ehofesmann can you support/review @bonlime's PR here?

Resolving patch:
diff --git a/fiftyone/utils/labelstudio.py b/fiftyone/utils/labelstudio.py
--- a/fiftyone/utils/labelstudio.py
+++ b/fiftyone/utils/labelstudio.py
@@ -17,6 +17,7 @@
 import string
 import webbrowser
 
+import os
 from bson import ObjectId
 import numpy as np
 
@@ -288,36 +289,69 @@ def _upload_tasks(self, project, tasks, predictions=None):
         Returns:
             a dict mapping ``task_id`` to ``sample_id``
         """
-        files = [
-            (
-                one["source_id"],
-                (open(one[one["media_type"]], "rb")),
+        ls_root = os.getenv("LABEL_STUDIO_LOCAL_FILES_DOCUMENT_ROOT", None)
+        ls_local_enabled = os.getenv(
+            "LABEL_STUDIO_LOCAL_FILES_SERVING_ENABLED", None
+        )
+        local_storage_enabled = (
+            ls_root is not None
+            and ls_local_enabled is not None
+            and ls_local_enabled.lower() == "true"
+        )
+        # if files are not inside local storage root, we can't use local storage
+        common_prefix = os.path.commonprefix(
+            [one[one["media_type"]] for one in tasks]
+        ).rstrip("/")
+        common_prefix = os.path.dirname(common_prefix)
+        if local_storage_enabled and ls_root in common_prefix:
+            logger.debug("Using local LabelStudio storage")
+            project.connect_local_import_storage(common_prefix)
+
+            def _get_file_path(file_path):
+                return "/data/local-files?d=" + os.path.relpath(
+                    file_path, ls_root
+                )
+
+            files_data = [
+                {one["media_type"]: _get_file_path(one[one["media_type"]])}
+                for one in tasks
+            ]
+
+            # Make the request to import tasks using URLs
+            upload_resp = self._client.make_request(
+                "POST",
+                f"/api/projects/{project.id}/import",
+                json=files_data,
             )
-            for one in tasks
-        ]
 
-        # upload files first and get their upload ids
-        upload_resp = self._client.make_request(
-            "POST",
-            f"/api/projects/{project.id}/import",
-            params={"commit_to_project": True},
-            files=files,
-        )
+        else:
+            logger.debug("Uploading as files")
+            files = [
+                (one["source_id"], open(one[one["media_type"]], "rb"))
+                for one in tasks
+            ]
+            # upload files first and get their upload ids
+            upload_resp = self._client.make_request(
+                "POST",
+                f"/api/projects/{project.id}/import",
+                params={"commit_to_project": True},
+                files=files,
+            )
 
-        # create tasks out of the uploaded files
-        payload = json.dumps(
-            {
-                "file_upload_ids": upload_resp.json()["file_upload_ids"],
-                "files_as_tasks_list": False,
-            }
-        )
-        self._client.headers.update({"Content-Type": "application/json"})
-        self._client.make_request(
-            "POST", f"/api/projects/{project.id}/reimport", data=payload
-        )
+            # create tasks out of the uploaded files
+            payload = json.dumps(
+                {
+                    "file_upload_ids": upload_resp.json()["file_upload_ids"],
+                    "files_as_tasks_list": False,
+                }
+            )
+            self._client.headers.update({"Content-Type": "application/json"})
+            self._client.make_request(
+                "POST", f"/api/projects/{project.id}/reimport", data=payload
+            )
 
         # get uploaded task ids
-        uploaded_ids = project.get_tasks(only_ids=True)[-len(files) :]
+        uploaded_ids = project.get_tasks(only_ids=True)[-len(tasks) :]
         uploaded_tasks = {
             i: t["source_id"] for i, t in zip(uploaded_ids, tasks)
         }


Test patch: