Repository: voxel51/fiftyone

Issue:
[FR] Make CVAT annotation run cleanup graceful
Currently, `CVATAnnotationResults.cleanup()` will raise an exception if not all of the tasks exist.

However, I think a better behavior would be to instead log a warning but continue on anyway, since the end result is the same: the run is now deleted.

This came up for me when I killed an expensive in-progress upload but then wanted to use the API to clean it up.

```py
import fiftyone as fo
import fiftyone.zoo as foz

dataset = foz.load_zoo_dataset("quickstart-video").limit(3).select_fields().clone()

dataset.annotate(
    "test",
    label_field="ground_truth",
    label_type="detections",
    classes=["vehicle"],
    launch_editor=True,
)

# Manually delete one of the tasks in CVAT...

# This will raise an error when trying to delete the task that was manually deleted
dataset.load_annotations("test", cleanup=True)
```


Discussion:

Resolving patch:
diff --git a/fiftyone/utils/annotations.py b/fiftyone/utils/annotations.py
--- a/fiftyone/utils/annotations.py
+++ b/fiftyone/utils/annotations.py
@@ -1012,8 +1012,8 @@ def load_annotations(
         found, in which case a dict containing the extra labels is returned
     """
     results = samples.load_annotation_results(anno_key, **kwargs)
-    label_schema = results.config.label_schema
     annotations = results.backend.download_annotations(results)
+    label_schema = results.config.label_schema
 
     unexpected_annos = defaultdict(dict)
 
diff --git a/fiftyone/utils/cvat.py b/fiftyone/utils/cvat.py
--- a/fiftyone/utils/cvat.py
+++ b/fiftyone/utils/cvat.py
@@ -2762,14 +2762,23 @@ def get_status(self):
         return self._get_status()
 
     def print_status(self):
-        """Print the status of the assigned tasks and jobs."""
+        """Prints the status of the assigned tasks and jobs."""
         self._get_status(log=True)
 
-    def cleanup(self):
-        """Deletes all tasks associated with this annotation run and any created
-        projects from the CVAT server.
+    def delete_tasks(self, task_ids):
+        """Deletes the given tasks from both the CVAT server and this run.
+
+        Args:
+            task_ids: an iterable of task IDs
         """
         api = self.connect_to_api()
+        api.delete_tasks(task_ids)
+
+        self._forget_tasks(task_ids)
+
+    def cleanup(self):
+        """Deletes all tasks and created projects associated with this run."""
+        api = self.connect_to_api()
 
         if self.task_ids:
             logger.info("Deleting tasks...")
@@ -2781,10 +2790,23 @@ def cleanup(self):
                 logger.info("Deleting projects...")
                 api.delete_projects(self.project_ids)
 
-        # @todo save updated results to DB?
         self.project_ids = []
         self.task_ids = []
         self.job_ids = {}
+        self.id_map = {}
+        self.frame_id_map = {}
+
+    def _forget_tasks(self, task_ids):
+        for task_id in task_ids:
+            self.job_ids.pop(task_id, None)
+            _frame_id_map = self.frame_id_map.pop(task_id, {})
+            sample_ids = set(fd["sample_id"] for fd in _frame_id_map.values())
+            for _id_map in self.id_map.values():
+                for sample_id in sample_ids:
+                    _id_map.pop(sample_id, None)
+
+        task_ids = set(task_ids)
+        self.task_ids = [_id for _id in self.task_ids if _id not in task_ids]
 
     def _get_status(self, log=False):
         api = self.connect_to_api()
@@ -2988,6 +3010,9 @@ def base_task_url(self, task_id):
     def base_job_url(self, task_id, job_id):
         return "%s/tasks/%d/jobs/%d" % (self.base_url, task_id, job_id)
 
+    def task_id_search_url(self, task_id):
+        return "%s/tasks?id=%d" % (self.base_api_url, task_id)
+
     def user_search_url(self, username):
         return "%s/users?search=%s" % (self.base_api_url, username)
 
@@ -3097,35 +3122,6 @@ def delete(self, url, **kwargs):
         self._validate(response, kwargs)
         return response
 
-    def _get_value_from_search(
-        self, search_url_fcn, target, target_key, value_key
-    ):
-        search_url = search_url_fcn(target)
-        resp = self.get(search_url).json()
-        for info in resp["results"]:
-            if info[target_key] == target:
-                return info[value_key]
-
-        return None
-
-    def _get_value_update_map(
-        self, name, id_map, result_name, search_url_fcn, name_type
-    ):
-        if name is None:
-            return None
-
-        if name in id_map:
-            return id_map[name]
-
-        _id = self._get_value_from_search(
-            search_url_fcn, name, result_name, "id"
-        )
-
-        if _id is not None:
-            id_map[name] = _id
-
-        return _id
-
     def get_user_id(self, username):
         """Retrieves the CVAT user ID for the given username.
 
@@ -3226,6 +3222,49 @@ def create_project(self, name, schema=None):
         project_resp = self.post(self.projects_url, json=project_json).json()
         return project_resp["id"]
 
+    def list_projects(self):
+        """Returns the list of project IDs.
+
+        Returns:
+            the list of project IDs
+        """
+        return self._get_paginated_results(self.projects_url, value="id")
+
+    def project_exists(self, project_id):
+        """Checks if the given project exists.
+
+        Args:
+            project_id: the project ID
+
+        Returns:
+            True/False
+        """
+        return (
+            self._get_value_from_search(
+                self.project_id_search_url, project_id, "id", "id",
+            )
+            is not None
+        )
+
+    def delete_project(self, project_id):
+        """Deletes the given project from the CVAT server.
+
+        Args:
+            project_id: the project ID
+        """
+        if self.project_exists(project_id):
+            self.delete(self.project_url(project_id))
+
+    def delete_projects(self, project_ids):
+        """Deletes the given projects from the CVAT server.
+
+        Args:
+            project_ids: an iterable of project IDs
+        """
+        with fou.ProgressBar() as pb:
+            for project_id in pb(list(project_ids)):
+                self.delete_project(project_id)
+
     def create_task(
         self,
         name,
@@ -3299,23 +3338,29 @@ def create_task(
 
         return task_id, class_id_map, attr_id_map
 
-    def delete_project(self, project_id):
-        """Deletes the given project from the CVAT server.
+    def list_tasks(self):
+        """Returns the list of task IDs.
 
-        Args:
-            project_id: the project ID
+        Returns:
+            the list of task IDs
         """
-        self.delete(self.project_url(project_id))
+        return self._get_paginated_results(self.tasks_url, value="id")
 
-    def delete_projects(self, project_ids):
-        """Deletes the given projects from the CVAT server.
+    def task_exists(self, task_id):
+        """Checks if the given task exists.
 
         Args:
-            project_ids: an iterable of project IDs
+            task_id: the task ID
+
+        Returns:
+            True/False
         """
-        with fou.ProgressBar() as pb:
-            for project_id in pb(list(project_ids)):
-                self.delete_project(project_id)
+        return (
+            self._get_value_from_search(
+                self.task_id_search_url, task_id, "id", "id",
+            )
+            is not None
+        )
 
     def delete_task(self, task_id):
         """Deletes the given task from the CVAT server.
@@ -3323,7 +3368,8 @@ def delete_task(self, task_id):
         Args:
             task_id: the task ID
         """
-        self.delete(self.task_url(task_id))
+        if self.task_exists(task_id):
+            self.delete(self.task_url(task_id))
 
     def delete_tasks(self, task_ids):
         """Deletes the given tasks from the CVAT server.
@@ -3606,8 +3652,18 @@ def download_annotations(self, results):
                 labels_task_map_rev[task].append(lf)
 
         annotations = {}
+        deleted_tasks = []
+
+        existing_tasks = set(self.list_tasks())
 
         for task_id in task_ids:
+            if task_id not in existing_tasks:
+                deleted_tasks.append(task_id)
+                logger.warning(
+                    "Skipping task %d, which no longer exists", task_id
+                )
+                continue
+
             # Download task data
             task_json = self.get(self.task_url(task_id)).json()
             attr_id_map = {}
@@ -3745,10 +3801,57 @@ def download_annotations(self, results):
                     annotations, {label_field: label_field_results}
                 )
 
+        if deleted_tasks:
+            results._forget_tasks(deleted_tasks)
+
         return annotations
 
+    def _get_paginated_results(self, url, value=None):
+        results = []
+        page = url
+        while page is not None:
+            response = self.get(page).json()
+            for result in response["results"]:
+                if value is not None:
+                    results.append(result[value])
+                else:
+                    results.append(result)
+
+            page = response["next"]
+
+        return results
+
+    def _get_value_from_search(
+        self, search_url_fcn, target, target_key, value_key
+    ):
+        search_url = search_url_fcn(target)
+        resp = self.get(search_url).json()
+        for info in resp["results"]:
+            if info[target_key] == target:
+                return info[value_key]
+
+        return None
+
+    def _get_value_update_map(
+        self, name, id_map, result_name, search_url_fcn, name_type
+    ):
+        if name is None:
+            return None
+
+        if name in id_map:
+            return id_map[name]
+
+        _id = self._get_value_from_search(
+            search_url_fcn, name, result_name, "id"
+        )
+
+        if _id is not None:
+            id_map[name] = _id
+
+        return _id
+
     def _get_project_labels(self, project_id):
-        if self.get_project_name(project_id) is None:
+        if not self.project_exists(project_id):
             raise ValueError("Project '%s' not found" % project_id)
 
         return self.get(self.project_url(project_id)).json()["labels"]


Test patch: