Repository: openvinotoolkit/datumaro

Issue:
Dataset patches do not remove empty (e.g. renamed) subsets
Steps to reproduce:
1. Create a project
2. Import a dataset
3. Rename a subset (e.g. `datum transform -t random_split`)

Depending on the format, the exported dataset will contain annotations from renamed-from and renamed-to subsets. This leads to duplication of annotations in different subsets, which are then found and merged together on importing.


Discussion:

Resolving patch:
diff --git a/datumaro/components/converter.py b/datumaro/components/converter.py
--- a/datumaro/components/converter.py
+++ b/datumaro/components/converter.py
@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: MIT
 
+from tempfile import mkdtemp
 from typing import Union
 import logging as log
 import os
@@ -11,6 +12,7 @@
 from datumaro.components.cli_plugin import CliPlugin
 from datumaro.components.dataset import DatasetPatch
 from datumaro.components.extractor import DatasetItem
+from datumaro.util import error_rollback, on_error_do
 from datumaro.util.image import Image
 
 
@@ -34,8 +36,34 @@ def convert(cls, extractor, save_dir, **options):
         return converter.apply()
 
     @classmethod
+    @error_rollback
     def patch(cls, dataset, patch, save_dir, **options):
-        return cls.convert(dataset, save_dir, **options)
+        # This solution is not any better in performance than just
+        # writing a dataset, but in case of patching (i.e. writing
+        # to the previous location), it allows to avoid many problems
+        # with removing and replacing existing files. Surely, this
+        # approach also has problems with removal of the given directory.
+        # Problems can occur if we can't remove the directory,
+        # or want to reuse the given directory. It can happen if it
+        # is mounted or (sym-)linked.
+        # Probably, a better solution could be to wipe directory
+        # contents and write new data there. Note that directly doing this
+        # also doesn't work, because images may be needed for writing.
+
+        if not osp.isdir(save_dir):
+            return cls.convert(dataset, save_dir, **options)
+
+        tmpdir = mkdtemp(dir=osp.dirname(save_dir),
+            prefix=osp.basename(save_dir), suffix='.tmp')
+        on_error_do(shutil.rmtree, tmpdir, ignore_errors=True)
+        shutil.copymode(save_dir, tmpdir)
+
+        retval = cls.convert(dataset, tmpdir, **options)
+
+        shutil.rmtree(save_dir)
+        os.replace(tmpdir, save_dir)
+
+        return retval
 
     def apply(self):
         raise NotImplementedError("Should be implemented in a subclass")


Test patch:
diff --git a/tests/test_dataset.py b/tests/test_dataset.py
--- a/tests/test_dataset.py
+++ b/tests/test_dataset.py
@@ -4,6 +4,7 @@
 
 import numpy as np
 
+from datumaro.components.converter import Converter
 from datumaro.components.dataset import (
     DEFAULT_FORMAT, Dataset, ItemStatus, eager_mode,
 )
@@ -1285,6 +1286,46 @@ def test_can_filter_annotations_in_merged_dataset(self):
 
         self.assertEqual(1, len(dataset))
 
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
+    def test_inplace_save_writes_only_updated_data(self):
+        class CustomConverter(Converter):
+            DEFAULT_IMAGE_EXT = '.jpg'
+
+            def apply(self):
+                assert osp.isdir(self._save_dir)
+
+                for item in self._extractor:
+                    name = f'{item.subset}_{item.id}'
+                    with open(osp.join(
+                            self._save_dir, name + '.txt'), 'w') as f:
+                        f.write('\n')
+
+                    if self._save_images and \
+                            item.has_image and item.image.has_data:
+                        self._save_image(item, name=name)
+
+        env = Environment()
+        env.converters.items = { 'test': CustomConverter }
+
+        with TestDir() as path:
+            dataset = Dataset.from_iterable([
+                DatasetItem(1, subset='train', image=np.ones((2, 4, 3))),
+                DatasetItem(2, subset='train',
+                    image=Image(path='2.jpg', size=(3, 2))),
+                DatasetItem(3, subset='valid', image=np.ones((2, 2, 3))),
+            ], categories=[], env=env)
+            dataset.export(path, 'test', save_images=True)
+
+            dataset.put(DatasetItem(2, subset='train', image=np.ones((3, 2, 3))))
+            dataset.remove(3, 'valid')
+            dataset.save(save_images=True)
+
+            self.assertEqual({
+                    'train_1.txt', 'train_1.jpg',
+                    'train_2.txt', 'train_2.jpg'
+                },
+                set(os.listdir(path)))
+
 
 class DatasetItemTest(TestCase):
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
diff --git a/tests/test_widerface_format.py b/tests/test_widerface_format.py
--- a/tests/test_widerface_format.py
+++ b/tests/test_widerface_format.py
@@ -1,4 +1,5 @@
 from unittest import TestCase
+import os
 import os.path as osp
 
 import numpy as np
@@ -9,7 +10,7 @@
     WiderFaceConverter, WiderFaceImporter,
 )
 from datumaro.util.image import Image
-from datumaro.util.test_utils import TestDir, compare_datasets
+from datumaro.util.test_utils import IGNORE_ALL, TestDir, compare_datasets
 
 from .requirements import Requirements, mark_requirement
 
@@ -152,6 +153,35 @@ def test_can_save_and_load_image_with_arbitrary_extension(self):
 
             compare_datasets(self, dataset, parsed_dataset, require_images=True)
 
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
+    def test_inplace_save_writes_only_updated_data(self):
+        expected = Dataset.from_iterable([
+            DatasetItem(1, subset='train', image=np.ones((2, 4, 3))),
+            DatasetItem(2, subset='train', image=np.ones((3, 2, 3))),
+        ], categories=[])
+
+        with TestDir() as path:
+            dataset = Dataset.from_iterable([
+                DatasetItem(1, subset='train', image=np.ones((2, 4, 3))),
+                DatasetItem(2, subset='train',
+                    image=Image(path='2.jpg', size=(3, 2))),
+                DatasetItem(3, subset='valid', image=np.ones((2, 2, 3))),
+            ], categories=[])
+            dataset.export(path, 'wider_face', save_images=True)
+
+            dataset.put(DatasetItem(2, subset='train', image=np.ones((3, 2, 3))))
+            dataset.remove(3, 'valid')
+            dataset.save(save_images=True)
+
+            self.assertEqual({'1.jpg', '2.jpg'},
+                set(os.listdir(osp.join(path,
+                    'WIDER_train', 'images', 'no_label'))))
+            self.assertEqual({'wider_face_train_bbx_gt.txt'},
+                set(os.listdir(osp.join(path, 'wider_face_split'))))
+            compare_datasets(self, expected,
+                Dataset.import_from(path, 'wider_face'),
+                require_images=True, ignored_attrs=IGNORE_ALL)
+
 DUMMY_DATASET_DIR = osp.join(osp.dirname(__file__), 'assets', 'widerface_dataset')
 
 class WiderFaceImporterTest(TestCase):