Repository: openvinotoolkit/datumaro

Issue:
Format detection fails to disambiguate labelme and voc on tfds voc2012
When I try to detect format of a tfds/voc2012, Datumaro detects both LabelMe and VOC. Probably, the problem is in XMLs with only image info inside, like this:

<details>

```
<annotation>
  <folder>2008</folder>
  <filename>2008_000001.jpg</filename>
  <source>
    <database>Unknown</database>
    <annotation>Unknown</annotation>
    <image>Unknown</image>
  </source>
  <segmented>0</segmented>
</annotation>
```

</details>


Discussion:

Resolving patch:
diff --git a/datumaro/components/format_detection.py b/datumaro/components/format_detection.py
--- a/datumaro/components/format_detection.py
+++ b/datumaro/components/format_detection.py
@@ -169,6 +169,25 @@ def require_file(self, pattern: str, *,
 
         self._start_requirement("require_file")
 
+        return next(self._require_files_iter(
+            pattern, exclude_fnames=exclude_fnames))
+
+    def require_files(self, pattern: str, *,
+        exclude_fnames: Union[str, Collection[str]] = (),
+    ) -> List[str]:
+        """
+        Same as `require_file`, but returns all matching paths in alphabetical
+        order.
+        """
+
+        self._start_requirement("require_files")
+
+        return sorted(self._require_files_iter(
+            pattern, exclude_fnames=exclude_fnames))
+
+    def _require_files_iter(self, pattern: str, *,
+        exclude_fnames: Union[str, Collection[str]] = (),
+    ) -> Iterator[str]:
         if isinstance(exclude_fnames, str):
             exclude_fnames = (exclude_fnames,)
 
@@ -183,6 +202,7 @@ def require_file(self, pattern: str, *,
             self.fail(requirement_desc)
 
         pattern_abs = osp.join(glob.escape(self._root_path), pattern)
+        none_found = True
         for path in glob.iglob(pattern_abs, recursive=True):
             if osp.isfile(path):
                 # Ideally, we should provide a way to filter out whole paths,
@@ -194,9 +214,11 @@ def require_file(self, pattern: str, *,
                         for pat in exclude_fnames):
                     continue
 
-                return osp.relpath(path, self._root_path)
+                yield osp.relpath(path, self._root_path)
+                none_found = False
 
-        self.fail(requirement_desc)
+        if none_found:
+            self.fail(requirement_desc)
 
     @contextlib.contextmanager
     def probe_text_file(
diff --git a/datumaro/plugins/labelme_format.py b/datumaro/plugins/labelme_format.py
--- a/datumaro/plugins/labelme_format.py
+++ b/datumaro/plugins/labelme_format.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 Intel Corporation
+# Copyright (C) 2020-2022 Intel Corporation
 #
 # SPDX-License-Identifier: MIT
 
@@ -284,31 +284,39 @@ def __iter__(self):
 class LabelMeImporter(Importer):
     @classmethod
     def detect(cls, context: FormatDetectionContext) -> None:
-        annot_file = context.require_file('**/*.xml')
-
-        with context.probe_text_file(
-            annot_file, "must be a LabelMe annotation file",
-        ) as f:
-            elem_parents = []
-
-            for event, elem in ElementTree.iterparse(f, events=('start', 'end')):
-                if event == 'start':
-                    if elem_parents == [] and elem.tag != 'annotation':
-                        raise Exception
-
-                    if elem_parents == ['annotation', 'object'] \
-                            and elem.tag in {'polygon', 'segm'}:
-                        return
-
-                    elem_parents.append(elem.tag)
-                elif event == 'end':
-                    elem_parents.pop()
-
-                    if elem_parents == ['annotation'] and elem.tag == 'object':
-                        # If we got here, then we found an object with no
-                        # polygon and no mask, so it's probably the wrong
-                        # format.
-                        raise Exception
+        annot_paths = context.require_files('**/*.xml')
+
+        for annot_path in annot_paths:
+            with context.probe_text_file(
+                annot_path, "must be a LabelMe annotation file",
+            ) as f:
+                elem_parents = []
+
+                for event, elem in ElementTree.iterparse(f, events=('start', 'end')):
+                    if event == 'start':
+                        if elem_parents == [] and elem.tag != 'annotation':
+                            raise Exception
+
+                        if elem_parents == ['annotation', 'object'] \
+                                and elem.tag in {'polygon', 'segm'}:
+                            return
+
+                        elem_parents.append(elem.tag)
+                    elif event == 'end':
+                        elem_parents.pop()
+
+                        if elem_parents == ['annotation'] and elem.tag == 'object':
+                            # If we got here, then we found an object with no
+                            # polygon and no mask, so it's probably the wrong
+                            # format.
+                            raise Exception
+
+            # If we got here, then the current file has no objects and is thus
+            # ambiguous - it could be ours or it could be from the VOC format.
+            # We'll proceed to test the next one.
+
+        # If we got here, then every file was ambiguous. We'll have to
+        # (implicitly) return a match.
 
     @classmethod
     def find_sources(cls, path):


Test patch:
diff --git a/tests/assets/voc_dataset/voc_dataset2/Annotations/@.xml b/tests/assets/voc_dataset/voc_dataset2/Annotations/@.xml
new file mode 100644
--- /dev/null
+++ b/tests/assets/voc_dataset/voc_dataset2/Annotations/@.xml
@@ -0,0 +1,19 @@
+<annotation>
+  <!--
+    The purpose of this file is to test format detection in the case where
+    at least one annotation file could be misdetected as being in the LabelMe
+    format (due to containing no objects). See
+    <https://github.com/openvinotoolkit/datumaro/issues/658>.
+
+    The name of the file is "@.xml" so that it's sorted before all other XML
+    files and thus the LabelMe detector finds it first.
+  -->
+  <folder></folder>
+  <filename>@.jpg</filename>
+  <source>
+    <database>Unknown</database>
+    <annotation>Unknown</annotation>
+    <image>Unknown</image>
+  </source>
+  <segmented>0</segmented>
+</annotation>
diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt
--- a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt
+++ b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Action/trainval.txt
@@ -1,3 +1,4 @@
+@
 a
 b
 c
diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt
--- a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt
+++ b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Layout/trainval.txt
@@ -1,3 +1,4 @@
+@
 a
 b
 c
diff --git a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt
--- a/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt
+++ b/tests/assets/voc_dataset/voc_dataset2/ImageSets/Main/trainval.txt
@@ -1,3 +1,4 @@
+@
 a
 b
 c
diff --git a/tests/test_format_detection.py b/tests/test_format_detection.py
--- a/tests/test_format_detection.py
+++ b/tests/test_format_detection.py
@@ -18,7 +18,7 @@ def setUp(self) -> None:
 
 class ApplyFormatDetectorTest(FormatDetectionTest):
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
-    def test_empty_detector(self):
+    def test_default_result(self):
         result = apply_format_detector(self._dataset_root, lambda c: None)
         self.assertEqual(result, FormatDetectionConfidence.MEDIUM)
 
@@ -58,9 +58,8 @@ def detect(context):
             nonlocal selected_file
             selected_file = context.require_file('**/[fg]oo*.t?t')
 
-        result = apply_format_detector(self._dataset_root, detect)
+        apply_format_detector(self._dataset_root, detect)
 
-        self.assertEqual(result, FormatDetectionConfidence.MEDIUM)
         self.assertEqual(selected_file, 'foobar.txt')
 
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
@@ -109,6 +108,33 @@ def detect(context):
         self.assertIn('*.txt', result.exception.failed_alternatives[0])
         self.assertIn('*.lst', result.exception.failed_alternatives[0])
 
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
+    def test_require_files_success(self):
+        for stem in 'cba':
+            with open(osp.join(self._dataset_root, f'{stem}.txt'), 'w'):
+                pass
+
+        selected_files = []
+
+        def detect(context):
+            selected_files.extend(
+                context.require_files('*.txt', exclude_fnames='b.txt'))
+
+        apply_format_detector(self._dataset_root, detect)
+
+        self.assertEqual(selected_files, ['a.txt', 'c.txt'])
+
+    @mark_requirement(Requirements.DATUM_GENERAL_REQ)
+    def test_require_files_failure(self):
+        def detect(context):
+            context.require_files('*.txt')
+
+        with self.assertRaises(FormatRequirementsUnmet) as result:
+            apply_format_detector(self._dataset_root, detect)
+
+        self.assertEqual(len(result.exception.failed_alternatives), 1)
+        self.assertIn('*.txt', result.exception.failed_alternatives[0])
+
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_probe_text_file_success(self):
         with open(osp.join(self._dataset_root, 'foobar.txt'), 'w') as f:
@@ -119,9 +145,7 @@ def detect(context):
                 if next(f) != '123\n':
                     raise Exception
 
-        result = apply_format_detector(self._dataset_root, detect)
-
-        self.assertEqual(result, FormatDetectionConfidence.MEDIUM)
+        apply_format_detector(self._dataset_root, detect)
 
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)
     def test_probe_text_file_failure_bad_file(self):
@@ -182,9 +206,8 @@ def detect(context):
                     alternatives_executed.add(3)
                     context.fail('bad alternative 3')
 
-        result = apply_format_detector(self._dataset_root, detect)
+        apply_format_detector(self._dataset_root, detect)
 
-        self.assertEqual(result, FormatDetectionConfidence.MEDIUM)
         self.assertEqual(alternatives_executed, {1, 2, 3})
 
     @mark_requirement(Requirements.DATUM_GENERAL_REQ)