Repository: voxel51/fiftyone
Issue: [BUG] 'KeyError' when loading dataset with relative paths on Windows

When attempting to load a COCO format dataset from disk in a Windows environment, an uncaught exception occurs if the labels json uses relative paths for the image. Output:
`Uncaught exception
Traceback (most recent call last):
  File "C:\\Users\\Trace\\Documents\\Work\\Ping\\load_coco_formatted.py", line 3, in <module>
    dataset = fo.Dataset.from_dir(
  File "C:\\Python39\\lib\\site-packages\\fiftyone\\core\\dataset.py", line 3664, in from_dir
    dataset.add_dir(
  File "C:\\Python39\\lib\\site-packages\\fiftyone\\core\\dataset.py", line 2454, in add_dir
    return self.add_importer(
  File "C:\\Python39\\lib\\site-packages\\fiftyone\\core\\dataset.py", line 2997, in add_importer
    return foud.import_samples(
  File "C:\\Python39\\lib\\site-packages\\fiftyone\\utils\\data\\importers.py", line 129, in import_samples
    sample_ids = dataset.add_samples(
  File "C:\\Python39\\lib\\site-packages\\fiftyone\\core\\dataset.py", line 1378, in add_samples
    for batch in batcher:
  File "C:\\Python39\\lib\\site-packages\\fiftyone\\core\\utils.py", line 900, in __next__
    batch.append(next(self._iter))
  File "C:\\Python39\\lib\\site-packages\\fiftyone\\utils\\coco.py", line 415, in __next__
    image_path = self._image_paths_map[filename]
KeyError: 'midv2019/images/DG/DG01_01.tif'`

This appears to be caused by the image_paths_map having normalized key values. Normalizing the filename in the `__next__` def of the COCODetectionDatasetImporter seems to resolve the issue.
https://github.com/voxel51/fiftyone/blob/1986f1162fb91c4f57d8413dca44fee9644378b6/fiftyone/utils/coco.py#L410
becomes `filename = os.path.normpath(next(self._iter_filenames))`

Changed files:
fiftyone/utils/geojson.py, fiftyone/utils/kitti.py, fiftyone/utils/coco.py, fiftyone/utils/bdd.py, fiftyone/utils/yolo.py, fiftyone/core/utils.py, fiftyone/utils/data/importers.py, fiftyone/utils/cvat.py, fiftyone/utils/openlabel.py, fiftyone/utils/voc.py

diff --git a/fiftyone/core/utils.py b/fiftyone/core/utils.py
index c3b8c1508..6406507f5 100644
--- a/fiftyone/core/utils.py
+++ b/fiftyone/core/utils.py
@@ -17,7 +17,9 @@ import inspect
 import io
 import itertools
 import logging
+import ntpath
 import os
+import posixpath
 import platform
 import signal
 import struct
@@ -291,6 +293,25 @@ def fill_patterns(string):
     return etau.fill_patterns(string, available_patterns())
 
 
+def normpath(path):
+    """Normalizes the given path by converting all slashes to forward slashes
+    on Unix and backslashes on Windows and removing duplicate slashes.
+
+    Use this function when you need a version of ``os.path.normpath`` that
+    converts ``\\\\`` to ``/`` on Unix.
+
+    Args:
+        path: a path
+
+    Returns:
+        the normalized path
+    """
+    if os.name == "nt":
+        return ntpath.normpath(path)
+
+    return posixpath.normpath(path.replace("\\\\", "/"))
+
+
 def normalize_path(path):
     """Normalizes the given path by converting it to an absolute path and
     expanding the user directory, if necessary.
diff --git a/fiftyone/utils/bdd.py b/fiftyone/utils/bdd.py
index 421b1efc0..7880caab8 100644
--- a/fiftyone/utils/bdd.py
+++ b/fiftyone/utils/bdd.py
@@ -17,6 +17,7 @@ import eta.core.utils as etau
 import fiftyone as fo
 import fiftyone.core.labels as fol
 import fiftyone.core.metadata as fom
+import fiftyone.core.utils as fou
 import fiftyone.utils.data as foud
 
 
@@ -171,6 +172,9 @@ class BDDDatasetImporter(
 
         if self.labels_path is not None and os.path.isfile(self.labels_path):
             anno_dict_map = load_bdd_annotations(self.labels_path)
+            anno_dict_map = {
+                fou.normpath(k): v for k, v in anno_dict_map.items()
+            }
         else:
             anno_dict_map = {}
 
diff --git a/fiftyone/utils/coco.py b/fiftyone/utils/coco.py
index 168cd25dc..b23416897 100644
--- a/fiftyone/utils/coco.py
+++ b/fiftyone/utils/coco.py
@@ -545,11 +545,13 @@ class COCODetectionDatasetImporter(
                 max_samples=self.max_samples,
             )
 
-            filenames = [images[_id]["file_name"] for _id in image_ids]
+            filenames = [
+                fou.normpath(images[_id]["file_name"]) for _id in image_ids
+            ]
 
             _image_ids = set(image_ids)
             image_dicts_map = {
-                i["file_name"]: i
+                fou.normpath(i["file_name"]): i
                 for _id, i in images.items()
                 if _id in _image_ids
             }
diff --git a/fiftyone/utils/cvat.py b/fiftyone/utils/cvat.py
index 11e1a5973..89503904e 100644
--- a/fiftyone/utils/cvat.py
+++ b/fiftyone/utils/cvat.py
@@ -592,7 +592,7 @@ class CVATImageDatasetImporter(
             else:
                 key = i.name
 
-            cvat_images_map[key] = i
+            cvat_images_map[fou.normpath(key)] = i
 
         filenames = set(cvat_images_map.keys())
 
@@ -759,9 +759,10 @@ class CVATVideoDatasetImporter(
         )
 
         if self.labels_path is not None and os.path.isdir(self.labels_path):
+            labels_path = fou.normpath(self.labels_path)
             labels_paths_map = {
-                os.path.splitext(p)[0]: os.path.join(self.labels_path, p)
-                for p in etau.list_files(self.labels_path, recursive=True)
+                os.path.splitext(p)[0]: os.path.join(labels_path, p)
+                for p in etau.list_files(labels_path, recursive=True)
             }
         else:
             labels_paths_map = {}
diff --git a/fiftyone/utils/data/importers.py b/fiftyone/utils/data/importers.py
index ab0363e61..4656b68d5 100644
--- a/fiftyone/utils/data/importers.py
+++ b/fiftyone/utils/data/importers.py
@@ -659,12 +659,12 @@ class ImportPathsMixin(object):
         manifest file into a UUID -> filepath map.
         """
         if ignore_exts:
-            to_uuid = lambda p: os.path.splitext(p)[0]
+            to_uuid = lambda p: fou.normpath(os.path.splitext(p)[0])
         else:
-            to_uuid = lambda p: p
+            to_uuid = lambda p: fou.normpath(p)
 
         if isinstance(data_path, dict):
-            return {to_uuid(k): v for k, v in data_path.items()}
+            return {to_uuid(k): fou.normpath(v) for k, v in data_path.items()}
 
         if not data_path:
             return {}
@@ -678,7 +678,7 @@ class ImportPathsMixin(object):
             data_map = etas.read_json(data_path)
             data_root = os.path.dirname(data_path)
             return {
-                to_uuid(k): os.path.join(data_root, v)
+                to_uuid(k): fou.normpath(os.path.join(data_root, v))
                 for k, v in data_map.items()
             }
 
@@ -686,7 +686,7 @@ class ImportPathsMixin(object):
             raise ValueError("Data directory '%s' does not exist" % data_path)
 
         return {
-            to_uuid(p): os.path.join(data_path, p)
+            to_uuid(p): fou.normpath(os.path.join(data_path, p))
             for p in etau.list_files(data_path, recursive=recursive)
         }
 
@@ -1968,6 +1968,7 @@ class FiftyOneImageClassificationDatasetImporter(
 
         if self.labels_path is not None and os.path.isfile(self.labels_path):
             labels = etas.read_json(self.labels_path)
+            labels = {fou.normpath(k): v for k, v in labels.items()}
         else:
             labels = {}
 
@@ -2420,6 +2421,7 @@ class FiftyOneImageDetectionDatasetImporter(
 
         if self.labels_path is not None and os.path.isfile(self.labels_path):
             labels = etas.read_json(self.labels_path)
+            labels = {fou.normpath(k): v for k, v in labels.items()}
         else:
             labels = {}
 
@@ -2602,6 +2604,7 @@ class FiftyOneTemporalDetectionDatasetImporter(
 
         if self.labels_path is not None and os.path.isfile(self.labels_path):
             labels = etas.read_json(self.labels_path)
+            labels = {fou.normpath(k): v for k, v in labels.items()}
         else:
             labels = {}
 
@@ -2782,9 +2785,10 @@ class ImageSegmentationDirectoryImporter(
             self.data_path, ignore_exts=True, recursive=True
         )
 
+        labels_path = fou.normpath(self.labels_path)
         labels_paths_map = {
-            os.path.splitext(p)[0]: os.path.join(self.labels_path, p)
-            for p in etau.list_files(self.labels_path, recursive=True)
+            os.path.splitext(p)[0]: os.path.join(labels_path, p)
+            for p in etau.list_files(labels_path, recursive=True)
         }
 
         uuids = set(labels_paths_map.keys())
@@ -2927,8 +2931,12 @@ class FiftyOneImageLabelsDatasetImporter(LabeledImageDatasetImporter):
         label_paths = []
         for idx in inds:
             record = index[idx]
-            image_paths.append(os.path.join(self.dataset_dir, record.data))
-            label_paths.append(os.path.join(self.dataset_dir, record.labels))
+            image_paths.append(
+                fou.normpath(os.path.join(self.dataset_dir, record.data))
+            )
+            label_paths.append(
+                fou.normpath(os.path.join(self.dataset_dir, record.labels))
+            )
 
         samples = list(zip(image_paths, label_paths))
 
@@ -3066,8 +3074,12 @@ class FiftyOneVideoLabelsDatasetImporter(LabeledVideoDatasetImporter):
         label_paths = []
         for idx in inds:
             record = index[idx]
-            video_paths.append(os.path.join(self.dataset_dir, record.data))
-            label_paths.append(os.path.join(self.dataset_dir, record.labels))
+            video_paths.append(
+                fou.normpath(os.path.join(self.dataset_dir, record.data))
+            )
+            label_paths.append(
+                fou.normpath(os.path.join(self.dataset_dir, record.labels))
+            )
 
         samples = list(zip(video_paths, label_paths))
 
diff --git a/fiftyone/utils/geojson.py b/fiftyone/utils/geojson.py
index 0a447b0d9..842f70710 100644
--- a/fiftyone/utils/geojson.py
+++ b/fiftyone/utils/geojson.py
@@ -453,7 +453,7 @@ class GeoJSONDatasetImporter(
             for feature in geojson.get("features", []):
                 properties = feature["properties"]
                 if "filename" in properties:
-                    filename = properties.pop("filename")
+                    filename = fou.normpath(properties.pop("filename"))
                     if os.path.isabs(filename):
                         filepath = filename
                     else:
diff --git a/fiftyone/utils/kitti.py b/fiftyone/utils/kitti.py
index e7e3dfa37..6c70a7cea 100644
--- a/fiftyone/utils/kitti.py
+++ b/fiftyone/utils/kitti.py
@@ -15,6 +15,7 @@ import eta.core.web as etaw
 
 import fiftyone.core.labels as fol
 import fiftyone.core.metadata as fom
+import fiftyone.core.utils as fou
 import fiftyone.utils.data as foud
 
 
@@ -166,9 +167,10 @@ class KITTIDetectionDatasetImporter(
         )
 
         if self.labels_path is not None and os.path.isdir(self.labels_path):
+            labels_path = fou.normpath(self.labels_path)
             labels_paths_map = {
-                os.path.splitext(p)[0]: os.path.join(self.labels_path, p)
-                for p in etau.list_files(self.labels_path, recursive=True)
+                os.path.splitext(p)[0]: os.path.join(labels_path, p)
+                for p in etau.list_files(labels_path, recursive=True)
             }
         else:
             labels_paths_map = {}
diff --git a/fiftyone/utils/openlabel.py b/fiftyone/utils/openlabel.py
index 64ff12273..8470d4d39 100644
--- a/fiftyone/utils/openlabel.py
+++ b/fiftyone/utils/openlabel.py
@@ -206,22 +206,22 @@ class OpenLABELImageDatasetImporter(
         image_paths_map = self._load_data_map(
             self.data_path, ignore_exts=True, recursive=True
         )
-        info = {}
-        potential_file_ids = []
+
+        file_ids = []
         annotations = OpenLABELAnnotations(fom.IMAGE)
 
         if self.labels_path is not None:
+            labels_path = fou.normpath(self.labels_path)
+
             base_dir = None
-            if os.path.isfile(self.labels_path):
-                label_paths = [self.labels_path]
-            elif os.path.isdir(self.labels_path):
-                base_dir = self.labels_path
+            if os.path.isfile(labels_path):
+                label_paths = [labels_path]
+            elif os.path.isdir(labels_path):
+                base_dir = labels_path
             elif os.path.basename(
-                self.labels_path
-            ) == "labels.json" and os.path.isdir(
-                _remove_ext(self.labels_path)
-            ):
-                base_dir = _remove_ext(self.labels_path)
+                labels_path
+            ) == "labels.json" and os.path.isdir(_remove_ext(labels_path)):
+                base_dir = _remove_ext(labels_path)
             else:
                 label_paths = []
 
@@ -230,16 +230,14 @@ class OpenLABELImageDatasetImporter(
                 label_paths = [l for l in label_paths if l.endswith(".json")]
 
             for label_path in label_paths:
-                potential_file_ids.extend(
-                    annotations.parse_labels(base_dir, label_path)
-                )
+                file_ids.extend(annotations.parse_labels(base_dir, label_path))
 
-        self._annotations = annotations
-        self._info = info
-        self._file_ids = _validate_file_ids(
-            potential_file_ids, image_paths_map
-        )
+        file_ids = _validate_file_ids(file_ids, image_paths_map)
+
+        self._info = {}
         self._image_paths_map = image_paths_map
+        self._annotations = annotations
+        self._file_ids = file_ids
 
     def get_dataset_info(self):
         return self._info
@@ -421,22 +419,22 @@ class OpenLABELVideoDatasetImporter(
         video_paths_map = self._load_data_map(
             self.data_path, ignore_exts=True, recursive=True
         )
-        info = {}
-        potential_file_ids = []
+
+        file_ids = []
         annotations = OpenLABELAnnotations(fom.VIDEO)
 
         if self.labels_path is not None:
+            labels_path = fou.normpath(self.labels_path)
+
             base_dir = None
-            if os.path.isfile(self.labels_path):
-                label_paths = [self.labels_path]
-            elif os.path.isdir(self.labels_path):
-                base_dir = self.labels_path
+            if os.path.isfile(labels_path):
+                label_paths = [labels_path]
+            elif os.path.isdir(labels_path):
+                base_dir = labels_path
             elif os.path.basename(
-                self.labels_path
-            ) == "labels.json" and os.path.isdir(
-                _remove_ext(self.labels_path)
-            ):
-                base_dir = _remove_ext(self.labels_path)
+                labels_path
+            ) == "labels.json" and os.path.isdir(_remove_ext(labels_path)):
+                base_dir = _remove_ext(labels_path)
             else:
                 label_paths = []
 
@@ -445,16 +443,14 @@ class OpenLABELVideoDatasetImporter(
                 label_paths = [l for l in label_paths if l.endswith(".json")]
 
             for label_path in label_paths:
-                potential_file_ids.extend(
-                    annotations.parse_labels(base_dir, label_path)
-                )
+                file_ids.extend(annotations.parse_labels(base_dir, label_path))
 
-        self._annotations = annotations
-        self._info = info
-        self._file_ids = _validate_file_ids(
-            potential_file_ids, video_paths_map
-        )
+        file_ids = _validate_file_ids(file_ids, video_paths_map)
+
+        self._info = {}
         self._video_paths_map = video_paths_map
+        self._annotations = annotations
+        self._file_ids = file_ids
 
     def get_dataset_info(self):
         return self._info
diff --git a/fiftyone/utils/voc.py b/fiftyone/utils/voc.py
index dea7572cf..095d7958a 100644
--- a/fiftyone/utils/voc.py
+++ b/fiftyone/utils/voc.py
@@ -143,6 +143,9 @@ class VOCDetectionDatasetImporter(
             else:
                 _uuid = None
 
+            if _uuid is not None:
+                _uuid = fou.normpath(_uuid)
+
             if _uuid not in self._image_paths_map:
                 _uuid = uuid
 
@@ -181,9 +184,10 @@ class VOCDetectionDatasetImporter(
         )
 
         if self.labels_path is not None and os.path.isdir(self.labels_path):
+            labels_path = fou.normpath(self.labels_path)
             labels_paths_map = {
-                os.path.splitext(p)[0]: os.path.join(self.labels_path, p)
-                for p in etau.list_files(self.labels_path, recursive=True)
+                os.path.splitext(p)[0]: os.path.join(labels_path, p)
+                for p in etau.list_files(labels_path, recursive=True)
             }
         else:
             labels_paths_map = {}
diff --git a/fiftyone/utils/yolo.py b/fiftyone/utils/yolo.py
index a44aaf6bb..2723210b9 100644
--- a/fiftyone/utils/yolo.py
+++ b/fiftyone/utils/yolo.py
@@ -14,6 +14,7 @@ import yaml
 import eta.core.utils as etau
 
 import fiftyone.core.labels as fol
+import fiftyone.core.utils as fou
 import fiftyone.utils.data as foud
 
 
@@ -310,7 +311,7 @@ class YOLOv4DatasetImporter(
                 if not os.path.isabs(path):
                     path = os.path.join(root_dir, path)
 
-                image_paths.append(path)
+                image_paths.append(fou.normpath(path))
         else:
             if self.images_path is not None:
                 logger.warning(
@@ -321,7 +322,7 @@ class YOLOv4DatasetImporter(
                 )
 
             image_paths = [
-                p
+                fou.normpath(p)
                 for p in etau.list_files(
                     self.data_path, abs_paths=True, recursive=True
                 )
@@ -338,6 +339,8 @@ class YOLOv4DatasetImporter(
                 # Labels are in same directory as images
                 labels_path = os.path.splitext(image_path)[0] + ".txt"
 
+            labels_path = fou.normpath(labels_path)
+
             if os.path.isfile(labels_path):
                 labels_paths_map[image_path] = labels_path
 
@@ -488,7 +491,7 @@ class YOLOv5DatasetImporter(
 
         if etau.is_str(data) and data.endswith(".txt"):
             txt_path = _parse_yolo_v5_data_path(data, self.yaml_path)
-            image_paths = _read_file_lines(txt_path)
+            image_paths = [fou.normpath(p) for p in _read_file_lines(txt_path)]
         else:
             if etau.is_str(data):
                 data_dirs = [data]
@@ -497,7 +500,9 @@ class YOLOv5DatasetImporter(
 
             image_paths = []
             for data_dir in data_dirs:
-                data_dir = _parse_yolo_v5_data_path(data_dir, self.yaml_path)
+                data_dir = fou.normpath(
+                    _parse_yolo_v5_data_path(data_dir, self.yaml_path)
+                )
                 image_paths.extend(
                     etau.list_files(data_dir, abs_paths=True, recursive=True)
                 )