Repository: zarr-developers/zarr-python

Issue:
Case insensitivity and DirectoryStore
The matching of keys in a DirectoryStore will depend on the underlying file system. If a DirectoryStore is running on a case-sensitive file system, as is usually default on most linux systems, then keys like 'foo' and 'FOO' will appear as distinct. However, if on a case-insensitive file system, as is usually default on mac and windows, then keys like 'foo' and 'FOO' will be considered as identical. I.e., if a key ``'foo'`` has been set, then ``'FOO' in store`` will return ``True``.

This is not a bug but a potential gotcha, as behaviour of code may vary between operating systems if multiple keys that are case insensitive equal are present in the data, and users are expecting case-sensitive behaviour. E.g., https://github.com/cggh/scikit-allel/issues/215 and https://github.com/ornl-oxford/genomics-benchmarks/issues/34.

At least this is something that needs to be documented. Not sure what more could be done.

Discussion:

Patch:
diff --git a/docs/release.rst b/docs/release.rst
index f4f99c2e..e87696b9 100644
--- a/docs/release.rst
+++ b/docs/release.rst
@@ -4,8 +4,12 @@ Release notes
 Upcoming Release
 ----------------
 
+* Add key normalization option for ``DirectoryStore``, ``NestedDirectoryStore``,
+  ``TempStore``, and ``N5Store``.
+  By :user:`James Bourbeau <jrbourbeau>`; :issue:`459`.
+
 * Add ``recurse`` keyword to ``Group.array_keys`` and ``Group.arrays`` methods.
-  By :user:`James Bourbeau <jrbourbeau>`; :issue:`458`
+  By :user:`James Bourbeau <jrbourbeau>`; :issue:`458`.
 
 * Use uniform chunking for all dimensions when specifying ``chunks`` as an integer.
   Also adds support for specifying ``-1`` to chunk across an entire dimension.
diff --git a/zarr/n5.py b/zarr/n5.py
index 573e4ee5..3aeb52ef 100644
--- a/zarr/n5.py
+++ b/zarr/n5.py
@@ -35,6 +35,11 @@ class N5Store(NestedDirectoryStore):
     ----------
     path : string
         Location of directory to use as the root of the storage hierarchy.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     Examples
     --------
diff --git a/zarr/storage.py b/zarr/storage.py
index 4f4e9c7a..2ad3bd33 100644
--- a/zarr/storage.py
+++ b/zarr/storage.py
@@ -681,6 +681,11 @@ class DirectoryStore(MutableMapping):
     ----------
     path : string
         Location of directory to use as the root of the storage hierarchy.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     Examples
     --------
@@ -727,7 +732,7 @@ class DirectoryStore(MutableMapping):
 
     """
 
-    def __init__(self, path):
+    def __init__(self, path, normalize_keys=False):
 
         # guard conditions
         path = os.path.abspath(path)
@@ -735,8 +740,13 @@ class DirectoryStore(MutableMapping):
             err_fspath_exists_notdir(path)
 
         self.path = path
+        self.normalize_keys = normalize_keys
+
+    def _normalize_key(self, key):
+        return key.lower() if self.normalize_keys else key
 
     def __getitem__(self, key):
+        key = self._normalize_key(key)
         filepath = os.path.join(self.path, key)
         if os.path.isfile(filepath):
             with open(filepath, 'rb') as f:
@@ -745,6 +755,7 @@ class DirectoryStore(MutableMapping):
             raise KeyError(key)
 
     def __setitem__(self, key, value):
+        key = self._normalize_key(key)
 
         # coerce to flat, contiguous array (ideally without copying)
         value = ensure_contiguous_ndarray(value)
@@ -785,6 +796,7 @@ class DirectoryStore(MutableMapping):
                 os.remove(temp_path)
 
     def __delitem__(self, key):
+        key = self._normalize_key(key)
         path = os.path.join(self.path, key)
         if os.path.isfile(path):
             os.remove(path)
@@ -796,6 +808,7 @@ class DirectoryStore(MutableMapping):
             raise KeyError(key)
 
     def __contains__(self, key):
+        key = self._normalize_key(key)
         file_path = os.path.join(self.path, key)
         return os.path.isfile(file_path)
 
@@ -910,14 +923,19 @@ class TempStore(DirectoryStore):
         Prefix for the temporary directory name.
     dir : string, optional
         Path to parent directory in which to create temporary directory.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     """
 
     # noinspection PyShadowingBuiltins
-    def __init__(self, suffix='', prefix='zarr', dir=None):
+    def __init__(self, suffix='', prefix='zarr', dir=None, normalize_keys=False):
         path = tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=dir)
         atexit.register(atexit_rmtree, path)
-        super(TempStore, self).__init__(path)
+        super(TempStore, self).__init__(path, normalize_keys=normalize_keys)
 
 
 _prog_ckey = re.compile(r'^(\d+)(\.\d+)+$')
@@ -944,6 +962,11 @@ class NestedDirectoryStore(DirectoryStore):
     ----------
     path : string
         Location of directory to use as the root of the storage hierarchy.
+    normalize_keys : bool, optional
+        If True, all store keys will be normalized to use lower case characters
+        (e.g. 'foo' and 'FOO' will be treated as equivalent). This can be
+        useful to avoid potential discrepancies between case-senstive and
+        case-insensitive file system. Default value is False.
 
     Examples
     --------
@@ -1000,8 +1023,8 @@ class NestedDirectoryStore(DirectoryStore):
 
     """
 
-    def __init__(self, path):
-        super(NestedDirectoryStore, self).__init__(path)
+    def __init__(self, path, normalize_keys=False):
+        super(NestedDirectoryStore, self).__init__(path, normalize_keys=normalize_keys)
 
     def __getitem__(self, key):
         key = _nested_map_ckey(key)


Test patch:
diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py
index 0b517c43..a4df9a90 100644
--- a/zarr/tests/test_storage.py
+++ b/zarr/tests/test_storage.py
@@ -735,10 +735,10 @@ class TestDictStore(StoreTests, unittest.TestCase):
 
 class TestDirectoryStore(StoreTests, unittest.TestCase):
 
-    def create_store(self):
+    def create_store(self, normalize_keys=False):
         path = tempfile.mkdtemp()
         atexit.register(atexit_rmtree, path)
-        store = DirectoryStore(path)
+        store = DirectoryStore(path, normalize_keys=normalize_keys)
         return store
 
     def test_filesystem_path(self):
@@ -774,13 +774,19 @@ class TestDirectoryStore(StoreTests, unittest.TestCase):
         store = self.create_store()
         setdel_hierarchy_checks(store)
 
+    def test_normalize_keys(self):
+        store = self.create_store(normalize_keys=True)
+        store['FOO'] = b'bar'
+        assert 'FOO' in store
+        assert 'foo' in store
+
 
 class TestNestedDirectoryStore(TestDirectoryStore, unittest.TestCase):
 
-    def create_store(self):
+    def create_store(self, normalize_keys=False):
         path = tempfile.mkdtemp()
         atexit.register(atexit_rmtree, path)
-        store = NestedDirectoryStore(path)
+        store = NestedDirectoryStore(path, normalize_keys=normalize_keys)
         return store
 
     def test_chunk_nesting(self):
@@ -798,10 +804,10 @@ class TestNestedDirectoryStore(TestDirectoryStore, unittest.TestCase):
 
 class TestN5Store(TestNestedDirectoryStore, unittest.TestCase):
 
-    def create_store(self):
+    def create_store(self, normalize_keys=False):
         path = tempfile.mkdtemp(suffix='.n5')
         atexit.register(atexit_rmtree, path)
-        store = N5Store(path)
+        store = N5Store(path, normalize_keys=normalize_keys)
         return store
 
     def test_equal(self):