Repository: joblib/joblib

Issue:
Cannot hash datetime64
```
In [59]: joblib.hashing.hash(numpy.zeros(dtype="<m8[s]", shape=(10,)))
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-59-8b9b8a0f034d> in <module>()
----> 1 joblib.hashing.hash(numpy.zeros(dtype="<m8[s]", shape=(10,)))

/export/data/home/gholl/venv/bleeding/lib/python3.4/site-packages/joblib/hashing.py in hash(obj, hash_name, coerce_mmap)
    199     else:
    200         hasher = Hasher(hash_name=hash_name)
--> 201     return hasher.hash(obj)

/export/data/home/gholl/venv/bleeding/lib/python3.4/site-packages/joblib/hashing.py in hash(self, obj, return_digest)
     51     def hash(self, obj, return_digest=True):
     52         try:
---> 53             self.dump(obj)
     54         except pickle.PicklingError as e:
     55             warnings.warn('PicklingError while hashing %r: %r' % (obj, e))

/usr/lib64/python3.4/pickle.py in dump(self, obj)
    408         if self.proto >= 4:
    409             self.framer.start_framing()
--> 410         self.save(obj)
    411         self.write(STOP)
    412         self.framer.end_framing()

/export/data/home/gholl/venv/bleeding/lib/python3.4/site-packages/joblib/hashing.py in save(self, obj)
    162                 # Python 3.2's memoryview raise a ValueError instead of a
    163                 # TypeError or a BufferError
--> 164                 self._hash.update(self._getbuffer(obj.flatten()))
    165 
    166             # We store the class, to be able to distinguish between

ValueError: cannot include dtype 'm' in a buffer
```

Discussion:
lesteve: Forgot to say, but I checked that this fix didn't affect the hash of non datetime-like numpy arrays.
ogrisel: Other than my comment, LGTM.

Patch:
diff --git a/joblib/hashing.py b/joblib/hashing.py
index 564817c..d3d783a 100644
--- a/joblib/hashing.py
+++ b/joblib/hashing.py
@@ -154,14 +154,21 @@ class NumpyHasher(Hasher):
         if isinstance(obj, self.np.ndarray) and not obj.dtype.hasobject:
             # Compute a hash of the object:
             try:
-                self._hash.update(self._getbuffer(obj))
+                # memoryview is not supported for some dtypes,
+                # e.g. datetime64, see
+                # https://github.com/numpy/numpy/issues/4983.  The
+                # workaround is to view the array as bytes before
+                # taking the memoryview
+                obj_bytes_view = obj.view(self.np.uint8)
+                self._hash.update(self._getbuffer(obj_bytes_view))
             except (TypeError, BufferError, ValueError):
                 # Cater for non-single-segment arrays: this creates a
                 # copy, and thus aleviates this issue.
                 # XXX: There might be a more efficient way of doing this
                 # Python 3.2's memoryview raise a ValueError instead of a
                 # TypeError or a BufferError
-                self._hash.update(self._getbuffer(obj.flatten()))
+                obj_bytes_view = obj.flatten().view(self.np.uint8)
+                self._hash.update(self._getbuffer(obj_bytes_view))
 
             # We store the class, to be able to distinguish between
             # Objects with the same binary content, but different


Test patch:
diff --git a/joblib/test/test_hashing.py b/joblib/test/test_hashing.py
index 15a5da9..a1f52b1 100644
--- a/joblib/test/test_hashing.py
+++ b/joblib/test/test_hashing.py
@@ -133,6 +133,18 @@ def test_hash_numpy():
     yield nose.tools.assert_not_equal, hash(arr1), hash(arr1.T)
 
 
+@with_numpy
+def test_numpy_datetime_array():
+    # memoryview is not supported for some dtypes e.g. datetime64
+    # see https://github.com/joblib/joblib/issues/188 for more details
+    dtypes = ['datetime64[s]', 'timedelta64[D]']
+
+    a_hash = hash(np.arange(10))
+    arrays = (np.arange(0, 10, dtype=dtype) for dtype in dtypes)
+    for array in arrays:
+        nose.tools.assert_not_equal(hash(array), a_hash)
+
+
 @with_numpy
 def test_hash_numpy_noncontiguous():
     a = np.asarray(np.arange(6000).reshape((1000, 2, 3)),