Repository: yt-project/unyt
Pull request #115 (2019-12-31 00:27:31)

Add support for latest versions of h5py, sympy, numpy, and pytest 
This fixes a number of issues caused by updates to upstream libraries we depend on. This is a bit of a grab-bag PR that makes the tests pass again on the latest versions of our dependencies. Unfortunately I can't easily issue these changes as separate PRs since the tests will be broken without all of these coming in at once. If you have questions about reviewing this I'm happy to help out.

To catch issues caused by upstream changes more quickly in the future I've updated the tox config to always recreate a virtualenv. This will ensure we get test failures on the cron job test whenever an upstream release breaks our test suite.

I also made the following updates to our tox config:

* Add testing for python 3.8, update older python minor versions
* Remove pytest version constraint now that issue with pytest-doctest-plus has been fixed.
* Pin `coverage` to be older than coverage 5.0, which allows us to work around https://github.com/nedbat/coveragepy/issues/883

There also some code changes, in particular

* Avoid deprecation warnings from h5py by always specifying the file mode in `h5py.open`.
* Fix usage of `rationalize` transformer in our usage of `sympy.parse_expr`. Before we were not using it correctly since we needed to run `auto_number` first.
* Fix latex output for non-float unit scalar multiples.
* Add support for NumPy 1.8 by fixing handling of the `out` keyword argument for ufuncs that have multiple outputs.
* Fix warnings from Pint caused by our implicity using `Pint.quantity.__array__`.
* Make the unit registry saved to HDF5 files smaller to avoid the attribute size limit triggered in newer versions of h5py. Fixes https://github.com/yt-project/yt/issues/2404.


Patch:
diff --git a/.travis.yml b/.travis.yml
index 865e8f7..15bec08 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,10 +29,11 @@ before_install:
   - export PYENV_ROOT="$HOME/.pyenv"
   - export PATH="$HOME/.pyenv/bin:$PATH"
   - eval "$(pyenv init -)"
-  - pyenv install -s 3.5.5
-  - pyenv install -s 3.6.5
-  - pyenv install -s 3.7.1
-  - pyenv local 3.5.5 3.6.5 3.7.1
+  - pyenv install -s 3.5.7
+  - pyenv install -s 3.6.9
+  - pyenv install -s 3.7.5
+  - pyenv install -s 3.8.0
+  - pyenv local 3.5.7 3.6.9 3.7.5 3.8.0
   - pip install tox tox-pyenv codecov twine
 
 # Command to run tests, e.g. python setup.py test
diff --git a/docs/usage.rst b/docs/usage.rst
index 3414a7f..acaf020 100644
--- a/docs/usage.rst
+++ b/docs/usage.rst
@@ -1057,7 +1057,7 @@ HDF5 Files
 
 The :mod:`unyt` library provides a hook for writing data both to a new HDF5 file and an existing file and then subsequently reading that data back in to restore the array. This works via the :meth:`unyt_array.write_hdf5 <unyt.array.unyt_array.write_hdf5>` and :meth:`unyt_array.from_hdf5 <unyt.array.unyt_array.from_hdf5>` methods. The simplest way to use these functions is to write data to a file that does not exist yet:
 
-  >>> from unyt import cm, unyt_array
+  >>> from unyt import cm
   >>> import os
   >>> data = [1, 2, 3]*cm
   >>> data.write_hdf5('my_data.h5')
diff --git a/tox.ini b/tox.ini
index 56ee095..2cc65dc 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,8 +1,9 @@
 [tox]
-envlist = py36-docs,begin,py35-dependencies,py35-versions,py{35,36,37},end
+envlist = py36-docs,begin,py35-dependencies,py35-versions,py{35,36,37,38},end
 
 [travis]
 python =
+    3.8: py38
     3.7: py37
     3.6: py36, py36-docs
     3.5: py35, py35-dependencies, py35-versions
@@ -10,15 +11,16 @@ python =
 [testenv]
 setenv =
     PYTHONPATH = {toxinidir}
+recreate = true
 depends = begin
 deps =
-    pytest<5.1
+    pytest
     sympy
     numpy
     h5py
     pint
     astropy
-    coverage
+    coverage<5.0
     pytest-cov
     pytest-doctestplus
     flake8
@@ -30,13 +32,13 @@ commands =
 
 [testenv:py35-versions]
 deps =
-    pytest<5.1
+    pytest
     sympy==1.2
     numpy==1.13.3
     h5py==2.6.0
     pint==0.6
     astropy==1.3.3
-    coverage
+    coverage<5.0
     pytest-cov
     pytest-doctestplus
 commands =
@@ -46,10 +48,10 @@ commands =
 
 [testenv:py35-dependencies]
 deps =
-    pytest<5.1
+    pytest
     sympy
     numpy
-    coverage
+    coverage<5.0
     pytest-cov
     pytest-doctestplus
 depends = begin
@@ -77,13 +79,13 @@ commands =
 depends =
 skip_install = true
 deps =
-    coverage
+    coverage<5.0
 
 [testenv:end]
 commands =
     coverage report --omit='.tox/*'
     coverage html --omit='.tox/*'
 skip_install = true
-depends = py{35,36,37}
+depends = py{35,36,37,38}
 deps =
-    coverage
+    coverage<5.0
diff --git a/unyt/_parsing.py b/unyt/_parsing.py
index 54a048f..2ce4393 100644
--- a/unyt/_parsing.py
+++ b/unyt/_parsing.py
@@ -76,7 +76,7 @@ global_dict = {
     "sqrt": sqrt,
 }
 
-unit_text_transform = (_auto_positive_symbol, rationalize, auto_number)
+unit_text_transform = (_auto_positive_symbol, auto_number, rationalize)
 
 
 def parse_unyt_expr(unit_expr):
diff --git a/unyt/array.py b/unyt/array.py
index b1c30f7..a58f90f 100644
--- a/unyt/array.py
+++ b/unyt/array.py
@@ -127,8 +127,13 @@ from unyt.exceptions import (
 from unyt.equivalencies import equivalence_registry
 from unyt._on_demand_imports import _astropy, _pint
 from unyt._pint_conversions import convert_pint_units
+from unyt._unit_lookup_table import default_unit_symbol_lut
 from unyt.unit_object import _check_em_conversion, _em_conversion, Unit
-from unyt.unit_registry import _sanitize_unit_system, UnitRegistry
+from unyt.unit_registry import (
+    _sanitize_unit_system,
+    UnitRegistry,
+    default_unit_registry,
+)
 
 NULL_UNIT = Unit()
 POWER_SIGN_MAPPING = {multiply: 1, divide: -1}
@@ -321,6 +326,8 @@ binary_operators = (
 
 trigonometric_operators = (sin, cos, tan)
 
+multiple_output_operators = {modf: 2, frexp: 2, divmod_: 2}
+
 LARGE_INPUT = {4: 16777217, 8: 9007199254740993}
 
 
@@ -1317,12 +1324,16 @@ class unyt_array(np.ndarray):
             info = {}
 
         info["units"] = str(self.units)
-        info["unit_registry"] = np.void(pickle.dumps(self.units.registry.lut))
+        lut = {}
+        for k, v in self.units.registry.lut.items():
+            if k not in default_unit_registry.lut:
+                lut[k] = v
+        info["unit_registry"] = np.void(pickle.dumps(lut))
 
         if dataset_name is None:
             dataset_name = "array_data"
 
-        f = h5py.File(filename)
+        f = h5py.File(filename, "a")
         if group_name is not None:
             if group_name in f:
                 g = f[group_name]
@@ -1372,7 +1383,7 @@ class unyt_array(np.ndarray):
         if dataset_name is None:
             dataset_name = "array_data"
 
-        f = h5py.File(filename)
+        f = h5py.File(filename, "r")
         if group_name is not None:
             g = f[group_name]
         else:
@@ -1380,7 +1391,9 @@ class unyt_array(np.ndarray):
         dataset = g[dataset_name]
         data = dataset[:]
         units = dataset.attrs.get("units", "")
-        unit_lut = pickle.loads(dataset.attrs["unit_registry"].tostring())
+        unit_lut = default_unit_symbol_lut.copy()
+        unit_lut_load = pickle.loads(dataset.attrs["unit_registry"].tostring())
+        unit_lut.update(unit_lut_load)
         f.close()
         registry = UnitRegistry(lut=unit_lut, add_default_symbols=False)
         return cls(data, units, registry=registry)
@@ -1584,18 +1597,29 @@ class unyt_array(np.ndarray):
     def __array_ufunc__(self, ufunc, method, *inputs, **kwargs):
         func = getattr(ufunc, method)
         if "out" not in kwargs:
-            out = None
-            out_func = None
+            if ufunc in multiple_output_operators:
+                out = (None,) * multiple_output_operators[ufunc]
+                out_func = out
+            else:
+                out = None
+                out_func = None
         else:
             # we need to get both the actual "out" object and a view onto it
             # in case we need to do in-place operations
-            out = kwargs.pop("out")[0]
-            if out.dtype.kind in ("u", "i"):
-                new_dtype = "f" + str(out.dtype.itemsize)
-                float_values = out.astype(new_dtype)
-                out.dtype = new_dtype
-                np.copyto(out, float_values)
-            out_func = out.view(np.ndarray)
+            out = kwargs.pop("out")
+            if ufunc in multiple_output_operators:
+                out_func = []
+                for arr in out:
+                    out_func.append(arr.view(np.ndarray))
+                out_func = tuple(out_func)
+            else:
+                out = out[0]
+                if out.dtype.kind in ("u", "i"):
+                    new_dtype = "f" + str(out.dtype.itemsize)
+                    float_values = out.astype(new_dtype)
+                    out.dtype = new_dtype
+                    np.copyto(out, float_values)
+                out_func = out.view(np.ndarray)
         if len(inputs) == 1:
             # Unary ufuncs
             inp = inputs[0]
@@ -1775,6 +1799,14 @@ class unyt_array(np.ndarray):
                 except AttributeError:
                     # out_arr is an ndarray
                     out.units = Unit("", registry=self.units.registry)
+            elif isinstance(out, tuple):
+                for o, oa in zip(out, out_arr):
+                    if o is None:
+                        continue
+                    try:
+                        o.units = oa.units
+                    except AttributeError:
+                        o.units = Unit("", registry=self.units.registry)
         if mul == 1:
             return out_arr
         return mul * out_arr
diff --git a/unyt/unit_object.py b/unyt/unit_object.py
index 54c34c1..dace50f 100644
--- a/unyt/unit_object.py
+++ b/unyt/unit_object.py
@@ -87,7 +87,7 @@ def _get_latex_representation(expr, registry):
     l_expr = expr
     if isinstance(expr, Mul):
         coeffs = expr.as_coeff_Mul()
-        if coeffs[0] == 1 or not isinstance(coeffs[0], Float):
+        if coeffs[0] == 1 or not isinstance(coeffs[0], Number):
             l_expr = coeffs[1]
         else:
             l_expr = coeffs[1]
@@ -317,7 +317,7 @@ class Unit(object):
         return self
 
     def __hash__(self):
-        return hash(self.registry.unit_system_id) ^ hash(self.expr)
+        return super(Unit, self).__hash__()
 
     # end sympy conventions
 


Test patch:
diff --git a/unyt/tests/test_units.py b/unyt/tests/test_units.py
index 35661e7..40e1f4d 100644
--- a/unyt/tests/test_units.py
+++ b/unyt/tests/test_units.py
@@ -791,14 +791,3 @@ def test_percent():
 
     assert a == c
     assert c == d
-
-
-def test_equal_has_same_hash():
-    a = Unit("m")
-    b = Unit("m")
-    c = Unit("m*s/s")
-
-    assert a == b
-    assert b == c
-    assert hash(a) == hash(b)
-    assert hash(b) == hash(c)
diff --git a/unyt/tests/test_unyt_array.py b/unyt/tests/test_unyt_array.py
index 5c031cc..18c1773 100644
--- a/unyt/tests/test_unyt_array.py
+++ b/unyt/tests/test_unyt_array.py
@@ -1010,7 +1010,10 @@ def unary_ufunc_comparison(ufunc, a):
 
 
 def binary_ufunc_comparison(ufunc, a, b):
-    out = b.copy()
+    if ufunc in [np.divmod]:
+        out = (b.copy(), b.copy())
+    else:
+        out = b.copy()
     if ufunc in yield_np_ufuncs(
         [
             "add",
@@ -1073,7 +1076,10 @@ def binary_ufunc_comparison(ufunc, a, b):
     ):
         assert not isinstance(ret, unyt_array) and isinstance(ret, np.ndarray)
     if isinstance(ret, tuple):
-        assert_array_equal(ret[0], out)
+        assert isinstance(out, tuple)
+        assert len(out) == len(ret)
+        for o, r in zip(out, ret):
+            assert_array_equal(r, o)
     else:
         assert_array_equal(ret, out)
     if ufunc in (np.divide, np.true_divide, np.arctan2) and (
@@ -1353,6 +1359,10 @@ def test_astropy():
 
 
 def test_pint():
+    def assert_pint_array_equal(arr1, arr2):
+        assert_array_equal(arr1.magnitude, arr2.magnitude)
+        assert str(arr1.units) == str(arr2.units)
+
     if isinstance(_pint.UnitRegistry, NotAModule):
         return
     ureg = _pint.UnitRegistry()
@@ -1365,13 +1375,11 @@ def test_pint():
     yt_quan = unyt_quantity(10.0, "sqrt(g)/mm**3")
     yt_quan2 = unyt_quantity.from_pint(p_quan)
 
-    assert_array_equal(p_arr, yt_arr.to_pint())
-    assert_equal(p_quan, yt_quan.to_pint())
+    assert_pint_array_equal(p_arr, yt_arr.to_pint())
     assert_array_equal(yt_arr, unyt_array.from_pint(p_arr))
     assert_array_equal(yt_arr, yt_arr2)
 
-    assert_equal(p_quan.magnitude, yt_quan.to_pint().magnitude)
-    assert_equal(p_quan, yt_quan.to_pint())
+    assert_pint_array_equal(p_quan, yt_quan.to_pint())
     assert_equal(yt_quan, unyt_quantity.from_pint(p_quan))
     assert_equal(yt_quan, yt_quan2)
 
@@ -1477,7 +1485,7 @@ def test_h5_io():
 
     # write to a group that does exist
 
-    with _h5py.File("test.h5") as f:
+    with _h5py.File("test.h5", "a") as f:
         f.create_group("/arrays/test_group")
 
     warr.write_hdf5(