Repository: dfm/emcee

Issue:
long double tests fail on ppc64el
**General information:**

- emcee version: 3.0.2
- platform: ppc64el
- installation method (pip/conda/source/other?): Ubuntu/Debian package

**Problem description:**

Version 3.0.2 of emcee is not migrating to the release pocket of Ubuntu because tests are failing on ppc64el: https://autopkgtest.ubuntu.com/packages/e/emcee/groovy/ppc64el

### Expected behavior

tests pass

### Actual behavior: 

test_longdouble_actually_needed[TempHDFBackend] and test_longdouble_preserved[TempHDFBackend] fail:

```
_______________ test_longdouble_actually_needed[TempHDFBackend] ________________

cls = <class 'emcee.backends.hdf.TempHDFBackend'>

    @pytest.mark.parametrize("cls", [emcee.backends.Backend,
                                     emcee.backends.TempHDFBackend])
    def test_longdouble_actually_needed(cls):
    
        mjd = np.longdouble(58000.)
        sigma = 100*np.finfo(np.longdouble).eps*mjd
    
        def log_prob(x):
            assert x.dtype == np.longdouble
            return -0.5 * np.sum(((x-mjd)/sigma) ** 2)
    
        ndim, nwalkers = 1, 20
        steps = 1000
        p0 = sigma*np.random.randn(nwalkers, ndim).astype(np.longdouble) + mjd
        assert not all(p0 == mjd)
    
        with cls(dtype=np.longdouble) as backend:
            sampler = emcee.EnsembleSampler(nwalkers,
                                            ndim,
                                            log_prob,
                                            backend=backend)
            sampler.run_mcmc(p0, steps)
    
            samples = sampler.get_chain().reshape((-1,))
>           assert samples.dtype == np.longdouble
E           AssertionError: assert dtype('<f8') == <class 'numpy.float128'>
E            +  where dtype('<f8') = array([ 5.80000000e+04, -4.60105223e-27,  5.80000000e+04, ...,\n       -6.24580733e-26,  5.80000000e+04, -7.38930048e-27]).dtype
E            +  and   <class 'numpy.float128'> = np.longdouble
```

(the other failure is similar)

### What have you tried so far?

Googling and getting a headache :)

I know that "long double" on PowerPC / POWER is a strange pair-of-64-bit-doubles type so I'm not surprised that it's on POWER that I see the problem. I also see from searching that hd5/h5py/numpy have a history of problems in this area (e.g. https://github.com/h5py/h5py/issues/817) so I'm not sure what the resolution should be. Skipping these tests on ppc64el for now (maybe in an Ubuntu specific patch) might make sense -- it's not like the failure of these new tests indicates any kind of regression.

Comments:

Patch:
diff --git a/src/emcee/autocorr.py b/src/emcee/autocorr.py
index 5156372..4a804ff 100644
--- a/src/emcee/autocorr.py
+++ b/src/emcee/autocorr.py
@@ -49,7 +49,7 @@ def integrated_time(x, c=5, tol=50, quiet=False):
     """Estimate the integrated autocorrelation time of a time series.
 
     This estimate uses the iterative procedure described on page 16 of
-    `Sokal's notes <http://www.stat.unc.edu/faculty/cji/Sokal.pdf>`_ to
+    `Sokal's notes <https://www.semanticscholar.org/paper/Monte-Carlo-Methods-in-Statistical-Mechanics%3A-and-Sokal/0bfe9e3db30605fe2d4d26e1a288a5e2997e7225>`_ to
     determine a reasonable window size.
 
     Args:
diff --git a/src/emcee/backends/hdf.py b/src/emcee/backends/hdf.py
index d4e6d6c..9503512 100644
--- a/src/emcee/backends/hdf.py
+++ b/src/emcee/backends/hdf.py
@@ -2,7 +2,7 @@
 
 from __future__ import division, print_function
 
-__all__ = ["HDFBackend", "TempHDFBackend"]
+__all__ = ["HDFBackend", "TempHDFBackend", "does_hdf5_support_longdouble"]
 
 import os
 from tempfile import NamedTemporaryFile
@@ -19,6 +19,25 @@ except ImportError:
     h5py = None
 
 
+def does_hdf5_support_longdouble():
+    if h5py is None:
+        return False
+    with NamedTemporaryFile(prefix="emcee-temporary-hdf5",
+                            suffix=".hdf5",
+                            delete=False) as f:
+        f.close()
+
+        with h5py.File(f.name, "w") as hf:
+            g = hf.create_group("group")
+            g.create_dataset("data", data=np.ones(1, dtype=np.longdouble))
+            if g["data"].dtype != np.longdouble:
+                return False
+        with h5py.File(f.name, "r") as hf:
+            if hf["group"]["data"].dtype != np.longdouble:
+                return False
+    return True
+
+
 class HDFBackend(Backend):
     """A backend that stores the chain in an HDF5 file using h5py
 
diff --git a/tox.ini b/tox.ini
index 6c09ae3..9f72a99 100644
--- a/tox.ini
+++ b/tox.ini
@@ -1,5 +1,5 @@
 [tox]
-envlist = py{36,37}
+envlist = py{36,37,38}
 
 [testenv]
 description = Unit tests


Test patch:
diff --git a/src/emcee/tests/integration/test_longdouble.py b/src/emcee/tests/integration/test_longdouble.py
index a2142d5..b8d2c7a 100644
--- a/src/emcee/tests/integration/test_longdouble.py
+++ b/src/emcee/tests/integration/test_longdouble.py
@@ -2,6 +2,7 @@ import numpy as np
 import pytest
 
 import emcee
+from emcee.backends.hdf import does_hdf5_support_longdouble, TempHDFBackend
 
 
 def test_longdouble_doesnt_crash_bug_312():
@@ -17,9 +18,11 @@ def test_longdouble_doesnt_crash_bug_312():
     sampler.run_mcmc(p0, 100)
 
 
-@pytest.mark.parametrize("cls", [emcee.backends.Backend,
-                                 emcee.backends.TempHDFBackend])
+@pytest.mark.parametrize("cls", emcee.backends.get_test_backends())
 def test_longdouble_actually_needed(cls):
+    if (issubclass(cls, TempHDFBackend)
+            and not does_hdf5_support_longdouble()):
+        pytest.xfail("HDF5 does not support long double on this platform")
 
     mjd = np.longdouble(58000.)
     sigma = 100*np.finfo(np.longdouble).eps*mjd
diff --git a/src/emcee/tests/unit/test_backends.py b/src/emcee/tests/unit/test_backends.py
index e5466ec..370008e 100644
--- a/src/emcee/tests/unit/test_backends.py
+++ b/src/emcee/tests/unit/test_backends.py
@@ -2,12 +2,18 @@
 
 import os
 from itertools import product
+from tempfile import NamedTemporaryFile
 
-import h5py
 import numpy as np
 import pytest
 
 from emcee import EnsembleSampler, backends, State
+from emcee.backends.hdf import does_hdf5_support_longdouble
+
+try:
+    import h5py
+except ImportError:
+    h5py = None
 
 __all__ = ["test_backend", "test_reload"]
 
@@ -55,6 +61,7 @@ def _custom_allclose(a, b):
             assert np.allclose(a[n], b[n])
 
 
+@pytest.mark.skipif(h5py is None, reason="HDF5 not available")
 def test_uninit(tmpdir):
     fn = str(tmpdir.join("EMCEE_TEST_FILE_DO_NOT_USE.h5"))
     if os.path.exists(fn):
@@ -208,6 +215,7 @@ def test_restart(backend, dtype):
         assert np.allclose(a, b), "inconsistent acceptance fraction"
 
 
+@pytest.mark.skipif(h5py is None, reason="HDF5 not available")
 def test_multi_hdf5():
     with backends.TempHDFBackend() as backend1:
         run_sampler(backend1)
@@ -227,6 +235,9 @@ def test_multi_hdf5():
 
 @pytest.mark.parametrize("backend", all_backends)
 def test_longdouble_preserved(backend):
+    if (issubclass(backend, backends.TempHDFBackend)
+            and not does_hdf5_support_longdouble()):
+        pytest.xfail("HDF5 does not support long double on this platform")
     nwalkers = 10
     ndim = 2
     nsteps = 5
@@ -252,14 +263,3 @@ def test_longdouble_preserved(backend):
 
             assert s.log_prob.dtype == np.longdouble
             assert np.all(s.log_prob == lp)
-
-
-def test_hdf5_dtypes():
-    nwalkers = 10
-    ndim = 2
-    with backends.TempHDFBackend(dtype=np.longdouble) as b:
-        assert b.dtype == np.longdouble
-        b.reset(nwalkers, ndim)
-        with h5py.File(b.filename, "r") as f:
-            g = f["test"]
-            assert g["chain"].dtype == np.longdouble