Repository: dask/dask

Issue:
Error requirements starting dask-scheduler on HPC cluster
Hi
I installed dask on python3 via pip3 on HPC node. when I run the scheduler I get the following error:

File "/home/x_simco/test/lib/python3.4/site-packages/dask/compatibility.py", line 23, in <module>
    from lzmaffi import (LZMAFile, compress as lzma_compress,
ImportError: No module named 'lzmaffi'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/x_simco/test/bin/dask-scheduler", line 7, in <module>
    from distributed.cli.dask_scheduler import go
  File "/home/x_simco/test/lib/python3.4/site-packages/distributed/**init**.py", line 3, in <module>
    from .center import Center
  File "/home/x_simco/test/lib/python3.4/site-packages/distributed/center.py", line 13, in <module>
    from .core import (Server, read, write, rpc, pingpong, send_recv,
  File "/home/x_simco/test/lib/python3.4/site-packages/distributed/core.py", line 26, in <module>
    from .utils import get_traceback, truncate_exception, ignoring
  File "/home/x_simco/test/lib/python3.4/site-packages/distributed/utils.py", line 15, in <module>
    from dask import istask
  File "/home/x_simco/test/lib/python3.4/site-packages/dask/**init**.py", line 5, in <module>
    from .async import get_sync as get
  File "/home/x_simco/test/lib/python3.4/site-packages/dask/async.py", line 126, in <module>
    from .optimize import cull
  File "/home/x_simco/test/lib/python3.4/site-packages/dask/optimize.py", line 6, in <module>
    from .compatibility import zip_longest
  File "/home/x_simco/test/lib/python3.4/site-packages/dask/compatibility.py", line 26, in <module>
    from lzma import (LZMAFile, compress as lzma_compress,
  File "/software/apps/python/3.4.1/snic-1/lib/python3.4/lzma.py", line 26, in <module>
    from _lzma import *



Discussion:
Hrm, that's surprising.  I thought that lzma was part of the standard library.  This should be fairly easy to guard against.  I'll post a fix soon.


Resolving patch:
diff --git a/dask/compatibility.py b/dask/compatibility.py
--- a/dask/compatibility.py
+++ b/dask/compatibility.py
@@ -9,7 +9,11 @@
 PY3 = sys.version_info[0] == 3
 PY2 = sys.version_info[0] == 2
 
-LZMA_AVAILABLE = True
+class LZMAFile:
+    def __init__(self, *args, **kwargs):
+        raise ValueError("xz files requires the lzma module. "
+                            "To use, install lzmaffi or backports.lzma.")
+LZMA_AVAILABLE = False
 
 if PY3:
     import builtins
@@ -20,11 +24,17 @@
     from gzip import (GzipFile, compress as gzip_compress,
             decompress as gzip_decompress)
     try:
-        from lzmaffi import (LZMAFile, compress as lzma_compress,
-                             decompress as lzma_decompress)
+        try:
+            from lzmaffi import (LZMAFile, compress as lzma_compress,
+                                 decompress as lzma_decompress)
+        except ImportError:
+            from lzma import (LZMAFile, compress as lzma_compress,
+                              decompress as lzma_decompress)
+        LZMA_AVAILABLE = True
     except ImportError:
-        from lzma import (LZMAFile, compress as lzma_compress,
-                          decompress as lzma_decompress)
+        # Fallback to top-level definition
+        pass
+
     from urllib.request import urlopen
     from urllib.parse import urlparse
     from urllib.parse import quote, unquote
@@ -204,12 +214,10 @@ def writelines(self, *args, **kwargs):
             from backports.lzma import LZMAFile
             from backports.lzma import (LZMAFile, compress as lzma_compress,
                                         decompress as lzma_decompress)
+        LZMA_AVAILABLE = True
     except ImportError:
-        class LZMAFile:
-            def __init__(self, *args, **kwargs):
-                raise ValueError("xz files requires the lzma module. "
-                                 "To use, install lzmaffi or backports.lzma.")
-        LZMA_AVAILABLE = False
+        # Fallback to top-level definition
+        pass
 
 
 def getargspec(func):


Test patch: