Repository: DataBiosphere/toil

Issue:
Hotdeployment fails when certain python-3-supporting libraries are in the virtualenv
Some libraries, e.g. jinja2, have source files with python-3-only syntax, using the `async` keyword. This isn't a problem when loading them using python 2, because those source files just aren't loaded when using python 2.

But toil seems to be parsing every python file in the entire virtualenv when preparing to hotdeploy it. This ends up crashing because the `async` keyword isn't recognized:

```
Registered with framework ID 98da91a2-889a-4c12-a798-8128c2832c13-0011
Saving user script ModuleDescriptor(dirPath='/venv/lib/python2.7/site-packages', name='cactus.progressive.cactus_progressive', fromVirtualEnv=True) as a resource
  File "/venv/lib/python2.7/site-packages/jinja2/asyncsupport.py", line 22
    async def concat_async(async_gen):
            ^
SyntaxError: invalid syntax

Traceback (most recent call last):
  File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/venv/lib/python2.7/site-packages/cactus/progressive/cactus_progressive.py", line 496, in <module>
    main()
  File "/venv/lib/python2.7/site-packages/cactus/progressive/cactus_progressive.py", line 461, in main
    halID = toil.restart()
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 699, in restart
    self._setupHotDeployment()
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 874, in _setupHotDeployment
    userScriptResource = userScript.saveAsResourceTo(self._jobStore)
  File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 445, in saveAsResourceTo
    return self._getResourceClass().create(jobStore, self._resourcePath)
  File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 85, in create
    with cls._load(leaderPath) as src:
  File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 316, in _load
    zipFile.writepy(path)
  File "/usr/lib/python2.7/zipfile.py", line 1402, in writepy
    self.write(fname, arcname)
  File "/usr/lib/python2.7/zipfile.py", line 1118, in write
    st = os.stat(filename)
OSError: [Errno 2] No such file or directory: '/venv/lib/python2.7/site-packages/jinja2/asyncsupport.pyc'
```

Is there a reason to use `writepy` on each .py file rather than just naively dumping the entire virtualenv to a zip? It looks like it's trying to compile the file before zipping.
Hotdeployment fails when certain python-3-supporting libraries are in the virtualenv
Some libraries, e.g. jinja2, have source files with python-3-only syntax, using the `async` keyword. This isn't a problem when loading them using python 2, because those source files just aren't loaded when using python 2.

But toil seems to be parsing every python file in the entire virtualenv when preparing to hotdeploy it. This ends up crashing because the `async` keyword isn't recognized:

```
Registered with framework ID 98da91a2-889a-4c12-a798-8128c2832c13-0011
Saving user script ModuleDescriptor(dirPath='/venv/lib/python2.7/site-packages', name='cactus.progressive.cactus_progressive', fromVirtualEnv=True) as a resource
  File "/venv/lib/python2.7/site-packages/jinja2/asyncsupport.py", line 22
    async def concat_async(async_gen):
            ^
SyntaxError: invalid syntax

Traceback (most recent call last):
  File "/usr/lib/python2.7/runpy.py", line 162, in _run_module_as_main
    "__main__", fname, loader, pkg_name)
  File "/usr/lib/python2.7/runpy.py", line 72, in _run_code
    exec code in run_globals
  File "/venv/lib/python2.7/site-packages/cactus/progressive/cactus_progressive.py", line 496, in <module>
    main()
  File "/venv/lib/python2.7/site-packages/cactus/progressive/cactus_progressive.py", line 461, in main
    halID = toil.restart()
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 699, in restart
    self._setupHotDeployment()
  File "/usr/local/lib/python2.7/dist-packages/toil/common.py", line 874, in _setupHotDeployment
    userScriptResource = userScript.saveAsResourceTo(self._jobStore)
  File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 445, in saveAsResourceTo
    return self._getResourceClass().create(jobStore, self._resourcePath)
  File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 85, in create
    with cls._load(leaderPath) as src:
  File "/usr/local/lib/python2.7/dist-packages/toil/resource.py", line 316, in _load
    zipFile.writepy(path)
  File "/usr/lib/python2.7/zipfile.py", line 1402, in writepy
    self.write(fname, arcname)
  File "/usr/lib/python2.7/zipfile.py", line 1118, in write
    st = os.stat(filename)
OSError: [Errno 2] No such file or directory: '/venv/lib/python2.7/site-packages/jinja2/asyncsupport.pyc'
```

Is there a reason to use `writepy` on each .py file rather than just naively dumping the entire virtualenv to a zip? It looks like it's trying to compile the file before zipping.


Discussion:

Resolving patch:
diff --git a/src/toil/resource.py b/src/toil/resource.py
--- a/src/toil/resource.py
+++ b/src/toil/resource.py
@@ -28,7 +28,7 @@
 from pydoc import locate
 from tempfile import mkdtemp
 from urllib2 import HTTPError
-from zipfile import ZipFile, PyZipFile
+from zipfile import ZipFile
 
 # Python 3 compatibility imports
 from bd2k.util.retry import retry
@@ -265,8 +265,8 @@ def localPath(self):
 class DirectoryResource(Resource):
     """
     A resource read from a directory on the leader. The URL will point to a ZIP archive of the
-    directory. Only Python script/modules will be included. The directory may be a package but it
-    does not need to be.
+    directory. All files in that directory (and any subdirectories) will be included. The directory
+    may be a package but it does not need to be.
     """
 
     @classmethod
@@ -275,10 +275,20 @@ def _load(cls, path):
         :type path: str
         """
         bytesIO = BytesIO()
-        # PyZipFile compiles .py files on the fly, filters out any non-Python files and
-        # distinguishes between packages and simple directories.
-        with PyZipFile(file=bytesIO, mode='w') as zipFile:
-            zipFile.writepy(path)
+        initfile = os.path.join(path, '__init__.py')
+        if os.path.isfile(initfile):
+            # This is a package directory. To emulate
+            # PyZipFile.writepy's behavior, we need to keep everything
+            # relative to this path's parent directory.
+            rootDir = os.path.dirname(path)
+        else:
+            # This is a simple user script (with possibly a few helper files)
+            rootDir = path
+        with ZipFile(file=bytesIO, mode='w') as zipFile:
+            for dirName, _, fileList in os.walk(path):
+                for fileName in fileList:
+                    fullPath = os.path.join(dirName, fileName)
+                    zipFile.write(fullPath, os.path.relpath(fullPath, rootDir))
         bytesIO.seek(0)
         return bytesIO
 
@@ -297,23 +307,21 @@ def localPath(self):
 class VirtualEnvResource(DirectoryResource):
     """
     A resource read from a virtualenv on the leader. All modules and packages found in the
-    virtualenv's site-packages directory will be included. Any .pth or .egg-link files will be
-    ignored.
+    virtualenv's site-packages directory will be included.
     """
-
     @classmethod
     def _load(cls, path):
-        sitePackages = path
-        assert os.path.basename(sitePackages) == 'site-packages'
+        """
+        :type path: str
+        """
+        assert os.path.basename(path) == 'site-packages'
         bytesIO = BytesIO()
-        with PyZipFile(file=bytesIO, mode='w') as zipFile:
-            # This adds the .py files but omits subdirectories since site-packages is not a package
-            zipFile.writepy(sitePackages)
-            # Now add the missing packages
-            for name in os.listdir(sitePackages):
-                path = os.path.join(sitePackages, name)
-                if os.path.isdir(path) and os.path.isfile(os.path.join(path, '__init__.py')):
-                    zipFile.writepy(path)
+        with ZipFile(file=bytesIO, mode='w') as zipFile:
+            for dirName, _, fileList in os.walk(path):
+                zipFile.write(dirName)
+                for fileName in fileList:
+                    fullPath = os.path.join(dirName, fileName)
+                    zipFile.write(fullPath, os.path.relpath(fullPath, path))
         bytesIO.seek(0)
         return bytesIO
 


Test patch:
diff --git a/src/toil/test/src/resourceTest.py b/src/toil/test/src/resourceTest.py
--- a/src/toil/test/src/resourceTest.py
+++ b/src/toil/test/src/resourceTest.py
@@ -75,7 +75,6 @@ def _testExternal(self, moduleName, pyFiles, virtualenv=False):
         else:
             oldPrefix = None
         try:
-            pycFiles = set(pyFile + 'c' for pyFile in pyFiles)
             for relPath in pyFiles:
                 path = os.path.join(dirPath, relPath)
                 mkdir_p(os.path.dirname(path))
@@ -86,8 +85,8 @@ def _testExternal(self, moduleName, pyFiles, virtualenv=False):
                 userScript = importlib.import_module(moduleName)
                 try:
                     self._test(userScript.__name__,
-                               expectedContents=pycFiles,
-                               allowExtraContents=virtualenv)
+                               expectedContents=pyFiles,
+                               allowExtraContents=True)
                 finally:
                     del userScript
                     while moduleName: