Repository: DataBiosphere/toil

Issue:
Workaround pip issue 4059
Symptom:

http://jenkins.cgcloud.info/job/toil-pull-requests/2029/testReport/src.toil.test.src.resourceTest/ResourceTest/testVirtualEnv/

Upstream ticket:

https://github.com/pypa/pip/issues/4059

We should prevent the latest pip from being installed in the virtualenv.
Workaround pip issue 4059
Symptom:

http://jenkins.cgcloud.info/job/toil-pull-requests/2029/testReport/src.toil.test.src.resourceTest/ResourceTest/testVirtualEnv/

Upstream ticket:

https://github.com/pypa/pip/issues/4059

We should prevent the latest pip from being installed in the virtualenv.


Discussion:

Resolving patch:
diff --git a/docker/Dockerfile.py b/docker/Dockerfile.py
--- a/docker/Dockerfile.py
+++ b/docker/Dockerfile.py
@@ -28,7 +28,8 @@
     # Include virtualenv, as it is still the recommended way to deploy pipelines
     RUN pip install virtualenv==15.0.3
 
-    RUN virtualenv /home/s3am
+    # --never-download prevents silent upgrades to pip, wheel and setuptools
+    RUN virtualenv --never-download /home/s3am
 
     RUN /home/s3am/bin/pip install s3am==2.0
 


Test patch:
diff --git a/src/toil/test/provisioners/aws/awsProvisionerTest.py b/src/toil/test/provisioners/aws/awsProvisionerTest.py
--- a/src/toil/test/provisioners/aws/awsProvisionerTest.py
+++ b/src/toil/test/provisioners/aws/awsProvisionerTest.py
@@ -49,13 +49,14 @@ def _test(self, spotInstances=False):
         leader = AWSProvisioner.launchCluster(instanceType=self.instanceType, keyName=self.keyName,
                                               clusterName=self.clusterName)
 
-        venv_command = 'virtualenv --system-site-packages /home/venv'
+        # --never-download prevents silent upgrades to pip, wheel and setuptools
+        venv_command = 'virtualenv --system-site-packages --never-download /home/venv'
         AWSProvisioner._sshAppliance(leader.ip_address, command=venv_command)
 
-        upgrade_command = '/home/venv/bin/pip install setuptools --upgrade'
+        upgrade_command = '/home/venv/bin/pip install setuptools==28.7.1'
         AWSProvisioner._sshAppliance(leader.ip_address, command=upgrade_command)
 
-        yaml_command = '/home/venv/bin/pip install pyyaml'
+        yaml_command = '/home/venv/bin/pip install pyyaml==3.12'
         AWSProvisioner._sshAppliance(leader.ip_address, command=yaml_command)
 
         # install toil scripts
diff --git a/src/toil/test/provisioners/cgcloud/provisionerTest.py b/src/toil/test/provisioners/cgcloud/provisionerTest.py
--- a/src/toil/test/provisioners/cgcloud/provisionerTest.py
+++ b/src/toil/test/provisioners/cgcloud/provisionerTest.py
@@ -107,7 +107,8 @@ def setUpClass(cls):
         os.environ['CGCLOUD_PLUGINS'] = 'cgcloud.toil'
         cls.sdistPath = cls._getSourceDistribution()
         path = cls._createTempDirEx('cgcloud-venv')
-        cls._run('virtualenv', path)
+        # --never-download prevents silent upgrades to pip, wheel and setuptools
+        cls._run('virtualenv', '--never-download', path)
         binPath = os.path.join(path, 'bin')
         cls.oldPath = os.environ['PATH']
         os.environ['PATH'] = os.pathsep.join(concat(binPath, cls.oldPath.split(os.pathsep)))
@@ -154,7 +155,8 @@ def _test(self, autoScaled=False, spotBid=None, preemptableCompensation=None):
                 self._leader('rm', sdistName, admin=True)
             if self.debugEggPath:
                 self._rsync('toil-leader', '-v', self.debugEggPath, ':')
-            self._leader('virtualenv', '--system-site-packages', '~/venv')
+            # --never-download prevents silent upgrades to pip, wheel and setuptools
+            self._leader('virtualenv', '--system-site-packages', '--never-download', '~/venv')
 
             self._getScript()
 
diff --git a/src/toil/test/src/hotDeploymentTest.py b/src/toil/test/src/hotDeploymentTest.py
--- a/src/toil/test/src/hotDeploymentTest.py
+++ b/src/toil/test/src/hotDeploymentTest.py
@@ -26,7 +26,7 @@ def _venvApplianceCluster(self):
             try:
                 leader.runOnAppliance('virtualenv',
                                       '--system-site-packages',
-                                      '--never-download',
+                                      '--never-download',  # prevent silent upgrades to pip etc
                                       'venv')
                 leader.runOnAppliance('venv/bin/pip', 'list') # For diagnostic purposes
                 yield leader, worker
diff --git a/src/toil/test/src/resourceTest.py b/src/toil/test/src/resourceTest.py
--- a/src/toil/test/src/resourceTest.py
+++ b/src/toil/test/src/resourceTest.py
@@ -67,7 +67,8 @@ def _testExternal(self, moduleName, pyFiles, virtualenv=False):
         dirPath = self._createTempDir()
         if virtualenv:
             self.assertTrue(inVirtualEnv())
-            check_call(['virtualenv', dirPath])
+            # --never-download prevents silent upgrades to pip, wheel and setuptools
+            check_call(['virtualenv', '--never-download', dirPath])
             sitePackages = os.path.join(dirPath, 'lib', 'python2.7', 'site-packages')
             # tuple assignment is necessary to make this line immediately precede the try:
             oldPrefix, sys.prefix, dirPath = sys.prefix, dirPath, sitePackages