internlm/SWE-Fixer-Train-110K huggingface dataset:
Repository: DataBiosphere/toil

Issue:
Sort test has slowed down
Runtime doubled to 13min. It started with my first stab at #274 (#275) and so I thought it was caused by it. Since then I made the existence check faster and more reliable (#283) but that didn't seem to have an effect. 

I did notice some "SDB is not available errors" today so maybe we should wait a bit.



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/awsJobStore.py b/src/toil/jobStores/awsJobStore.py
--- a/src/toil/jobStores/awsJobStore.py
+++ b/src/toil/jobStores/awsJobStore.py
@@ -34,7 +34,7 @@
 from boto.sdb.connection import SDBConnection
 from boto.sdb.item import Item
 import boto.s3
-from boto.exception import SDBResponseError, S3ResponseError
+from boto.exception import SDBResponseError, S3ResponseError, BotoServerError
 import itertools
 import time
 
@@ -245,7 +245,7 @@ def writeSharedFileStream(self, sharedFileName, isProtected=True):
         jobStoreFileID = self._newFileID(sharedFileName)
         oldVersion = self._getFileVersion(jobStoreFileID)
         with self._uploadStream(jobStoreFileID, self.files,
-                                encrypted=isProtected) as ( writable, key):
+                                encrypted=isProtected) as (writable, key):
             yield writable
         newVersion = key.version_id
         jobStoreId = str(self.sharedFileJobID) if oldVersion is None else None
@@ -424,7 +424,15 @@ def _getOrCreateDomain(self, domain_name):
 
         :rtype : Domain
         """
-        return self.db.create_domain(domain_name)
+        try:
+            return self.db.get_domain(domain_name)
+        except SDBResponseError as e:
+            if no_such_domain(e):
+                for attempt in retry_sdb(retry_while=sdb_unavailable):
+                    with attempt:
+                        return self.db.create_domain(domain_name)
+            else:
+                raise
 
     def _newJobID(self):
         return str(uuid.uuid4())
@@ -724,7 +732,11 @@ def toItem(self, parentJobStoreID=None):
 
 
 def no_such_domain(e):
-    return e.error_code.endswith('NoSuchDomain')
+    return isinstance(e, SDBResponseError) and e.error_code.endswith('NoSuchDomain')
+
+
+def sdb_unavailable(e):
+    return e.__class__ == BotoServerError and e.status.startswith("503")
 
 
 def true(_):
@@ -806,7 +818,7 @@ def retry_sdb(retry_after=a_short_time,
         def repeated_attempt():
             try:
                 yield
-            except SDBResponseError as e:
+            except BotoServerError as e:
                 if time.time() + retry_after < expiration:
                     if retry_while(e):
                         log.info('... got %s, trying again in %is ...', e.error_code, retry_after)


Test patch:
Repository: DataBiosphere/toil

Issue:
Adjust mesos logging



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/mesos/batchSystem.py b/src/toil/batchSystems/mesos/batchSystem.py
--- a/src/toil/batchSystems/mesos/batchSystem.py
+++ b/src/toil/batchSystems/mesos/batchSystem.py
@@ -274,7 +274,7 @@ def _sortJobsByResourceReq(self):
 
     def _declineAllOffers(self, driver, offers):
         for offer in offers:
-            log.warning("No jobs to assign. Rejecting offer".format(offer.id.value))
+            log.debug("No jobs to assign. Rejecting offer".format(offer.id.value))
             driver.declineOffer(offer.id)
 
     def _determineOfferResources(self, offer):
@@ -359,7 +359,8 @@ def resourceOffers(self, driver, offers):
                 log.info("...launching Mesos task %s" % task.task_id.value)
 
             if len(tasks) == 0:
-                log.info("Offer not large enough to run any tasks. Required: %s Offered: %s" % (job_types[-1], (offerMem*1000000, offerCores, offerStor*1000000)))
+                log.debug("Offer %s not large enough to run any tasks. Required: %s Offered: %s"
+                          % (offer.id.value, job_types[-1], (offerMem*1000000, offerCores, offerStor*1000000)))
 
     def _createTask(self, jt_job, offer):
         """
@@ -435,7 +436,7 @@ def frameworkMessage(self, driver, executorId, slaveId, message):
         """
         Invoked when an executor sends a message.
         """
-        log.info("Executor {} on slave {} sent a message: {}".format(executorId, slaveId, message))
+        log.debug("Executor {} on slave {} sent a message: {}".format(executorId, slaveId, message))
 
     def __reconcile(self, driver):
         """


Test patch:
Repository: DataBiosphere/toil

Issue:
Check mesos has minimum disk/mem resources
If you specify disk/memory less than 1mb, mesos will not run the task. We should check this early on.



Discussion:
Round up to the minimum amount required and print a warning message


Resolving patch:
diff --git a/src/toil/batchSystems/mesos/batchSystem.py b/src/toil/batchSystems/mesos/batchSystem.py
--- a/src/toil/batchSystems/mesos/batchSystem.py
+++ b/src/toil/batchSystems/mesos/batchSystem.py
@@ -360,7 +360,7 @@ def resourceOffers(self, driver, offers):
 
             if len(tasks) == 0:
                 log.debug("Offer %s not large enough to run any tasks. Required: %s Offered: %s"
-                          % (offer.id.value, job_types[-1], (offerMem*1000000, offerCores, offerStor*1000000)))
+                          % (offer.id.value, job_types[-1], (self.__mbToBytes(offerMem), offerCores, self.__mbToBytes(offerStor))))
 
     def _createTask(self, jt_job, offer):
         """
@@ -384,12 +384,21 @@ def _createTask(self, jt_job, offer):
         disk = task.resources.add()
         disk.name = "disk"
         disk.type = mesos_pb2.Value.SCALAR
-        disk.scalar.value = jt_job.resources.disk / 1000000
-
+        if self.__bytesToMB(jt_job.resources.disk) > 1:
+            disk.scalar.value = self.__bytesToMB(jt_job.resources.disk)
+        else:
+            log.warning("Job %s uses less disk than mesos requires. Rounding %s bytes up to 1 mb" %
+                        (jt_job.jobID, jt_job.resources.disk))
+            disk.scalar.value = 1
         mem = task.resources.add()
         mem.name = "mem"
         mem.type = mesos_pb2.Value.SCALAR
-        mem.scalar.value = jt_job.resources.memory / 1000000
+        if self.__bytesToMB(jt_job.resources.memory) > 1:
+            mem.scalar.value = self.__bytesToMB(jt_job.resources.memory)
+        else:
+            log.warning("Job %s uses less memory than mesos requires. Rounding %s bytes up to 1 mb" %
+                        (jt_job.jobID, jt_job.resources.memory))
+            mem.scalar.value = 1
         return task
 
     def __updateState(self, intID, exitStatus):
@@ -467,3 +476,10 @@ def __bytesToMB(mem):
         used when converting toil reqs to Mesos reqs
         """
         return mem / 1024 / 1024
+
+    @staticmethod
+    def __mbToBytes(mem):
+        """
+        used when converting Mesos reqs to Toil reqs
+        """
+        return mem * 1024 * 1024


Test patch:
Repository: DataBiosphere/toil

Issue:
"make test" breaks if Mesos isn't installed
Reported by @joelarmstrong in #296.



Discussion:
I think we need to use the technique from #296 in order to exclude certain classes from test discovery.


Resolving patch:
diff --git a/src/toil/batchSystems/mesos/conftest.py b/src/toil/batchSystems/mesos/conftest.py
new file mode 100644
--- /dev/null
+++ b/src/toil/batchSystems/mesos/conftest.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2015 UCSC Computational Genomics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+collect_ignore = []
+
+try:
+    import mesos.interface
+except ImportError:
+    collect_ignore.append("batchSystem.py")
+    collect_ignore.append("executor.py")


Test patch:
Repository: DataBiosphere/toil

Issue:
User scripts don't work if they are named test.py



Discussion:
The original idea was to prepend the module's containing directory to the sys.path. But this would break importing the builtin module. Instead, add a test that checks early on whether importing a user script would succeed.

Might be hard to solve this generally. CJ proposes to catch the worker exception and add a hint to the user.


Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -667,7 +667,12 @@ def _loadClass(className, userModule):
         userModule = importlib.import_module(userModule.name)
         thisModule = sys.modules[__name__]
         #TODO: Document what this magic is doing
-        thisModule.__dict__[className] = userModule.__dict__[className]
+        try:
+            thisModule.__dict__[className] = userModule.__dict__[className]
+        except KeyError:
+            logger.error("Error retrieving user module. Confirm that the script's name is not also "
+                         "a standard python module")
+            raise
     
     @staticmethod
     def _loadJob(command, jobStore):
@@ -941,7 +946,12 @@ def _getUserFunction(self):
         if userFunctionModule.dirPath not in sys.path:
             # FIXME: prepending to sys.path will probably fix #103
             sys.path.append(userFunctionModule.dirPath)
-        return getattr(importlib.import_module(userFunctionModule.name), self.userFunctionName)
+        try:
+            return getattr(importlib.import_module(userFunctionModule.name), self.userFunctionName)
+        except AttributeError:
+            logger.error("Error retrieving user module. Confirm that the script's name is not also "
+                         "a standard python module")
+            raise
 
     def run(self,fileStore):
         userFunction = self._getUserFunction( )


Test patch:
Repository: DataBiosphere/toil

Issue:
Toil directory deletes itself upon ctrl+c (keyboard interrupt)



Discussion:

Resolving patch:
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -490,7 +490,12 @@ def mainLoop(config, batchSystem, jobStore, rootJob):
 
         logger.info("Finished the main loop")
 
-
+    except:
+        if config.clean == "onError":
+            jobStore.deleteJobStore()
+    else:
+        if config.clean == "onSuccess" and totalFailedJobs == 0:
+            jobStore.deleteJobStore()
     finally:
         ##########################################
         #Finish up the stats/logging aggregation process
@@ -500,8 +505,8 @@ def mainLoop(config, batchSystem, jobStore, rootJob):
         stopStatsAndLoggingAggregatorProcess.put(True)
         worker.join()
         logger.info("Stats/logging finished collating in %s seconds", time.time() - startTime)
-        clean = config.clean
-        if clean=="always" or clean == "onError" and totalFailedJobs>0 or clean == "onSuccess" and totalFailedJobs==0:
+        # in addition to cleaning on exceptions, onError should clean if there are any failed jobs
+        if config.clean == "always" or config.clean == "onError" and totalFailedJobs > 0:
             jobStore.deleteJobStore()
     
     if totalFailedJobs > 0:


Test patch:
Repository: DataBiosphere/toil

Issue:
Azure doesn't read stats files correctly
I completely misread the docstring and was passing strings to the stats callback rather than file handles (causing failure on #384).



Discussion:

Resolving patch:
diff --git a/src/toil/jobStores/azureJobStore.py b/src/toil/jobStores/azureJobStore.py
--- a/src/toil/jobStores/azureJobStore.py
+++ b/src/toil/jobStores/azureJobStore.py
@@ -264,8 +264,8 @@ def readStatsAndLogging(self, statsAndLoggingCallbackFn):
         numStatsFiles = 0
         for entity in self.statsFileIDs.query_entities():
             jobStoreFileID = entity.RowKey
-            string = self.statsFiles.get_blob_to_text(blob_name=jobStoreFileID)
-            statsAndLoggingCallbackFn(string)
+            with self._downloadStream(jobStoreFileID, self.statsFiles, encrypted=False) as fd:
+                statsAndLoggingCallbackFn(fd)
             self.statsFiles.delete_blob(blob_name=jobStoreFileID)
             self.statsFileIDs.delete_entity(row_key=jobStoreFileID)
             numStatsFiles += 1
@@ -425,10 +425,10 @@ def _downloadStream(self, jobStoreFileID, container, encrypted=False):
                 def writer():
                     try:
                         chunkStartPos = 0
-                        fileSize = int(self.files.get_blob_properties(blob_name=jobStoreFileID)['Content-Length'])
+                        fileSize = int(container.get_blob_properties(blob_name=jobStoreFileID)['Content-Length'])
                         while chunkStartPos < fileSize:
                             chunkEndPos = chunkStartPos + self._maxAzureBlockBytes - 1
-                            buf = self.files.get_blob(blob_name=jobStoreFileID,
+                            buf = container.get_blob(blob_name=jobStoreFileID,
                                                       x_ms_range="bytes=%d-%d" % (chunkStartPos,
                                                                                   chunkEndPos))
                             if encrypted:


Test patch:
Repository: DataBiosphere/toil

Issue:
addToilOptions runtime error says it accepts OptionParser



Discussion:
Alden, unless you plan on fixing this yourself, this would need a little more information on how to reproduce the problem.

In common.addOptions, a runtime error is thrown if anything other than an ArgumentParser is passed, but the error says it needs either an OptionParser or an ArgumentParser. I can take the issue, but I'm not sure whether we're eliminating support for OptionParser or not. 

@cket already eliminated optparse. Assigning to him.


Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -277,7 +277,7 @@ def addGroup(headingString, bodyString):
         _addOptions(addGroup, config)
     else:
         raise RuntimeError("Unanticipated class passed to addOptions(), %s. Expecting "
-                           "Either optparse.OptionParser or argparse.ArgumentParser" % parser.__class__)
+                           "argparse.ArgumentParser" % parser.__class__)
 
 def loadBatchSystem(config):
     """
diff --git a/src/toil/lib/bioio.py b/src/toil/lib/bioio.py
--- a/src/toil/lib/bioio.py
+++ b/src/toil/lib/bioio.py
@@ -25,7 +25,7 @@
 import math
 import shutil
 from argparse import ArgumentParser
-from optparse import OptionParser, OptionContainer, OptionGroup
+from optparse import OptionContainer, OptionGroup
 import subprocess
 import xml.etree.cElementTree as ET
 from xml.dom import minidom  # For making stuff pretty
@@ -93,19 +93,13 @@ def logStream(fileHandle, shortName, printFunction=logger.info):
 def addLoggingOptions(parser):
     # Wrapper function that allows toil to be used with both the optparse and
     # argparse option parsing modules
-    if isinstance(parser, OptionContainer):
-        group = OptionGroup(parser, "Logging options",
-                            "Options that control logging")
-        _addLoggingOptions(group.add_option)
-        parser.add_option_group(group)
-    elif isinstance(parser, ArgumentParser):
+    if isinstance(parser, ArgumentParser):
         group = parser.add_argument_group("Logging Options",
                                           "Options that control logging")
         _addLoggingOptions(group.add_argument)
     else:
         raise RuntimeError("Unanticipated class passed to "
                            "addLoggingOptions(), %s. Expecting "
-                           "Either optparse.OptionParser or "
                            "argparse.ArgumentParser" % parser.__class__)
 
 supportedLogLevels = (logging.CRITICAL, logging.ERROR, logging.WARNING, logging.INFO, logging.DEBUG)


Test patch:
Repository: DataBiosphere/toil

Issue:
Port GridEngine terabyte fix from jobTree
https://github.com/benedictpaten/jobTree/pull/41

Port GridEngine terabyte fix from jobTree
https://github.com/benedictpaten/jobTree/pull/41



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/gridengine.py b/src/toil/batchSystems/gridengine.py
--- a/src/toil/batchSystems/gridengine.py
+++ b/src/toil/batchSystems/gridengine.py
@@ -31,7 +31,7 @@
 
 class MemoryString:
     def __init__(self, string):
-        if string[-1] == 'K' or string[-1] == 'M' or string[-1] == 'G':
+        if string[-1] == 'K' or string[-1] == 'M' or string[-1] == 'G' or string[-1] == 'T':
             self.unit = string[-1]
             self.val = float(string[:-1])
         else:
@@ -54,6 +54,8 @@ def byteVal(self):
             return self.val * 1048576
         elif self.unit == 'G':
             return self.val * 1073741824
+        elif self.unit == 'T':
+            return self.val * 1099511627776
 
     def __cmp__(self, other):
         return cmp(self.bytes, other.bytes)
@@ -76,7 +78,8 @@ def prepareQsub(cpu, mem, jobID):
     if len(reqline) > 0:
         qsubline.extend(["-hard", "-l", ",".join(reqline)])
     if cpu is not None and math.ceil(cpu) > 1:
-        qsubline.extend(["-pe", "smp", str(int(math.ceil(cpu)))])
+        peConfig = os.getenv('TOIL_GRIDENGINE_PE') or "shm"
+        qsubline.extend(["-pe", peConfig, str(int(math.ceil(cpu)))])
     return qsubline
 
 


Test patch:
Repository: DataBiosphere/toil

Issue:
resolveEntryPoint() fails on a system-wide installation of Toil
Python is installed in `/usr/bin` while pip installed the entry points in `/usr/local/bin`. That's why the lookup logic does not work.

resolveEntryPoint() fails on a system-wide installation of Toil
Python is installed in `/usr/bin` while pip installed the entry points in `/usr/local/bin`. That's why the lookup logic does not work.



Discussion:

Resolving patch:
diff --git a/src/toil/__init__.py b/src/toil/__init__.py
--- a/src/toil/__init__.py
+++ b/src/toil/__init__.py
@@ -27,6 +27,23 @@ def toilPackageDirPath():
     assert result.endswith('/toil')
     return result
 
-def resolveEntryPoint(entryPoint):
-    return os.path.join(os.path.dirname(sys.executable),entryPoint)
 
+def resolveEntryPoint(entryPoint):
+    """
+    Returns the path to the given entry point (see setup.py) that *should* work on a worker. The
+    return value may be an absolute or a relative path.
+    """
+    if hasattr(sys, 'real_prefix'):
+        # Inside a virtualenv we will use absolute paths to the entrypoints. For clusters this
+        # means that if Toil is installed in a virtualenv on the leader, it must be installed in
+        # a virtualenv located at the same path on the worker.
+        path = os.path.join(os.path.dirname(sys.executable), entryPoint)
+        assert os.path.isfile(path)
+        assert os.access(path, os.X_OK)
+        return path
+    else:
+        # Outside a virtualenv it is hard to predict where the entry points got installed. It is
+        # the reponsibility of the user to ensure that they are present on PATH and point to the
+        # correct version of Toil. This is still better than an absolute path because it gives
+        # the user control over Toil's location on both leader and workers.
+        return entryPoint


Test patch:
Repository: DataBiosphere/toil

Issue:
Toil status throws NoSuchJobException



Discussion:

Resolving patch:
diff --git a/src/toil/utils/toilStatus.py b/src/toil/utils/toilStatus.py
--- a/src/toil/utils/toilStatus.py
+++ b/src/toil/utils/toilStatus.py
@@ -82,10 +82,12 @@ def main():
         sys.exit(0)
     
     toilState = ToilState(jobStore, rootJob )
-    
-    failedJobs = [ job for job in toilState.updatedJobs | \
-                  set(toilState.successorCounts.keys()) \
-                  if job.remainingRetryCount == 0 ]
+
+    # The first element of the toilState.updatedJobs tuple is the jobWrapper we want to inspect
+    totalJobs = set(toilState.successorCounts.keys()) | \
+                {jobTuple[0] for jobTuple in toilState.updatedJobs}
+
+    failedJobs = [ job for job in totalJobs if job.remainingRetryCount == 0 ]
     
     print "There are %i active jobs, %i parent jobs with children, and \
     %i totally failed jobs currently in toil workflow: %s" % \


Test patch:
Repository: DataBiosphere/toil

Issue:
Need `toil --version`

Need `toil --version`



Discussion:

Resolving patch:
diff --git a/src/toil/utils/toilMain.py b/src/toil/utils/toilMain.py
--- a/src/toil/utils/toilMain.py
+++ b/src/toil/utils/toilMain.py
@@ -1,4 +1,6 @@
 from __future__ import absolute_import, print_function
+from toil.version import version
+import pkg_resources
 import os
 import sys
 
@@ -11,6 +13,11 @@ def main():
     else:
         if command == '--help':
             printHelp(modules)
+        elif command == '--version':
+            try:
+                print(pkg_resources.get_distribution('toil').version)
+            except:
+                print("Version gathered from toil.version: "+version)
         else:
             try:
                 module = modules[command]


Test patch:
Repository: DataBiosphere/toil

Issue:
CWL test fails with "Unsupported requirement InlineJavascriptRequirement"
http://jenkins.cgcloud.info/job/toil-pull-requests/653/testReport/junit/src.toil.test.cwl.cwlTest/CWLTest/test_run_revsort/

CWL test fails with "Unsupported requirement InlineJavascriptRequirement"
http://jenkins.cgcloud.info/job/toil-pull-requests/653/testReport/junit/src.toil.test.cwl.cwlTest/CWLTest/test_run_revsort/



Discussion:

Resolving patch:
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -39,7 +39,7 @@
         'encryption': [
             'pynacl==0.3.0'],
         'cwl': [
-            'cwltool==1.0.20151112194920']},
+            'cwltool==1.0.20151122025918']},
     package_dir={'': 'src'},
     packages=find_packages('src', exclude=['*.test']),
     entry_points={
diff --git a/src/toil/cwl/cwltoil.py b/src/toil/cwl/cwltoil.py
--- a/src/toil/cwl/cwltoil.py
+++ b/src/toil/cwl/cwltoil.py
@@ -344,6 +344,7 @@ def run(self, fileStore):
 
 supportedProcessRequirements = ["DockerRequirement",
                                 "ExpressionEngineRequirement",
+                                "InlineJavascriptRequirement",
                                 "SchemaDefRequirement",
                                 "EnvVarRequirement",
                                 "CreateFileRequirement",


Test patch: