Repository: DataBiosphere/toil

Issue:
Allow toil-debug-job to select jobs by WDL name/display name
If I know that a particular WDL job failed, I should be able to rerun it locally and debug it with `toil debug-job` without knowing its actual Toil job store ID. I should be able to refer to it by a WDL-level identifier.



┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1510)
┆Epic: Improve debugging experience
┆Issue Number: TOIL-1510



Discussion:

Resolving patch:
diff --git a/src/toil/utils/toilDebugJob.py b/src/toil/utils/toilDebugJob.py
--- a/src/toil/utils/toilDebugJob.py
+++ b/src/toil/utils/toilDebugJob.py
@@ -21,6 +21,7 @@
 from toil.jobStores.fileJobStore import FileJobStore
 from toil.statsAndLogging import set_logging_from_options
 from toil.utils.toilDebugFile import printContentsOfJobStore
+from toil.utils.toilStatus import ToilStatus
 from toil.worker import workerScript
 
 logger = logging.getLogger(__name__)
@@ -28,10 +29,10 @@
 
 def main() -> None:
     parser = parser_with_common_options(jobstore_option=True, prog="toil debug-job")
-    parser.add_argument("jobID", type=str, nargs='?', default=None,
-                        help="The job store id of a job within the provided jobstore to run by itself.")
-    parser.add_argument("--printJobInfo", type=str,
-                        help="Dump debugging info about this job ID")
+    parser.add_argument("job", type=str,
+                        help="The job store id or job name of a job within the provided jobstore")
+    parser.add_argument("--printJobInfo", action="store_true",
+                        help="Dump debugging info about the job instead of runnign it")
 
     options = parser.parse_args()
     set_logging_from_options(options)
@@ -42,36 +43,67 @@ def main() -> None:
     # But override its options
     config.setOptions(options)
 
-    did_something = False
+    # Find the job
+
+    if jobStore.job_exists(options.job):
+        # The user asked for a particular job and it exists
+        job_id = options.job
+    else:
+        # Go search by name and fill in job_id
+
+        # TODO: break out job store scan logic so it doesn't need to re-connect
+        # to the job store.
+        status = ToilStatus(options.jobStore)
+        hits = []
+        suggestion = None
+        for job in status.jobsToReport:
+            if options.job in (job.jobName, job.unitName, job.displayName):
+                # Find all the jobs that sort of match
+                hits.append(job)
+            if suggestion is None and job.remainingTryCount == 0:
+                # How about this nice failing job instead?
+                suggestion = job
+        if len(hits) == 0:
+            # No hits
+            if suggestion is None:
+                logger.critical("No job found with ID or name \"%s\". No jobs are completely failed.", options.job)
+            else:
+                logger.critical("No job found with ID or name \"%s\". How about the failed job %s instead?", options.job, suggestion)
+            sys.exit(1)
+        elif len(hits) > 1:
+            # Several hits, maybe only one has failed
+            completely_failed_hits = [job for job in hits if job.remainingTryCount == 0]
+            if len(completely_failed_hits) == 0:
+                logger.critical("Multiple jobs match \"%s\" but none are completely failed: %s", options.job, hits)
+                sys.exit(1)
+            elif len(completely_failed_hits) > 0:
+                logger.critical("Multiple jobs matching \"%s\" are completely failed: %s", options.job, completely_failed_hits)
+                sys.exit(1)
+            else:
+                # We found one completely failed job, they probably mean that one.
+                logger.info("There are %s jobs matching \"%s\"; assuming you mean the failed one: %s", options.job, completely_failed_hits[0])
+                job_id = completely_failed_hits[0].jobStoreID
+        else:
+            # We found one job with this name, so they must mean that one
+            logger.info("Looked up job named \"%s\": %s", options.job, hits[0])
+            job_id = hits[0].jobStoreID
 
     if options.printJobInfo:
+        # Report on the job
+
         if isinstance(jobStore, FileJobStore):
             # List all its files if we can
-            printContentsOfJobStore(job_store=jobStore, job_id=options.printJobInfo)
+            printContentsOfJobStore(job_store=jobStore, job_id=job_id)
         # Print the job description itself
-        job_desc = jobStore.load_job(options.printJobInfo)
+        job_desc = jobStore.load_job(job_id)
         print(f"Job: {job_desc}")
         pprint.pprint(job_desc.__dict__)
-
-        did_something = True
+    else:
+        # Run the job
+        logger.debug(f"Running the following job locally: {job_id}")
+        workerScript(jobStore, config, job_id, job_id, redirectOutputToLogFile=False)
+        logger.debug(f"Finished running: {job_id}")
 
     # TODO: Option to print list of successor jobs
     # TODO: Option to run job within python debugger, allowing step through of arguments
     # idea would be to have option to import pdb and set breakpoint at the start of the user's code
-
-    if options.jobID is not None:
-        # We actually want to run a job.
-
-        jobID = options.jobID
-        logger.debug(f"Running the following job locally: {jobID}")
-        workerScript(jobStore, config, jobID, jobID, redirectOutputToLogFile=False)
-        logger.debug(f"Finished running: {jobID}")
-        # Even if the job fails, the worker script succeeds unless something goes wrong with it internally.
-
-        did_something = True
-
-    if not did_something:
-        # Somebody forgot to tell us to do anything.
-        # Show the usage instructions.
-        parser.print_help()
-        sys.exit(1)
diff --git a/src/toil/utils/toilLaunchCluster.py b/src/toil/utils/toilLaunchCluster.py
--- a/src/toil/utils/toilLaunchCluster.py
+++ b/src/toil/utils/toilLaunchCluster.py
@@ -31,7 +31,11 @@
 def create_tags_dict(tags: List[str]) -> Dict[str, str]:
     tags_dict = dict()
     for tag in tags:
-        key, value = tag.split('=')
+        try:
+            key, value = tag.split('=')
+        except ValueError:
+            logger.error("Tag specification '%s' must contain '='", tag)
+            raise
         tags_dict[key] = value
     return tags_dict
 
diff --git a/src/toil/utils/toilStatus.py b/src/toil/utils/toilStatus.py
--- a/src/toil/utils/toilStatus.py
+++ b/src/toil/utils/toilStatus.py
@@ -94,65 +94,99 @@ def printJobChildren(self) -> None:
                 children += "\t(CHILD_JOB:%s,PRECEDENCE:%i)" % (childJob, level)
             print(children)
 
-    def printAggregateJobStats(self, properties: List[str], childNumber: int) -> None:
-        """Prints a job's ID, log file, remaining tries, and other properties."""
-        for job in self.jobsToReport:
+    def printAggregateJobStats(self, properties: List[Set[str]], childNumber: List[int]) -> None:
+        """
+        Prints each job's ID, log file, remaining tries, and other properties.
+
+        :param properties: A set of string flag names for each job in self.jobsToReport.
+        :param childNumber: A list of child counts for each job in self.jobsToReport.
+        """
+        for job, job_properties, job_child_number in zip(self.jobsToReport, properties, childNumber):
 
             def lf(x: str) -> str:
-                return f"{x}:{str(x in properties)}"
-            print("\t".join(("JOB:%s" % job,
-                             "LOG_FILE:%s" % job.logJobStoreFileID,
-                             "TRYS_REMAINING:%i" % job.remainingTryCount,
-                             "CHILD_NUMBER:%s" % childNumber,
-                             lf("READY_TO_RUN"), lf("IS_ZOMBIE"),
-                             lf("HAS_SERVICES"), lf("IS_SERVICE"))))
+                return f"{x}:{str(x in job_properties)}"
+            # We use a sort of not-really-machine-readable key:value TSV format here.
+            # But we only include important keys to help the humans, and flags
+            # don't have a value, just a key.
+            parts = [f"JOB:{job}"]
+            for flag in ["COMPLETELY_FAILED", "READY_TO_RUN", "IS_ZOMBIE", "HAS_SERVICES", "IS_SERVICE"]:
+                if flag in job_properties:
+                    parts.append(flag)
+            if job.logJobStoreFileID:
+                parts.append(f"LOG_FILE:{job.logJobStoreFileID}")
+            if job.remainingTryCount > 0:
+                parts.append(f"TRYS_REMAINING:{job.remainingTryCount}")
+            if job_child_number > 0:
+                parts.append(f"CHILD_NUMBER:{job_child_number}")
+
+            print("\t".join(parts))
 
     def report_on_jobs(self) -> Dict[str, Any]:
         """
         Gathers information about jobs such as its child jobs and status.
 
-        :returns jobStats: Pairings of a useful category and a list of jobs which fall into it.
-        :rtype dict:
+        :returns jobStats: Dict containing some lists of jobs by category, and
+            some lists of job properties for each job in self.jobsToReport.
         """
+        # These are lists of the matching jobs
         hasChildren = []
         readyToRun = []
         zombies = []
         hasLogFile: List[JobDescription] = []
         hasServices = []
         services: List[ServiceJobDescription] = []
-        properties = set()
+        completely_failed = []
+
+        # These are stats for jobs in self.jobsToReport
+        child_number: List[int] = []
+        properties: List[Set[str]] = []
+
+        # TODO: This mix of semantics is confusing and made per-job status be
+        # wrong for multiple years because it was not understood. Redesign it!
 
         for job in self.jobsToReport:
+            job_properties: Set[str] = set()
             if job.logJobStoreFileID is not None:
                 hasLogFile.append(job)
 
-            childNumber = len(list(job.allSuccessors()))
-            if childNumber > 0:  # Total number of successors > 0
+            job_child_number = len(list(job.allSuccessors()))
+            child_number.append(job_child_number)
+            if job_child_number > 0:  # Total number of successors > 0
                 hasChildren.append(job)
-                properties.add("HAS_CHILDREN")
+                job_properties.add("HAS_CHILDREN")
             elif job.command is not None:
                 # Job has no children and a command to run. Indicates job could be run.
                 readyToRun.append(job)
-                properties.add("READY_TO_RUN")
+                job_properties.add("READY_TO_RUN")
             else:
                 # Job has no successors and no command, so is a zombie job.
                 zombies.append(job)
-                properties.add("IS_ZOMBIE")
+                job_properties.add("IS_ZOMBIE")
             if job.services:
                 hasServices.append(job)
-                properties.add("HAS_SERVICES")
+                job_properties.add("HAS_SERVICES")
             if isinstance(job, ServiceJobDescription):
                 services.append(job)
-                properties.add("IS_SERVICE")
-
-        jobStats = {'hasChildren': hasChildren,
-                    'readyToRun': readyToRun,
-                    'zombies': zombies,
-                    'hasServices': hasServices,
-                    'services': services,
-                    'hasLogFile': hasLogFile,
-                    'properties': properties,
-                    'childNumber': childNumber}
+                job_properties.add("IS_SERVICE")
+            if job.remainingTryCount == 0:
+                # Job is out of tries (and thus completely failed)
+                job_properties.add("COMPLETELY_FAILED")
+                completely_failed.append(job)
+            properties.append(job_properties)
+
+        jobStats = {
+            # These are lists of the mathcing jobs
+            'hasChildren': hasChildren,
+            'readyToRun': readyToRun,
+            'zombies': zombies,
+            'hasServices': hasServices,
+            'services': services,
+            'hasLogFile': hasLogFile,
+            'completelyFailed': completely_failed,
+            # These are stats for jobs in self.jobsToReport
+            'properties': properties,
+            'childNumber': child_number
+        }
         return jobStats
 
     @staticmethod
@@ -326,7 +360,7 @@ def main() -> None:
                         help="Do not print overall, aggregate status of workflow.",
                         default=True)
 
-    parser.add_argument("--printDot", action="store_true",
+    parser.add_argument("--dot", "--printDot", dest="print_dot", action="store_true",
                         help="Print dot formatted description of the graph. If using --jobs will "
                              "restrict to subgraph including only those jobs. default=%(default)s",
                         default=False)
@@ -335,20 +369,24 @@ def main() -> None:
                         help="Restrict reporting to the following jobs (allows subsetting of the report).",
                         default=None)
 
-    parser.add_argument("--printPerJobStats", action="store_true",
+    parser.add_argument("--perJob", "--printPerJobStats", dest="print_per_job_stats", action="store_true",
                         help="Print info about each job. default=%(default)s",
                         default=False)
 
-    parser.add_argument("--printLogs", action="store_true",
+    parser.add_argument("--logs", "--printLogs", dest="print_logs", action="store_true",
                         help="Print the log files of jobs (if they exist). default=%(default)s",
                         default=False)
 
-    parser.add_argument("--printChildren", action="store_true",
+    parser.add_argument("--children", "--printChildren", dest="print_children", action="store_true",
                         help="Print children of each job. default=%(default)s",
                         default=False)
 
-    parser.add_argument("--printStatus", action="store_true",
+    parser.add_argument("--status", "--printStatus", dest="print_status", action="store_true",
                         help="Determine which jobs are currently running and the associated batch system ID")
+
+    parser.add_argument("--failed", "--printFailed", dest="print_failed", action="store_true",
+                        help="List jobs which seem to have failed to run")
+
     options = parser.parse_args()
     set_logging_from_options(options)
 
@@ -356,11 +394,8 @@ def main() -> None:
         parser.print_help()
         sys.exit(0)
 
-    config = Config()
-    config.setOptions(options)
-
     try:
-        status = ToilStatus(config.jobStore, options.jobs)
+        status = ToilStatus(options.jobStore, options.jobs)
     except NoSuchJobStoreException:
         print('No job store found.')
         return
@@ -370,34 +405,44 @@ def main() -> None:
     jobStats = status.report_on_jobs()
 
     # Info to be reported.
+    # These are lists of matching jobs.
     hasChildren = jobStats['hasChildren']
     readyToRun = jobStats['readyToRun']
     zombies = jobStats['zombies']
     hasServices = jobStats['hasServices']
     services = jobStats['services']
     hasLogFile = jobStats['hasLogFile']
+    completely_failed = jobStats['completelyFailed']
+    # These are results for corresponding jobs in status.jobsToReport
     properties = jobStats['properties']
     childNumber = jobStats['childNumber']
 
-    if options.printPerJobStats:
+    if options.print_per_job_stats:
         status.printAggregateJobStats(properties, childNumber)
-    if options.printLogs:
+    if options.print_logs:
         status.printJobLog()
-    if options.printChildren:
+    if options.print_children:
         status.printJobChildren()
-    if options.printDot:
+    if options.print_dot:
         status.print_dot_chart()
+    if options.print_failed:
+        print("Failed jobs:")
+        for job in completely_failed:
+            print(job)
     if options.stats:
         print('Of the %i jobs considered, '
-              'there are %i jobs with children, '
+              'there are '
+              '%i completely failed jobs, '
+              '%i jobs with children, '
               '%i jobs ready to run, '
               '%i zombie jobs, '
               '%i jobs with services, '
               '%i services, '
               'and %i jobs with log files currently in %s.' %
-              (len(status.jobsToReport), len(hasChildren), len(readyToRun), len(zombies),
-               len(hasServices), len(services), len(hasLogFile), status.jobStore))
-    if options.printStatus:
+              (len(status.jobsToReport), len(completely_failed), len(hasChildren),
+               len(readyToRun), len(zombies), len(hasServices), len(services),
+               len(hasLogFile), status.jobStore))
+    if options.print_status:
         status.print_bus_messages()
     if len(status.jobsToReport) > 0 and options.failIfNotComplete:
         # Upon workflow completion, all jobs will have been removed from job store


Test patch:
diff --git a/src/toil/test/docs/scripts/example_alwaysfail.py b/src/toil/test/docs/scripts/example_alwaysfail.py
--- a/src/toil/test/docs/scripts/example_alwaysfail.py
+++ b/src/toil/test/docs/scripts/example_alwaysfail.py
@@ -16,7 +16,7 @@ def main():
 
     Then you can inspect the job store with tools like `toil status`:
 
-        toil status --printLogs ./jobstore
+        toil status --logs ./jobstore
 
     """
     parser = ArgumentParser(description=main.__doc__)