Repository: DataBiosphere/toil

Issue:
Debug logs from Slurm runs are too noisy because they dump full command output
I'm going back and forth with someone trying to get their WDL workflow to work on Slurm. They keep sending me debug logs that are almost unusable because they are swamped with a line of output for every job on the whole cluster, every tine Toil polls jobs. This is because at debug log level we log the entire output of every command we use to interact with the scheduler, even when that output isn't relevant to the problem we are debugging.

We need to move that output to a lower log level than debug, or otherwise not have it in the debug logs.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1336)
┆Issue Number: TOIL-1336



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/slurm.py b/src/toil/batchSystems/slurm.py
--- a/src/toil/batchSystems/slurm.py
+++ b/src/toil/batchSystems/slurm.py
@@ -41,7 +41,7 @@ def getRunningJobIDs(self):
             # -h for no header
             # --format to get jobid i, state %t and time days-hours:minutes:seconds
 
-            lines = call_command(['squeue', '-h', '--format', '%i %t %M']).split('\n')
+            lines = call_command(['squeue', '-h', '--format', '%i %t %M'], quiet=True).split('\n')
             for line in lines:
                 values = line.split()
                 if len(values) < 3:
@@ -165,7 +165,7 @@ def _getJobDetailsFromSacct(self, job_id_list: list) -> dict:
                     '--format', 'JobIDRaw,State,ExitCode',  # specify output columns
                     '-P',  # separate columns with pipes
                     '-S', '1970-01-01']  # override start time limit
-            stdout = call_command(args)
+            stdout = call_command(args, quiet=True)
 
             # Collect the job statuses in a dict; key is the job-id, value is a tuple containing
             # job state and exit status. Initialize dict before processing output of `sacct`.
@@ -174,7 +174,6 @@ def _getJobDetailsFromSacct(self, job_id_list: list) -> dict:
                 job_statuses[job_id] = (None, None)
 
             for line in stdout.splitlines():
-                #logger.debug("%s output %s", args[0], line)
                 values = line.strip().split('|')
                 if len(values) < 3:
                     continue
@@ -210,7 +209,7 @@ def _getJobDetailsFromScontrol(self, job_id_list: list) -> dict:
             if len(job_id_list) == 1:
                 args.append(str(job_id_list[0]))
 
-            stdout = call_command(args)
+            stdout = call_command(args, quiet=True)
 
             # Job records are separated by a blank line.
             if isinstance(stdout, str):
@@ -233,7 +232,6 @@ def _getJobDetailsFromScontrol(self, job_id_list: list) -> dict:
                 job = {}
                 for line in record.splitlines():
                     for item in line.split():
-                        #logger.debug("%s output %s", args[0], item)
                         # Output is in the form of many key=value pairs, multiple pairs on each line
                         # and multiple lines in the output. Each pair is pulled out of each line and
                         # added to a dictionary.
diff --git a/src/toil/lib/misc.py b/src/toil/lib/misc.py
--- a/src/toil/lib/misc.py
+++ b/src/toil/lib/misc.py
@@ -109,14 +109,24 @@ def __str__(self) -> str:
 
 
 def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeout: Optional[float] = None,
-                useCLocale: bool = True, env: Optional[typing.Dict[str, str]] = None) -> str:
-    """Simplified calling of external commands.  This always returns
-    stdout and uses utf- encode8.  If process fails, CalledProcessErrorStderr
-    is raised.  The captured stderr is always printed, regardless of
-    if an expect occurs, so it can be logged.  At the debug log level, the
-    command and captured out are always used.  With useCLocale, C locale
-    is force to prevent failures that occurred in some batch systems
-    with UTF-8 locale.
+                useCLocale: bool = True, env: Optional[typing.Dict[str, str]] = None, quiet: Optional[bool] = False) -> str:
+    """
+    Simplified calling of external commands.
+
+    If the process fails, CalledProcessErrorStderr is raised.
+
+    The captured stderr is always printed, regardless of
+    if an exception occurs, so it can be logged.
+
+    Always logs the command at debug log level.
+
+    :param quiet: If True, do not log the command output. If False (the
+           default), do log the command output at debug log level.
+
+    :param useCLocale: If True, C locale is forced, to prevent failures that
+           can occur in some batch systems when using UTF-8 locale.
+
+    :returns: Command standard output, decoded as utf-8.
     """
 
     # using non-C locales can cause GridEngine commands, maybe other to
@@ -136,5 +146,5 @@ def call_command(cmd: List[str], *args: str, input: Optional[str] = None, timeou
     if proc.returncode != 0:
         logger.debug("command failed in {}s: {}: {}".format(runtime, " ".join(cmd), stderr.rstrip()))
         raise CalledProcessErrorStderr(proc.returncode, cmd, output=stdout, stderr=stderr)
-    logger.debug("command succeeded in {}s: {}: {}".format(runtime, " ".join(cmd), stdout.rstrip()))
+    logger.debug("command succeeded in {}s: {}{}".format(runtime, " ".join(cmd), (': ' + stdout.rstrip()) if not quiet else ''))
     return stdout


Test patch: