Repository: DataBiosphere/toil

Issue:
Add colored logs to Toil
Toil's logging can be very verbose and large sometimes. Colored logs might help with readability. I took a short glance at how MiniWDL does colored logs and it looks fairly easy to add colors. MiniWDL does make its text bold though, and I'm not sure if Toil should as well.

On a related note, I also believe configargparse is fully compatible with argcomplete, so it's possible to add that too to make using Toil on CLI easier.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-1516)
┆Issue Number: TOIL-1516



Discussion:

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -149,6 +149,7 @@ class Config:
     workflowAttemptNumber: int
     jobStore: str
     logLevel: str
+    colored_logs: bool
     workDir: Optional[str]
     coordination_dir: Optional[str]
     noStdOutErr: bool
@@ -400,6 +401,7 @@ def set_option(option_name: str,
         set_option("badWorker")
         set_option("badWorkerFailInterval")
         set_option("logLevel")
+        set_option("colored_logs")
 
         # Apply overrides as highest priority
         # Override workDir with value of TOIL_WORKDIR_OVERRIDE if it exists
diff --git a/src/toil/statsAndLogging.py b/src/toil/statsAndLogging.py
--- a/src/toil/statsAndLogging.py
+++ b/src/toil/statsAndLogging.py
@@ -22,6 +22,7 @@
 from threading import Event, Thread
 from typing import IO, TYPE_CHECKING, Any, Callable, List, Optional, Union
 
+from toil.lib.conversions import strtobool
 from toil.lib.expando import Expando
 from toil.lib.resources import get_total_cpu_time
 
@@ -39,6 +40,7 @@
 
 class StatsAndLogging:
     """A thread to aggregate statistics and logging."""
+
     def __init__(self, jobStore: 'AbstractJobStore', config: 'Config') -> None:
         self._stop = Event()
         self._worker = Thread(target=self.statsAndLoggingAggregator,
@@ -74,10 +76,10 @@ def formatLogStream(cls, stream: Union[IO[str], IO[bytes]], stream_name: str) ->
 
         return '\n'.join(lines)
 
-
     @classmethod
-    def logWithFormatting(cls, stream_name: str, jobLogs: Union[IO[str], IO[bytes]], method: Callable[[str], None] = logger.debug,
-                            message: Optional[str] = None) -> None:
+    def logWithFormatting(cls, stream_name: str, jobLogs: Union[IO[str], IO[bytes]],
+                          method: Callable[[str], None] = logger.debug,
+                          message: Optional[str] = None) -> None:
         if message is not None:
             method(message)
 
@@ -100,7 +102,7 @@ def createName(logPath: str, jobName: str, logExtension: str, failed: bool = Fal
                 fullName = os.path.join(logPath, logName + suffix)
                 #  The maximum file name size in the default HFS+ file system is 255 UTF-16 encoding units, so basically 255 characters
                 if len(fullName) >= 255:
-                    return fullName[:(255-len(suffix))] + suffix
+                    return fullName[:(255 - len(suffix))] + suffix
                 if not os.path.exists(fullName):
                     return fullName
                 counter += 1
@@ -236,12 +238,40 @@ def set_log_level(level: str, set_logger: Optional[logging.Logger] = None) -> No
     level = "CRITICAL" if level.upper() == "OFF" else level.upper()
     set_logger = set_logger if set_logger else root_logger
     set_logger.setLevel(level)
-
     # Suppress any random loggers introduced by libraries we use.
     # Especially boto/boto3.  They print too much.  -__-
     suppress_exotic_logging(__name__)
 
 
+def install_log_color(set_logger: Optional[logging.Logger] = None) -> None:
+    """Make logs colored."""
+    # Most of this code is taken from miniwdl
+    # delayed import
+    import coloredlogs  # type: ignore[import-untyped]
+
+    level_styles = dict(coloredlogs.DEFAULT_LEVEL_STYLES)
+    level_styles["debug"]["color"] = 242
+    level_styles["notice"] = {"color": "green", "bold": True}
+    level_styles["error"]["bold"] = True
+    level_styles["warning"]["bold"] = True
+    level_styles["info"] = {}
+    field_styles = dict(coloredlogs.DEFAULT_FIELD_STYLES)
+    field_styles["asctime"] = {"color": "blue"}
+    field_styles["name"] = {"color": "magenta"}
+    field_styles["levelname"] = {"color": "blue"}
+    field_styles["threadName"] = {"color": "blue"}
+    fmt = "[%(asctime)s] [%(threadName)s] [%(levelname).1s] [%(name)s] %(message)s"  # mimic old toil logging format
+    set_logger = set_logger if set_logger else root_logger
+    coloredlogs.install(
+        level=set_logger.getEffectiveLevel(),
+        logger=set_logger,
+        level_styles=level_styles,
+        field_styles=field_styles,
+        datefmt="%Y-%m-%dT%H:%M:%S%z",  # mimic old toil date format
+        fmt=fmt,
+    )
+
+
 def add_logging_options(parser: ArgumentParser) -> None:
     """Add logging options to set the global log level."""
     group = parser.add_argument_group("Logging Options")
@@ -262,6 +292,8 @@ def add_logging_options(parser: ArgumentParser) -> None:
     group.add_argument("--logFile", dest="logFile", help="File to log in.")
     group.add_argument("--rotatingLogging", dest="logRotating", action="store_true", default=False,
                        help="Turn on rotating logging, which prevents log files from getting too big.")
+    group.add_argument("--logColors", dest="colored_logs", default=True, type=strtobool, metavar="BOOL",
+                       help="Enable or disable colored logging. Default: %(default)s")
 
 
 def configure_root_logger() -> None:
@@ -292,6 +324,8 @@ def set_logging_from_options(options: Union["Config", Namespace]) -> None:
     configure_root_logger()
     options.logLevel = options.logLevel or logging.getLevelName(root_logger.getEffectiveLevel())
     set_log_level(options.logLevel)
+    if options.colored_logs:
+        install_log_color()
     logger.debug(f"Root logger is at level '{logging.getLevelName(root_logger.getEffectiveLevel())}', "
                  f"'toil' logger at level '{logging.getLevelName(toil_logger.getEffectiveLevel())}'.")
 
diff --git a/src/toil/worker.py b/src/toil/worker.py
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -42,7 +42,7 @@
 from toil.lib.io import make_public_dir
 from toil.lib.resources import (get_total_cpu_time,
                                 get_total_cpu_time_and_memory_usage)
-from toil.statsAndLogging import configure_root_logger, set_log_level
+from toil.statsAndLogging import configure_root_logger, set_log_level, install_log_color
 
 logger = logging.getLogger(__name__)
 
@@ -144,6 +144,9 @@ def workerScript(jobStore: AbstractJobStore, config: Config, jobName: str, jobSt
     configure_root_logger()
     set_log_level(config.logLevel)
 
+    if config.colored_logs:
+        install_log_color()
+
     ##########################################
     #Create the worker killer, if requested
     ##########################################
@@ -518,7 +521,7 @@ def blockFn() -> bool:
     #Trapping where worker goes wrong
     ##########################################
     except BaseException as e: #Case that something goes wrong in worker, or we are asked to stop
-        traceback.print_exc()
+        logger.critical("Worker crashed with traceback:\n%s", traceback.format_exc())
         logger.error("Exiting the worker because of a failed job on host %s", socket.gethostname())
         if isinstance(e, CWL_UNSUPPORTED_REQUIREMENT_EXCEPTION):
             # We need to inform the leader that this is a CWL workflow problem


Test patch:
diff --git a/src/toil/test/src/realtimeLoggerTest.py b/src/toil/test/src/realtimeLoggerTest.py
--- a/src/toil/test/src/realtimeLoggerTest.py
+++ b/src/toil/test/src/realtimeLoggerTest.py
@@ -30,6 +30,9 @@ def testRealtimeLogger(self):
         # Set up a log message detector to the root logger
         logging.getLogger().addHandler(detector)
 
+        # I believe coloredlogs replaces handlers with its own when doing handler formatting, preserving only filters
+        # https://github.com/xolox/python-coloredlogs/blob/65bdfe976ac0bf81e8c0bd9a98242b9d666b2859/coloredlogs/__init__.py#L453-L459
+        options.colored_logs = False
         Job.Runner.startToil(LogTest(), options)
 
         # We need the message we're supposed to see
@@ -46,14 +49,20 @@ class MessageDetector(logging.StreamHandler):
     def __init__(self):
         self.detected = False  # Have we seen the message we want?
         self.overLogged = False  # Have we seen the message we don't want?
+        self.entire_string = ""
+
         super().__init__()
 
     def emit(self, record):
+        self.entire_string += record.msg
         if record.msg == 'This should be logged at info level':
             self.detected = True
         if record.msg == 'This should be logged at debug level':
             self.overLogged = True
 
+    def entire(self) -> str:
+        return self.entire_string
+
 
 class LogTest(Job):
     def __init__(self):