# Copyright (c) Alibaba, Inc. and its affiliates. import importlib import logging from typing import Optional init_loggers = {} formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') def get_logger(log_file: Optional[str] = None, log_level: int = logging.INFO, file_mode: str = 'w'): """ Get logging logger Args: log_file: Log filename, if specified, file handler will be added to logger log_level: Logging level. file_mode: Specifies the mode to open the file, if filename is specified (if filemode is unspecified, it defaults to 'w'). """ logger_name = __name__.split('.')[0] logger = logging.getLogger(logger_name) logger.propagate = False if logger_name in init_loggers: add_file_handler_if_needed(logger, log_file, file_mode, log_level) return logger # handle duplicate logs to the console # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler (NOTSET) # to the root logger. As logger.propagate is True by default, this root # level handler causes logging messages from rank>0 processes to # unexpectedly show up on the console, creating much unwanted clutter. # To fix this issue, we set the root logger's StreamHandler, if any, to log # at the ERROR level. torch_dist = False is_worker0 = True if importlib.util.find_spec('torch') is not None: from modelscope.utils.torch_utils import is_dist, is_master torch_dist = is_dist() is_worker0 = is_master() if torch_dist: for handler in logger.root.handlers: if type(handler) is logging.StreamHandler: handler.setLevel(logging.ERROR) stream_handler = logging.StreamHandler() handlers = [stream_handler] if is_worker0 and log_file is not None: file_handler = logging.FileHandler(log_file, file_mode) handlers.append(file_handler) for handler in handlers: handler.setFormatter(formatter) handler.setLevel(log_level) logger.addHandler(handler) if is_worker0: logger.setLevel(log_level) else: logger.setLevel(logging.ERROR) init_loggers[logger_name] = True return logger def add_file_handler_if_needed(logger, log_file, file_mode, log_level): for handler in logger.handlers: if isinstance(handler, logging.FileHandler): return if importlib.util.find_spec('torch') is not None: from modelscope.utils.torch_utils import is_master is_worker0 = is_master() else: is_worker0 = True if is_worker0 and log_file is not None: file_handler = logging.FileHandler(log_file, file_mode) file_handler.setFormatter(formatter) file_handler.setLevel(log_level) logger.addHandler(file_handler)