Elron commited on
Commit
99b503f
1 Parent(s): 6489550

Upload logging_utils.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. logging_utils.py +27 -12
logging_utils.py CHANGED
@@ -4,6 +4,10 @@ import sys
4
  import threading
5
  from typing import Optional
6
 
 
 
 
 
7
  _lock = threading.Lock()
8
  _default_handler: Optional[logging.Handler] = None
9
 
@@ -15,25 +19,32 @@ log_levels = {
15
  "critical": logging.CRITICAL,
16
  }
17
 
18
- _default_log_level = logging.DEBUG
19
-
20
 
21
  def _get_default_logging_level():
22
- env_level_str = os.getenv("UNITXT_VERBOSITY", None)
23
- if env_level_str is not None:
24
- try:
25
- return log_levels[env_level_str]
26
- except KeyError as e:
27
- raise ValueError(
28
- f"UNITXT_VERBOSITY has to be one of: { ', '.join(log_levels.keys()) }. Got {env_level_str}."
29
- ) from e
30
- return _default_log_level
31
 
32
 
33
  def _get_library_root_logger() -> logging.Logger:
34
  return logging.getLogger(__name__.split(".")[0])
35
 
36
 
 
 
 
 
 
 
 
 
 
 
 
 
37
  def _configure_library_root_logger() -> None:
38
  global _default_handler
39
 
@@ -45,6 +56,7 @@ def _configure_library_root_logger() -> None:
45
  sys.stderr = open(os.devnull, "w")
46
 
47
  _default_handler.flush = sys.stderr.flush
 
48
 
49
  library_root_logger = _get_library_root_logger()
50
  library_root_logger.addHandler(_default_handler)
@@ -61,6 +73,9 @@ def get_logger(name: Optional[str] = None) -> logging.Logger:
61
  return logging.getLogger(name)
62
 
63
 
 
 
 
64
  def set_verbosity(level):
65
  _configure_library_root_logger()
66
  _get_library_root_logger().setLevel(log_levels.get(level))
@@ -68,7 +83,7 @@ def set_verbosity(level):
68
 
69
  def enable_explicit_format() -> None:
70
  for handler in _get_library_root_logger().handlers:
71
- formatter = logging.Formatter(
72
  "[Unitxt|%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"
73
  )
74
  handler.setFormatter(formatter)
 
4
  import threading
5
  from typing import Optional
6
 
7
+ from .settings_utils import get_settings
8
+
9
+ settings = get_settings()
10
+
11
  _lock = threading.Lock()
12
  _default_handler: Optional[logging.Handler] = None
13
 
 
19
  "critical": logging.CRITICAL,
20
  }
21
 
 
 
22
 
23
  def _get_default_logging_level():
24
+ try:
25
+ return log_levels[settings.default_verbosity]
26
+ except KeyError as e:
27
+ raise ValueError(
28
+ f"unitxt.settings.default_verobsity or env variable UNITXT_DEFAULT_VERBOSITY has to be one of: { ', '.join(log_levels.keys()) }. Got {settings.default_verbosity}."
29
+ ) from e
 
 
 
30
 
31
 
32
  def _get_library_root_logger() -> logging.Logger:
33
  return logging.getLogger(__name__.split(".")[0])
34
 
35
 
36
+ class SizeLimitedFormatter(logging.Formatter):
37
+ def format(self, record):
38
+ original_message = super().format(record)
39
+ max_size = settings.max_log_message_size
40
+ if len(original_message) > max_size:
41
+ return (
42
+ original_message[:max_size]
43
+ + f"...\n(Message is too long > {max_size}. Can be set thorugh unitxt.settings.max_log_message_size or UNITXT_MAX_LOG_MESSAGE_SIZE environment variable.)"
44
+ )
45
+ return original_message
46
+
47
+
48
  def _configure_library_root_logger() -> None:
49
  global _default_handler
50
 
 
56
  sys.stderr = open(os.devnull, "w")
57
 
58
  _default_handler.flush = sys.stderr.flush
59
+ _default_handler.setFormatter(SizeLimitedFormatter())
60
 
61
  library_root_logger = _get_library_root_logger()
62
  library_root_logger.addHandler(_default_handler)
 
73
  return logging.getLogger(name)
74
 
75
 
76
+ settings._logger = get_logger("settings")
77
+
78
+
79
  def set_verbosity(level):
80
  _configure_library_root_logger()
81
  _get_library_root_logger().setLevel(log_levels.get(level))
 
83
 
84
  def enable_explicit_format() -> None:
85
  for handler in _get_library_root_logger().handlers:
86
+ formatter = SizeLimitedFormatter(
87
  "[Unitxt|%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s"
88
  )
89
  handler.setFormatter(formatter)