Repository: pfnet/pytorch-pfn-extras

Issue:
Create internal debug tools
We need some kind of logging mechanism for debug

When loading snapshots or verifying trigger checks having a 
`PPE_LOG_LEVEL` environment variable that allows us to obtain DEBUG information 
(snapshot not found, snapshot invalid, etc.) will be very useful.


Discussion:
We can use `logging` module and switch the log level (e.g. `ERROR` by default, `DEBUG` if PPE_LOG_LEVEL is set)

Resolving patch:
diff --git a/pytorch_pfn_extras/__init__.py b/pytorch_pfn_extras/__init__.py
--- a/pytorch_pfn_extras/__init__.py
+++ b/pytorch_pfn_extras/__init__.py
@@ -1,3 +1,7 @@
+# Configure the logging before instantiating anything else
+from pytorch_pfn_extras import logging  # NOQA
+logging._configure_logging()
+
 from pytorch_pfn_extras import config  # NOQA
 from pytorch_pfn_extras import cuda  # NOQA
 from pytorch_pfn_extras import dataset  # NOQA
diff --git a/pytorch_pfn_extras/logging.py b/pytorch_pfn_extras/logging.py
new file mode 100644
--- /dev/null
+++ b/pytorch_pfn_extras/logging.py
@@ -0,0 +1,41 @@
+import logging
+import os
+
+from logging import DEBUG, INFO, WARNING, ERROR, CRITICAL  # NOQA
+
+_logger_name = 'ppe'
+_logger_format = '[%(name)s] %(asctime)s: (%(levelname)s) %(message)s'
+_logger = None
+
+
+def _configure_logging(*, filename=None, level='ERROR', format=_logger_format):
+    global _logger
+    filename = os.environ.get('PPE_LOG_FILENAME', filename)
+    if filename is None:
+        handler = logging.StreamHandler()
+    else:
+        handler = logging.FileHandler(filename)
+    handler.setFormatter(logging.Formatter(format))
+    # To dynamically change the level if needed
+    # basicConfig does not allow to change the level right after
+    _logger = logging.getLogger(_logger_name)
+    level = os.environ.get('PPE_LOG_LEVEL', level)
+    for lvl in (logging.DEBUG, logging.INFO,
+                logging.WARNING, logging.ERROR, logging.CRITICAL):
+        if logging.getLevelName(lvl) == level:
+            _logger.setLevel(lvl)
+            break
+    else:
+        _logger.setLevel(logging.INFO)
+        _logger.warning('invalid PPE_LOG_LEVEL (%s); using INFO', level)
+    _logger.addHandler(handler)
+
+
+def _get_root_logger():
+    """Returns a logger to be used by pytorch-pfn-extras."""
+    return _logger
+
+
+def get_logger(name):
+    """Returns a child logger to be used by applications."""
+    return _logger.getChild(name)
diff --git a/pytorch_pfn_extras/training/extensions/_snapshot.py b/pytorch_pfn_extras/training/extensions/_snapshot.py
--- a/pytorch_pfn_extras/training/extensions/_snapshot.py
+++ b/pytorch_pfn_extras/training/extensions/_snapshot.py
@@ -3,9 +3,13 @@
 import torch
 import torch.distributed
 
+from pytorch_pfn_extras import logging
 from pytorch_pfn_extras.training import extension
 
 
+logger = logging._get_root_logger()
+
+
 def _find_snapshot_files(fmt, path, fs):
     '''Only prefix and suffix match
 
@@ -35,7 +39,6 @@ def _find_snapshot_files(fmt, path, fs):
     def _prepend_mtime(f):
         t = fs.stat(os.path.join(path, f)).last_modified
         return (t, f)
-
     return sorted(_prepend_mtime(file) for file in matched_files)
 
 
@@ -56,11 +59,10 @@ def _find_latest_snapshot(fmt, path, fs):
 
     """
     snapshot_files = _find_snapshot_files(fmt, path, fs)
-
+    logger.debug('found snapshot files {}'.format(snapshot_files))
     if len(snapshot_files) > 0:
         _, filename = snapshot_files[-1]
         return filename
-
     return None
 
 


Test patch:
diff --git a/tests/pytorch_pfn_extras_tests/test_logging.py b/tests/pytorch_pfn_extras_tests/test_logging.py
new file mode 100644
--- /dev/null
+++ b/tests/pytorch_pfn_extras_tests/test_logging.py
@@ -0,0 +1,23 @@
+import tempfile
+
+from pytorch_pfn_extras import logging
+
+
+def test_file_output():
+    try:
+        with tempfile.NamedTemporaryFile() as logfile:
+            logfile.close()  # this is needed for Windows
+            logging._configure_logging(filename=logfile.name, level='DEBUG')
+            logger = logging._get_root_logger()
+            logger.info('TEST LOG MESSAGE')
+            with open(logfile.name) as f:
+                assert 'TEST LOG MESSAGE' in f.read()
+    finally:
+        logging._configure_logging()
+
+
+def test_get_logger():
+    logger = logging.get_logger('app')
+    logger.setLevel(logging.DEBUG)
+    assert logger.name == 'ppe.app'
+    assert logger.level == logging.DEBUG