Repository: modin-project/modin

Issue:
Split logs into multiple separate files
Right now we have different kinds of logs intermixed into a single log file. For starters, I'd like to separate the memory usage history into a separate log.

This will require creating a logs folder and putting all logs there. 


Discussion:
Related to the fix in #4502 

Resolving patch:
diff --git a/modin/config/envvars.py b/modin/config/envvars.py
--- a/modin/config/envvars.py
+++ b/modin/config/envvars.py
@@ -415,16 +415,16 @@ def get(cls):
         return log_memory_interval
 
 
-class LogMemorySize(EnvironmentVariable, type=int):
-    """Size of logs (in MBs) to store per Modin job."""
+class LogFileSize(EnvironmentVariable, type=int):
+    """Max size of logs (in MBs) to store per Modin job."""
 
-    varname = "MODIN_LOG_MEMORY_SIZE"
-    default = 100
+    varname = "MODIN_LOG_FILE_SIZE"
+    default = 10
 
     @classmethod
     def put(cls, value):
         """
-        Set ``LogMemorySize`` with extra checks.
+        Set ``LogFileSize`` with extra checks.
 
         Parameters
         ----------
@@ -432,21 +432,21 @@ def put(cls, value):
             Config value to set.
         """
         if value <= 0:
-            raise ValueError(f"Log memory size should be > 0 MB, passed value {value}")
+            raise ValueError(f"Log file size should be > 0 MB, passed value {value}")
         super().put(value)
 
     @classmethod
     def get(cls):
         """
-        Get ``LogMemorySize`` with extra checks.
+        Get ``LogFileSize`` with extra checks.
 
         Returns
         -------
         int
         """
-        log_memory_size = super().get()
-        assert log_memory_size > 0, "`LogMemorySize` should be > 0"
-        return log_memory_size
+        log_file_size = super().get()
+        assert log_file_size > 0, "`LogFileSize` should be > 0"
+        return log_file_size
 
 
 class PersistentPickle(EnvironmentVariable, type=bool):
diff --git a/modin/logging/config.py b/modin/logging/config.py
--- a/modin/logging/config.py
+++ b/modin/logging/config.py
@@ -27,7 +27,7 @@
 import pkg_resources
 import threading
 import time
-from modin.config import LogMemoryInterval, LogMemorySize, LogMode
+from modin.config import LogMemoryInterval, LogFileSize, LogMode
 
 __LOGGER_CONFIGURED__: bool = False
 
@@ -93,14 +93,14 @@ def configure_logging():
     global __LOGGER_CONFIGURED__
     logger = logging.getLogger("modin.logger")
     job_id = uuid.uuid4().hex
-    log_filename = f".modin/logs/job_{job_id}.log"
+    log_filename = f".modin/logs/job_{job_id}/trace.log"
 
     os.makedirs(os.path.dirname(log_filename), exist_ok=True)
 
     logfile = RotatingFileHandler(
         filename=log_filename,
         mode="a",
-        maxBytes=LogMemorySize.get() * int(1e5),
+        maxBytes=LogFileSize.get() * int(1e6),
         backupCount=10,
     )
     formatter = ModinFormatter(
@@ -120,19 +120,31 @@ def configure_logging():
     pandas_version = pkg_resources.get_distribution("pandas").version
     num_physical_cores = str(psutil.cpu_count(logical=False))
     num_total_cores = str(psutil.cpu_count(logical=True))
-    svmem = psutil.virtual_memory()
     logger.info(f"Modin Version: {modin_version}")
     logger.info(f"Pandas Version: {pandas_version}")
     logger.info(f"Physical Cores: {num_physical_cores}")
     logger.info(f"Total Cores: {num_total_cores}")
-    logger.info(f"Memory Total: {bytes_int_to_str(svmem.total)}")
-    logger.info(f"Memory Available: {bytes_int_to_str(svmem.available)}")
-    logger.info(f"Memory Used: {bytes_int_to_str(svmem.used)}")
 
     if LogMode.get() != "enable_api_only":
         mem_sleep = LogMemoryInterval.get()
+        mem_logger = logging.getLogger("modin_memory.logger")
+        mem_log_filename = f".modin/logs/job_{job_id}/memory.log"
+        logfile = RotatingFileHandler(
+            filename=mem_log_filename,
+            mode="a",
+            maxBytes=LogFileSize.get() * int(1e6),
+            backupCount=10,
+        )
+        logfile.setFormatter(formatter)
+        mem_logger.addHandler(logfile)
+        mem_logger.setLevel(logging.DEBUG)
+
+        svmem = psutil.virtual_memory()
+        mem_logger.info(f"Memory Total: {bytes_int_to_str(svmem.total)}")
+        mem_logger.info(f"Memory Available: {bytes_int_to_str(svmem.available)}")
+        mem_logger.info(f"Memory Used: {bytes_int_to_str(svmem.used)}")
         mem = threading.Thread(
-            target=memory_thread, args=[logger, mem_sleep], daemon=True
+            target=memory_thread, args=[mem_logger, mem_sleep], daemon=True
         )
         mem.start()
 


Test patch: