Repository: pantsbuild/pants

Issue:
`log_level_by_target` doesn't seem to work well for Python targets
**Describe the bug**
Using the following:
```
level = "warn"
log_levels_by_target = {"pants.goal.stats_aggregator" = "info"}
```
when running `./pants --stats-log ...` I don't see the stats being logged.

**Pants version**
`main`

**OS**
Ubuntu

**Additional info**
I suspect this is because `log_levels_by_target` isn't used to configure the Python logger, and therefore the python logger for any Python module is configured to use the level set by `level`.

This can be seen by inspecting the logger in `src/python/pants/goal/stats_aggregator.py` which is set to the level `WARN`. Therefore I assume the log call never gets forwarded to the Rust-implemented handler, and therefore `log_levels_by_target` isn't considered.

`log_level_by_target` doesn't seem to work well for Python targets
**Describe the bug**
Using the following:
```
level = "warn"
log_levels_by_target = {"pants.goal.stats_aggregator" = "info"}
```
when running `./pants --stats-log ...` I don't see the stats being logged.

**Pants version**
`main`

**OS**
Ubuntu

**Additional info**
I suspect this is because `log_levels_by_target` isn't used to configure the Python logger, and therefore the python logger for any Python module is configured to use the level set by `level`.

This can be seen by inspecting the logger in `src/python/pants/goal/stats_aggregator.py` which is set to the level `WARN`. Therefore I assume the log call never gets forwarded to the Rust-implemented handler, and therefore `log_levels_by_target` isn't considered.



Discussion:

Resolving patch:
diff --git a/src/python/pants/init/logging.py b/src/python/pants/init/logging.py
--- a/src/python/pants/init/logging.py
+++ b/src/python/pants/init/logging.py
@@ -99,7 +99,9 @@ def stdio_destination_use_color(use_color: bool) -> None:
 
 
 @contextmanager
-def _python_logging_setup(level: LogLevel, *, print_stacktrace: bool) -> Iterator[None]:
+def _python_logging_setup(
+    level: LogLevel, log_levels_by_target: dict[str, LogLevel], *, print_stacktrace: bool
+) -> Iterator[None]:
     """Installs a root Python logger that routes all logging through a Rust logger."""
 
     def trace_fn(self, message, *args, **kwargs):
@@ -130,6 +132,9 @@ def set_logging_handlers(handlers):
         logger.addHandler(handler)
         level.set_level_for(logger)
 
+        for key, level in log_levels_by_target.items():
+            level.set_level_for(logging.getLogger(key))
+
         if logger.isEnabledFor(LogLevel.TRACE.level):
             http.client.HTTPConnection.debuglevel = 1  # type: ignore[attr-defined]
             requests_logger = logging.getLogger("requests.packages.urllib3")
@@ -213,7 +218,9 @@ def initialize_stdio_raw(
 
         sys.__stdin__, sys.__stdout__, sys.__stderr__ = sys.stdin, sys.stdout, sys.stderr
         # Install a Python logger that will route through the Rust logger.
-        with _python_logging_setup(global_level, print_stacktrace=print_stacktrace):
+        with _python_logging_setup(
+            global_level, log_levels_by_target, print_stacktrace=print_stacktrace
+        ):
             yield
     finally:
         sys.stdin, sys.stdout, sys.stderr = original_stdin, original_stdout, original_stderr


Test patch:
diff --git a/src/python/pants/init/logging_integration_test.py b/src/python/pants/init/logging_integration_test.py
new file mode 100644
--- /dev/null
+++ b/src/python/pants/init/logging_integration_test.py
@@ -0,0 +1,84 @@
+# Copyright 2021 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from pants.testutil.pants_integration_test import run_pants, setup_tmpdir
+
+PLUGIN = """
+import logging
+
+from pants.engine.goal import GoalSubsystem, Goal
+from pants.engine.rules import collect_rules, goal_rule
+
+class LogSubsystem(GoalSubsystem):
+    name = "logger"
+    help = "foo"
+
+
+class LogGoal(Goal):
+    subsystem_cls = LogSubsystem
+
+
+@goal_rule
+def write_logs() -> LogGoal:
+    for logger_name in ("globalLevel", "infoOverride", "debugOverride"):
+        logger = logging.getLogger(f"plugins.logger.{{logger_name}}")
+        logger.debug("debug log")
+        logger.info("info log")
+        logger.warning("warn log")
+    return LogGoal(exit_code=0)
+
+def rules():
+    return collect_rules()
+"""
+
+REGISTER = """
+from plugins import logger
+
+def rules():
+    return logger.rules()
+"""
+
+
+def test_log_by_level() -> None:
+    """Check that overriding log levels works for logs both coming from Rust and from Python.
+
+    This also checks that we correctly log `Starting` and `Completed` messages when the dynamic UI
+    is disabled.
+    """
+    with setup_tmpdir({"plugins/logger.py": PLUGIN, "plugins/register.py": REGISTER}) as tmpdir:
+        result = run_pants(
+            [
+                f"--pythonpath={tmpdir}",
+                "--backend-packages=plugins",
+                "--no-dynamic-ui",
+                "--show-log-target",
+                "--level=warn",
+                (
+                    "--log-levels-by-target={"
+                    "'plugins.logger.infoOverride': 'info', "
+                    "'plugins.logger.debugOverride': 'debug', "
+                    "'workunit_store': 'debug'}"
+                ),
+                "logger",
+            ]
+        )
+
+    global_level = "globalLevel"
+    info_override = "infoOverride"
+    debug_override = "debugOverride"
+
+    for logger in (global_level, info_override, debug_override):
+        assert f"[WARN] (plugins.logger.{logger}) warn log" in result.stderr
+
+    for logger in (info_override, debug_override):
+        assert f"[INFO] (plugins.logger.{logger}) info log" in result.stderr
+    assert "[INFO] (plugins.logger.globalLevel) info log" not in result.stderr
+
+    assert "[DEBUG] (plugins.logger.debugOverride) debug log" in result.stderr
+    for logger in (global_level, info_override):
+        assert f"[DEBUG] (plugins.logger.{logger} debug log" not in result.stderr
+
+    # Check that overriding levels for Rust code works, and also that we log Starting and Completed
+    # properly.
+    assert "[DEBUG] (workunit_store) Starting: `logger` goal" in result.stderr
+    assert "[DEBUG] (workunit_store) Completed: `logger` goal" in result.stderr
diff --git a/tests/python/pants_test/integration/BUILD b/tests/python/pants_test/integration/BUILD
--- a/tests/python/pants_test/integration/BUILD
+++ b/tests/python/pants_test/integration/BUILD
@@ -33,8 +33,6 @@ python_tests(
 
 python_tests(name="prelude_integration", sources=["prelude_integration_test.py"], timeout=180)
 
-python_tests(name="log_output_integration", sources=["log_output_integration_test.py"], timeout=180)
-
 python_tests(
     name="remote_cache_integration",
     sources=["remote_cache_integration_test.py"],
diff --git a/tests/python/pants_test/integration/log_output_integration_test.py b/tests/python/pants_test/integration/log_output_integration_test.py
deleted file mode 100644
--- a/tests/python/pants_test/integration/log_output_integration_test.py
+++ /dev/null
@@ -1,55 +0,0 @@
-# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
-# Licensed under the Apache License, Version 2.0 (see LICENSE).
-
-from textwrap import dedent
-
-from pants.testutil.pants_integration_test import run_pants, setup_tmpdir
-
-
-def test_completed_log_output() -> None:
-    sources = {
-        "src/python/project/__init__.py": "",
-        "src/python/project/lib.py": dedent(
-            """\
-            def add(x: int, y: int) -> int:
-                return x + y
-            """
-        ),
-        "src/python/project/BUILD": "python_sources()",
-    }
-    with setup_tmpdir(sources) as tmpdir:
-        result = run_pants(
-            [
-                "--no-dynamic-ui",
-                "--backend-packages=['pants.backend.python', 'pants.backend.python.typecheck.mypy']",
-                "-ldebug",
-                "check",
-                f"{tmpdir}/src/python/project",
-            ]
-        )
-
-    result.assert_success()
-    assert "[DEBUG] Starting: Run MyPy on" in result.stderr
-    assert "[DEBUG] Completed: Run MyPy on" in result.stderr
-
-
-def test_log_filtering_by_target() -> None:
-    sources = {
-        "src/python/project/__init__.py": "",
-        "src/python/project/BUILD": "python_sources()",
-    }
-    with setup_tmpdir(sources) as tmpdir:
-        result = run_pants(
-            [
-                "--no-pantsd",
-                "--backend-packages=['pants.backend.python']",
-                "--no-dynamic-ui",
-                "--level=info",
-                "--show-log-target",
-                '--log-levels-by-target={"workunit_store": "debug"}',
-                "list",
-                f"{tmpdir}/src/python/project",
-            ]
-        )
-
-        assert "[DEBUG] (workunit_store) Starting: `list` goal" in result.stderr