Repository: benoitc/gunicorn

Issue:
Do not log the number of workers all the time
To support statsD instrumentation, [`manage_workers`](https://github.com/benoitc/gunicorn/blob/master/gunicorn/arbiter.py#L485) logs the number of workers, which is unsightly in logs.

While the call will remain to have smoother statsd data, no log line will be generated.



Discussion:
The statsd integration via the logging system is really convenient. I wonder if it has any need to respect log level at all, though. After all, it's designed to be really cheap to send metrics and not every log call generates metrics. So maybe we can just ensure that metrics are sent even if the log level is disabled (assuming `logging` will still bubble it up to the gunicorn handler) and just move the worker count to a debug log level.

i concur with @tilgovi can't we [log](https://github.com/benoitc/gunicorn/blob/master/gunicorn/arbiter.py#L485-L488) only when the number changes? It can be moved at the end of the [spawn_workers](https://github.com/benoitc/gunicorn/blob/master/gunicorn/arbiter.py#L532) function or in the [if case](https://github.com/benoitc/gunicorn/blob/master/gunicorn/arbiter.py#L476) after the execution of the function.

Thoughts?

Statsd works better if it is feed gauges regularly. I will remove the
logging entry from any file and simply generate the metric in manage
workers.

I may also generate a spawn rate metric in spawn workers to catch cases
when there is a lot of churn.

Either way I will submit a PR over the weekend to clean up the log rapidly.
On Aug 1, 2014 3:09 AM, "Benoit Chesneau" notifications@github.com wrote:

> i concur with @tilgovi https://github.com/tilgovi can't we lof (
> https://github.com/benoitc/gunicorn/blob/master/gunicorn/arbiter.py#L485-L488)
> only when the number changes? It can be moved at the end of the
> spawn_workers
> https://github.com/benoitc/gunicorn/blob/master/gunicorn/arbiter.py#L532
> function or in the if case
> https://github.com/benoitc/gunicorn/blob/master/gunicorn/arbiter.py#L476
> after the execution of the function.
> 
> Thoughts?
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/benoitc/gunicorn/issues/834#issuecomment-50855502.


Resolving patch:
diff --git a/gunicorn/arbiter.py b/gunicorn/arbiter.py
--- a/gunicorn/arbiter.py
+++ b/gunicorn/arbiter.py
@@ -482,10 +482,10 @@ def manage_workers(self):
             (pid, _) = workers.pop(0)
             self.kill_worker(pid, signal.SIGTERM)
 
-        self.log.info("{0} workers".format(len(workers)),
-                      extra={"metric": "gunicorn.workers",
-                          "value": len(workers),
-                          "mtype": "gauge"})
+        self.log.debug("{0} workers".format(len(workers)),
+                        extra={ "metric": "gunicorn.workers",
+                                "value": len(workers),
+                                "mtype": "gauge"})
 
     def spawn_worker(self):
         self.worker_age += 1
diff --git a/gunicorn/instrument/statsd.py b/gunicorn/instrument/statsd.py
--- a/gunicorn/instrument/statsd.py
+++ b/gunicorn/instrument/statsd.py
@@ -6,6 +6,7 @@
 "Bare-bones implementation of statsD's protocol, client-side"
 
 import socket
+import logging
 from gunicorn.glogging import Logger
 
 # Instrumentation constants
@@ -50,30 +51,37 @@ def exception(self, msg, *args, **kwargs):
 
     # Special treatement for info, the most common log level
     def info(self, msg, *args, **kwargs):
-        """Log a given statistic if metric, value and type are present
-        """
-        Logger.info(self, msg, *args, **kwargs)
-        extra = kwargs.get("extra", None)
-        if extra is not None:
-            metric = extra.get(METRIC_VAR, None)
-            value = extra.get(VALUE_VAR, None)
-            typ = extra.get(MTYPE_VAR, None)
-            if metric and value and typ:
-                if typ == GAUGE_TYPE:
-                    self.gauge(metric, value)
-                elif typ == COUNTER_TYPE:
-                    self.increment(metric, value)
-                elif typ == HISTOGRAM_TYPE:
-                    self.histogram(metric, value)
-                else:
-                    pass
+        self.log(logging.INFO, msg, *args, **kwargs)
 
     # skip the run-of-the-mill logs
     def debug(self, msg, *args, **kwargs):
-        Logger.debug(self, msg, *args, **kwargs)
+        self.log(logging.DEBUG, msg, *args, **kwargs)
 
     def log(self, lvl, msg, *args, **kwargs):
-        Logger.log(self, lvl, msg, *args, **kwargs)
+        """Log a given statistic if metric, value and type are present
+        """
+        try:
+            extra = kwargs.get("extra", None)
+            if extra is not None:
+                metric = extra.get(METRIC_VAR, None)
+                value = extra.get(VALUE_VAR, None)
+                typ = extra.get(MTYPE_VAR, None)
+                if metric and value and typ:
+                    if typ == GAUGE_TYPE:
+                        self.gauge(metric, value)
+                    elif typ == COUNTER_TYPE:
+                        self.increment(metric, value)
+                    elif typ == HISTOGRAM_TYPE:
+                        self.histogram(metric, value)
+                    else:
+                        pass
+             
+            # Log to parent logger only if there is something to say
+            if msg is not None and len(msg) > 0:
+                Logger.log(self, lvl, msg, *args, **kwargs)
+        except Exception:
+            pass
+
 
     # access logging
     def access(self, resp, req, environ, request_time):


Test patch:
diff --git a/tests/test_010-statsd.py b/tests/test_010-statsd.py
--- a/tests/test_010-statsd.py
+++ b/tests/test_010-statsd.py
@@ -2,9 +2,17 @@
 
 import t
 
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+import logging
+
 from gunicorn.config import Config
 from gunicorn.instrument.statsd import Statsd
 
+
 class TestException(Exception): pass
 
 class MockSocket(object):
@@ -38,10 +46,27 @@ def test_statsd_fail():
 
 def test_instrument():
     logger = Statsd(Config())
+    # Capture logged messages
+    sio = StringIO()
+    logger.error_log.addHandler(logging.StreamHandler(sio))
     logger.sock = MockSocket(False)
 
+    # Regular message
     logger.info("Blah", extra={"mtype": "gauge", "metric": "gunicorn.test", "value": 666})
     t.eq(logger.sock.msgs[0], "gunicorn.test:666|g")
+    t.eq(sio.getvalue(), "Blah\n")
+    logger.sock.reset()
+
+    # Only metrics, no logging
+    logger.info("", extra={"mtype": "gauge", "metric": "gunicorn.test", "value": 666})
+    t.eq(logger.sock.msgs[0], "gunicorn.test:666|g")
+    t.eq(sio.getvalue(), "Blah\n")  # log is unchanged
+    logger.sock.reset()
+    
+    # Debug logging also supports metrics
+    logger.debug("", extra={"mtype": "gauge", "metric": "gunicorn.debug", "value": 667})
+    t.eq(logger.sock.msgs[0], "gunicorn.debug:667|g")
+    t.eq(sio.getvalue(), "Blah\n")  # log is unchanged
     logger.sock.reset()
 
     logger.critical("Boom")