Repository: aio-libs/aiohttp

Issue:
access log format is invalid when using gunicorn worker
It seems you have to pass in `--access-logformat='%a %l %u %t "%r" %s %b "%{Referrer}i" "%{User-Agent}i"'` to gunicorn for logging to work, they default format from gunicorn is `"%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"` which doesn't work with aiohttp



Discussion:
I have found this problem too, just today.  Thank you for the workaround.

Just ran into this issue... Took me a couple of hours to find this answer. Thanks for the workaround, I was starting to get grey hairs.

https://github.com/benoitc/gunicorn/issues/1300
might be related, can't get metrics on statsd

This is still an issue.

Gunicorn config defaults are invalid for `aiohttp.helpers.AccessLogger.log_format`. To fix this, you need to change at least `%a` behavior, and may be other format arguments that intersect between http://docs.gunicorn.org/en/stable/settings.html#access-log-format and http://aiohttp.readthedocs.io/en/stable/logging.html#format-specification.
This at least backward-incompatible change, but it'll be very nice to fix it finally.

Other option is to pass access logger class to  `RequestHandler` arguments from gunicorn worker, but it's too complicated.

We can make a workaround.
1. Replace _default gunicorn_ logger format with _default aiohttp_ one on worker initialization.
Follow step may be:
2. Add a checker for _gunicorn logging style_ and raise an explicit error on worker initialization
It can be just regexp for `%(name)s` parts in logging spec.

We can also add a support for _gunicorn style_ but it requires much more work and the solution is very complicated from my perspective.

@f0t0n maybe you have an interest in fixing the issue?

@asvetlov Gunicorn's `Arbiter` reads the gunicorn config, then [creates a logger](https://github.com/benoitc/gunicorn/blob/13d193736d0097245237ec235de4248a765b61ba/gunicorn/arbiter.py#L95) and passes it to [worker's constructor](https://github.com/benoitc/gunicorn/blob/13d193736d0097245237ec235de4248a765b61ba/gunicorn/arbiter.py#L542).

> 1. Replace default gunicorn logger format with default aiohttp one on worker initialization.

In this way if we call `self.log.setFormatter()` inside worker's constructor with formatter that uses default aiohttp's format string `aiohttp.helpers.LOG_FORMAT` we're not able to read one from gunicorn config since it's too late to override.

> 1. Add a checker for gunicorn logging style and raise an explicit error on worker initialization
>    It can be just regexp for %(name)s parts in logging spec.

Also I don't see a correct way to retrieve a format string from logger  like `self.log.handlers[0].formatter._fmt` since this string is a [protected member](https://hg.python.org/cpython/file/5c4ca109af1c/Lib/logging/__init__.py#l462) of `logging.Formatter` class.

access_logformat from gunicorn config is passed from [GunicornWebWorker](https://github.com/KeepSafe/aiohttp/blob/master/aiohttp/worker.py#L51) to ServerHttpProtocol, which [constructs](https://github.com/KeepSafe/aiohttp/blob/master/aiohttp/server.py#L130) AccessLogger "wrapper" with gunicorn (invalid) default format instead of aiohttp's one.

For me, access logs work with:

``` sh
gunicorn --worker-class=aiohttp.worker.GunicornWebWorker 
  --access-logformat="..." 
  --log-config=logging.conf
```

logging.conf part:

``` ini
[logger_gunicorn.access]
level=INFO
handlers=access_file
propagate=0
qualname=gunicorn.access

[handler_access_file]
class=logging.handlers.TimedRotatingFileHandler
formatter=access
args=('/data/logs/access.log', 'midnight')

[formatter_access]
format=%(message)s # That's what glogger receives from AccessLogger wrapper
class=logging.Formatter
```

Ok, I'll try to work with `self.cfg.access_log_format`.


Resolving patch:
diff --git a/aiohttp/worker.py b/aiohttp/worker.py
--- a/aiohttp/worker.py
+++ b/aiohttp/worker.py
@@ -2,19 +2,24 @@
 
 import asyncio
 import os
+import re
 import signal
 import ssl
 import sys
 
 import gunicorn.workers.base as base
 
-from aiohttp.helpers import ensure_future
+from gunicorn.config import AccessLogFormat as GunicornAccessLogFormat
+from aiohttp.helpers import AccessLogger, ensure_future
 
 __all__ = ('GunicornWebWorker', 'GunicornUVLoopWebWorker')
 
 
 class GunicornWebWorker(base.Worker):
 
+    DEFAULT_AIOHTTP_LOG_FORMAT = AccessLogger.LOG_FORMAT
+    DEFAULT_GUNICORN_LOG_FORMAT = GunicornAccessLogFormat.default
+
     def __init__(self, *args, **kw):  # pragma: no cover
         super().__init__(*args, **kw)
 
@@ -48,7 +53,8 @@ def make_handler(self, app):
             timeout=self.cfg.timeout,
             keep_alive=self.cfg.keepalive,
             access_log=self.log.access_log,
-            access_log_format=self.cfg.access_log_format)
+            access_log_format=self._get_valid_log_format(
+                self.cfg.access_log_format))
 
     @asyncio.coroutine
     def close(self):
@@ -158,6 +164,20 @@ def _create_ssl_context(cfg):
             ctx.set_ciphers(cfg.ciphers)
         return ctx
 
+    def _get_valid_log_format(self, source_format):
+        if source_format == self.DEFAULT_GUNICORN_LOG_FORMAT:
+            return self.DEFAULT_AIOHTTP_LOG_FORMAT
+        elif re.search(r'%\([^\)]+\)', source_format):
+            raise ValueError(
+                "Gunicorn's style options in form of `%(name)s` are not "
+                "supported for the log formatting. Please use aiohttp's "
+                "format specification to configure access log formatting: "
+                "http://aiohttp.readthedocs.io/en/stable/logging.html"
+                "#format-specification"
+            )
+        else:
+            return source_format
+
 
 class GunicornUVLoopWebWorker(GunicornWebWorker):
 


Test patch:
diff --git a/tests/test_worker.py b/tests/test_worker.py
--- a/tests/test_worker.py
+++ b/tests/test_worker.py
@@ -17,6 +17,10 @@
     uvloop = None
 
 
+WRONG_LOG_FORMAT = '%a "%{Referrer}i" %(h)s %(l)s %s'
+ACCEPTABLE_LOG_FORMAT = '%a "%{Referrer}i" %s'
+
+
 class BaseTestWorker:
 
     def __init__(self):
@@ -89,14 +93,32 @@ def test_init_signals(worker):
     assert worker.loop.add_signal_handler.called
 
 
-def test_make_handler(worker):
+def test_make_handler(worker, mocker):
     worker.wsgi = mock.Mock()
     worker.loop = mock.Mock()
     worker.log = mock.Mock()
     worker.cfg = mock.Mock()
+    worker.cfg.access_log_format = ACCEPTABLE_LOG_FORMAT
+    mocker.spy(worker, '_get_valid_log_format')
 
     f = worker.make_handler(worker.wsgi)
     assert f is worker.wsgi.make_handler.return_value
+    assert worker._get_valid_log_format.called
+
+
+@pytest.mark.parametrize('source,result', [
+    (ACCEPTABLE_LOG_FORMAT, ACCEPTABLE_LOG_FORMAT),
+    (AsyncioWorker.DEFAULT_GUNICORN_LOG_FORMAT,
+     AsyncioWorker.DEFAULT_AIOHTTP_LOG_FORMAT),
+])
+def test__get_valid_log_format_ok(worker, source, result):
+    assert result == worker._get_valid_log_format(source)
+
+
+def test__get_valid_log_format_exc(worker):
+    with pytest.raises(ValueError) as exc:
+        worker._get_valid_log_format(WRONG_LOG_FORMAT)
+    assert '%(name)s' in str(exc)
 
 
 @asyncio.coroutine
@@ -115,6 +137,7 @@ def test__run_ok(worker, loop):
     worker.wsgi.make_handler.return_value.requests_count = 1
     worker.cfg.max_requests = 100
     worker.cfg.is_ssl = True
+    worker.cfg.access_log_format = ACCEPTABLE_LOG_FORMAT
 
     ssl_context = mock.Mock()
     with mock.patch('ssl.SSLContext', return_value=ssl_context):
@@ -205,6 +228,7 @@ def test__run_ok_no_max_requests(worker, loop):
     worker.loop = loop
     loop.create_server = make_mocked_coro(sock)
     worker.wsgi.make_handler.return_value.requests_count = 1
+    worker.cfg.access_log_format = ACCEPTABLE_LOG_FORMAT
     worker.cfg.max_requests = 0
     worker.cfg.is_ssl = True
 
@@ -239,6 +263,7 @@ def test__run_ok_max_requests_exceeded(worker, loop):
     worker.loop = loop
     loop.create_server = make_mocked_coro(sock)
     worker.wsgi.make_handler.return_value.requests_count = 15
+    worker.cfg.access_log_format = ACCEPTABLE_LOG_FORMAT
     worker.cfg.max_requests = 10
     worker.cfg.is_ssl = True