Repository: aio-libs/aiohttp

Issue:
[bug] Gunicorn worker logs are incorrect
Gunicorn workers log the date ('%(t)s' fields) with the apache format. However, for some reason the `GunicornWorker` is using another format. The same happens to the response size ('%(b) field'). Aiohttp instead of logging the size of the response body is logging the size of the whole response, including the headers. 

The problem is in the `helper.atoms` function
https://github.com/KeepSafe/aiohttp/blob/51ea69d4c863b9a184293371b24e8d3e6651eb25/aiohttp/helpers.py#L228-L260

Below are the results for an app that simply returns `Hello, world` (12 characters).

```
Plain Gunicorn
127.0.0.1 - - [03/Aug/2015:14:59:09 -0300] "GET / HTTP/1.1" 200 12 "-" "curl/7.37.1"

Gunicorn + aiohttp.worker.GunicornWorker
127.0.0.1 - - Mon, 03 Aug 2015 17:51:39 GMT "GET / HTTP/1.1" 200 149 "-" "curl/7.37.1"
```



Discussion:

Resolving patch:
diff --git a/aiohttp/helpers.py b/aiohttp/helpers.py
--- a/aiohttp/helpers.py
+++ b/aiohttp/helpers.py
@@ -1,10 +1,12 @@
 """Various helper functions"""
 import base64
+import datetime
+import functools
 import io
 import os
+import re
 from urllib.parse import quote, urlencode
 from collections import namedtuple
-from wsgiref.handlers import format_date_time
 
 from . import hdrs, multidict
 from .errors import InvalidURL
@@ -226,69 +228,158 @@ def parse_remote_addr(forward):
     return remote[0], str(remote[1])
 
 
-def atoms(message, environ, response, transport, request_time):
-    """Gets atoms for log formatting."""
-    if message:
-        r = '{} {} HTTP/{}.{}'.format(
-            message.method, message.path,
-            message.version[0], message.version[1])
-        headers = message.headers
-    else:
-        r = ''
-        headers = {}
+class AccessLogger:
+    """Helper object to log access.
+
+    Usage:
+        log = logging.getLogger("spam")
+        log_format = "%a %{User-Agent}i"
+        access_logger = AccessLogger(log, log_format)
+        access_logger.log(message, environ, response, transport, time)
+
+    Format:
+        %%  The percent sign
+        %a  Remote IP-address (IP-address of proxy if using reverse proxy)
+        %t  Time the request was received
+        %P  The process ID of the child that serviced the request
+        %r  First line of request
+        %s  Status
+        %b  Size of response in bytes, excluding HTTP headers
+        %O  Bytes sent, including headers
+        %T  The time taken to serve the request, in seconds
+        %D  The time taken to serve the request, in microseconds
+        %{Foobar}i  The contents of Foobar: header line(s) in
+                    the request sent to the server
+        %{Foobar}o  The contents of Foobar: header line(s) in the reply
+        %{FOOBAR}e  The contents of the environment variable FOOBAR
 
-    if transport is not None:
-        remote_addr = parse_remote_addr(
-            transport.get_extra_info('peername', ('127.0.0.1', )))
-    else:
-        remote_addr = ('',)
-
-    atoms = {
-        'h': remote_addr[0],
-        'l': '-',
-        'u': '-',
-        't': format_date_time(None),
-        'r': r,
-        's': str(getattr(response, 'status', '')),
-        'b': str(getattr(response, 'output_length', '')),
-        'f': headers.get(hdrs.REFERER, '-'),
-        'a': headers.get(hdrs.USER_AGENT, '-'),
-        'T': str(int(request_time)),
-        'D': str(request_time).split('.', 1)[-1][:6],
-        'p': "<%s>" % os.getpid()
-    }
-
-    return atoms
-
-
-class SafeAtoms(dict):
-    """Copy from gunicorn"""
-
-    def __init__(self, atoms, i_headers, o_headers):
-        dict.__init__(self)
-
-        self._i_headers = i_headers
-        self._o_headers = o_headers
-
-        for key, value in atoms.items():
-            self[key] = value.replace('"', '\\"')
-
-    def __getitem__(self, k):
-        if k.startswith('{'):
-            if k.endswith('}i'):
-                headers = self._i_headers
-            elif k.endswith('}o'):
-                headers = self._o_headers
-            else:
-                headers = None
+    """
 
-            if headers is not None:
-                return headers.get(k[1:-2], '-')
+    LOG_FORMAT = '%a %l %u %t "%r" %s %b "%{Referrer}i" "%{User-Agent}i"'
+    FORMAT_RE = re.compile(r'%(\{([A-Za-z\-]+)\}([ioe])|[atPrsbOTD])')
+    CLEANUP_RE = re.compile(r'(%[^s])')
+    _FORMAT_CACHE = {}
 
-        if k in self:
-            return super(SafeAtoms, self).__getitem__(k)
-        else:
+    def __init__(self, logger, log_format=None):
+        """Initialise the logger.
+
+        :param logger: logger object to be used for logging
+        :param log_format: apache compatible log format
+
+        """
+        self.logger = logger
+        log_format = log_format or self.LOG_FORMAT
+        _compiled_format = AccessLogger._FORMAT_CACHE.get(log_format)
+        if not _compiled_format:
+            _compiled_format = self.compile_format(log_format)
+            AccessLogger._FORMAT_CACHE[log_format] = _compiled_format
+        self._log_format, self._methods = _compiled_format
+
+    def compile_format(self, log_format):
+        """Translate log_format into form usable by modulo formatting
+
+        All known atoms will be replaced with %s
+        Also methods for formatting of those atoms will be added to
+        _methods in apropriate order
+
+        For example we have log_format = "%a %t"
+        This format will be translated to "%s %s"
+        Also contents of _methods will be
+        [self._format_a, self._format_t]
+        These method will be called and results will be passed
+        to translated string format.
+
+        Each _format_* method receive 'args' which is list of arguments
+        given to self.log
+
+        Exceptions are _format_e, _format_i and _format_o methods which
+        also receive key name (by functools.partial)
+
+        """
+
+        log_format = log_format.replace("%l", "-")
+        log_format = log_format.replace("%u", "-")
+        methods = []
+
+        for atom in self.FORMAT_RE.findall(log_format):
+            if atom[1] == '':
+                methods.append(getattr(AccessLogger, '_format_%s' % atom[0]))
+            else:
+                m = getattr(AccessLogger, '_format_%s' % atom[2])
+                methods.append(functools.partial(m, atom[1]))
+        log_format = self.FORMAT_RE.sub(r'%s', log_format)
+        log_format = self.CLEANUP_RE.sub(r'%\1', log_format)
+        return log_format, methods
+
+    @staticmethod
+    def _format_e(key, args):
+        return (args[1] or {}).get(multidict.upstr(key), '-')
+
+    @staticmethod
+    def _format_i(key, args):
+        return args[0].headers.get(multidict.upstr(key), '-')
+
+    @staticmethod
+    def _format_o(key, args):
+        return args[2].headers.get(multidict.upstr(key), '-')
+
+    @staticmethod
+    def _format_a(args):
+        return args[3].get_extra_info('peername')[0]
+
+    @staticmethod
+    def _format_t(args):
+        return datetime.datetime.utcnow().strftime('[%d/%b/%Y:%H:%M:%S +0000]')
+
+    @staticmethod
+    def _format_P(args):
+        return "<%s>" % os.getpid()
+
+    @staticmethod
+    def _format_r(args):
+        msg = args[0]
+        if not msg:
             return '-'
+        return '%s %s HTTP/%s.%s' % tuple((msg.method,
+                                           msg.path) + msg.version)
+
+    @staticmethod
+    def _format_s(args):
+        return args[2].status
+
+    @staticmethod
+    def _format_b(args):
+        return args[2].body_length
+
+    @staticmethod
+    def _format_O(args):
+        return args[2].output_length
+
+    @staticmethod
+    def _format_T(args):
+        return round(args[4])
+
+    @staticmethod
+    def _format_D(args):
+        return round(args[4] * 1000000)
+
+    def _format_line(self, args):
+        return tuple(m(args) for m in self._methods)
+
+    def log(self, message, environ, response, transport, time):
+        """Log access.
+
+        :param message: Request object. May be None.
+        :param environ: Environment dict. May be None.
+        :param response: Response object.
+        :param transport: Tansport object.
+        :param float time: Time taken to serve the request.
+        """
+        try:
+            self.logger.info(self._log_format % self._format_line(
+                [message, environ, response, transport, time]))
+        except Exception:
+            self.logger.exception("Error in logging")
 
 
 _marker = object()
diff --git a/aiohttp/protocol.py b/aiohttp/protocol.py
--- a/aiohttp/protocol.py
+++ b/aiohttp/protocol.py
@@ -546,8 +546,13 @@ def __init__(self, transport, version, close):
         self.headers = CIMultiDict()
         self.headers_sent = False
         self.output_length = 0
+        self.headers_length = 0
         self._output_size = 0
 
+    @property
+    def body_length(self):
+        return self.output_length - self.headers_length
+
     def force_close(self):
         self.closing = True
         self.keepalive = False
@@ -651,6 +656,7 @@ def send_headers(self, _sep=': ', _end='\r\n'):
         headers = headers.encode('utf-8') + b'\r\n'
 
         self.output_length += len(headers)
+        self.headers_length = len(headers)
         self.transport.write(headers)
 
     def _add_default_headers(self):
diff --git a/aiohttp/server.py b/aiohttp/server.py
--- a/aiohttp/server.py
+++ b/aiohttp/server.py
@@ -26,9 +26,6 @@
   </body>
 </html>"""
 
-ACCESS_LOG_FORMAT = (
-    '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"')
-
 
 if hasattr(socket, 'SO_KEEPALIVE'):
     def tcp_keepalive(server, transport):
@@ -91,7 +88,7 @@ def __init__(self, *, loop=None,
                  timeout=0,
                  logger=server_logger,
                  access_log=None,
-                 access_log_format=ACCESS_LOG_FORMAT,
+                 access_log_format=None,
                  host="",
                  port=0,
                  debug=False,
@@ -111,7 +108,11 @@ def __init__(self, *, loop=None,
         self.logger = log or logger
         self.debug = debug
         self.access_log = access_log
-        self.access_log_format = access_log_format
+        if access_log:
+            self.access_logger = helpers.AccessLogger(access_log,
+                                                      access_log_format)
+        else:
+            self.access_logger = None
 
     @property
     def keep_alive_timeout(self):
@@ -187,17 +188,9 @@ def keep_alive(self, val):
         self._keep_alive = val
 
     def log_access(self, message, environ, response, time):
-        if self.access_log and self.access_log_format:
-            try:
-                environ = environ if environ is not None else {}
-                atoms = helpers.SafeAtoms(
-                    helpers.atoms(
-                        message, environ, response, self.transport, time),
-                    getattr(message, 'headers', None),
-                    getattr(response, 'headers', None))
-                self.access_log.info(self.access_log_format % atoms)
-            except:
-                self.logger.error(traceback.format_exc())
+        if self.access_logger:
+            self.access_logger.log(message, environ, response,
+                                   self.transport, time)
 
     def log_debug(self, *args, **kw):
         if self.debug:


Test patch:
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -1,7 +1,7 @@
 import pytest
-
+from unittest import mock
 from aiohttp import helpers
-from aiohttp import MultiDict
+import datetime
 
 
 def test_parse_mimetype_1():
@@ -107,6 +107,67 @@ def test_invalid_formdata_content_transfer_encoding():
                            content_transfer_encoding=invalid_val)
 
 
+def test_access_logger_format():
+    log_format = '%T {%{SPAM}e} "%{ETag}o" %X {X} %%P'
+    mock_logger = mock.Mock()
+    access_logger = helpers.AccessLogger(mock_logger, log_format)
+    expected = '%s {%s} "%s" %%X {X} %%%s'
+    assert expected == access_logger._log_format
+
+
+@mock.patch("aiohttp.helpers.datetime")
+@mock.patch("os.getpid")
+def test_access_logger_atoms(mock_getpid, mock_datetime):
+    utcnow = datetime.datetime(1843, 1, 1, 0, 0)
+    mock_datetime.datetime.utcnow.return_value = utcnow
+    mock_getpid.return_value = 42
+    log_format = '%a %t %P %l %u %r %s %b %O %T %D'
+    mock_logger = mock.Mock()
+    access_logger = helpers.AccessLogger(mock_logger, log_format)
+    message = mock.Mock(headers={}, method="GET", path="/path", version=(1, 1))
+    environ = {}
+    response = mock.Mock(headers={}, output_length=123,
+                         body_length=42, status=200)
+    transport = mock.Mock()
+    transport.get_extra_info.return_value = ("127.0.0.2", 1234)
+    access_logger.log(message, environ, response, transport, 3.1415926)
+    assert False == mock_logger.exception.called
+    expected = ('127.0.0.2 [01/Jan/1843:00:00:00 +0000] <42> - - '
+                'GET /path HTTP/1.1 200 42 123 3 3141593')
+    mock_logger.info.assert_called_with(expected)
+
+
+def test_access_logger_dicts():
+    log_format = '%{User-Agent}i %{Content-Length}o %{SPAM}e %{None}i'
+    mock_logger = mock.Mock()
+    access_logger = helpers.AccessLogger(mock_logger, log_format)
+    message = mock.Mock(headers={"USER-AGENT": "Mock/1.0"}, version=(1, 1))
+    environ = {"SPAM": "EGGS"}
+    response = mock.Mock(headers={"CONTENT-LENGTH": 123})
+    transport = mock.Mock()
+    transport.get_extra_info.return_value = ("127.0.0.2", 1234)
+    access_logger.log(message, environ, response, transport, 0.0)
+    assert False == mock_logger.error.called
+    expected = 'Mock/1.0 123 EGGS -'
+    mock_logger.info.assert_called_with(expected)
+
+
+def test_access_logger_error():
+    mock_logger = mock.Mock()
+    access_logger = helpers.AccessLogger(mock_logger, "")
+    access_logger.log(None, None, None, None, None)
+    assert True == mock_logger.exception.called
+
+
+def test_logger_no_message_and_environ():
+    mock_logger = mock.Mock()
+    mock_transport = mock.Mock()
+    mock_transport.get_extra_info.return_value = ("127.0.0.3", 0)
+    access_logger = helpers.AccessLogger(mock_logger, "%r %{FOOBAR}e")
+    access_logger.log(None, None, None, mock_transport, 0.0)
+    mock_logger.info.assert_called_with("- -")
+
+
 def test_reify():
     class A:
         @helpers.reify
@@ -140,33 +201,6 @@ def prop(self):
         a.prop = 123
 
 
-def test_get_seconds_and_milliseconds():
-    response = dict(status=200, output_length=1)
-    request_time = 321.012345678901234
-
-    atoms = helpers.atoms(None, None, response, None, request_time)
-    assert atoms['T'] == '321'
-    assert atoms['D'] == '012345'
-
-
-def test_get_non_existing():
-    atoms = helpers.SafeAtoms(
-        {}, MultiDict(), MultiDict())
-    assert atoms['unknown'] == '-'
-
-
-def test_get_lower():
-    i_headers = MultiDict([('test', '123')])
-    o_headers = MultiDict([('TEST', '123')])
-
-    atoms = helpers.SafeAtoms({}, i_headers, o_headers)
-    assert atoms['{test}i'] == '123'
-    assert atoms['{test}o'] == '-'
-    assert atoms['{TEST}o'] == '123'
-    assert atoms['{UNKNOWN}o'] == '-'
-    assert atoms['{UNKNOWN}'] == '-'
-
-
 def test_requote_uri_with_unquoted_percents():
     # Ensure we handle unquoted percent signs in redirects.
     bad_uri = 'http://example.com/fiz?buz=%ppicture'
diff --git a/tests/test_http_server.py b/tests/test_http_server.py
--- a/tests/test_http_server.py
+++ b/tests/test_http_server.py
@@ -533,37 +533,6 @@ def test_keep_alive_close_existing(self):
         self.assertIsNone(srv._keep_alive_handle)
         self.assertTrue(transport.close.called)
 
-    def test_log_access_error(self):
-        transport = unittest.mock.Mock()
-        transport.get_extra_info.return_value = '127.0.0.1'
-
-        srv = server.ServerHttpProtocol(loop=self.loop)
-        srv.transport = transport
-        srv.logger = unittest.mock.Mock()
-        srv.access_log = unittest.mock.Mock()
-
-        message = unittest.mock.Mock()
-        message.headers = []
-        message.version = (1, 1)
-        srv.log_access(None, None, None, None)
-
-        self.assertTrue(srv.logger.error.called)
-
-    def test_log_access_disabled(self):
-        transport = unittest.mock.Mock()
-
-        srv = server.ServerHttpProtocol(loop=self.loop, access_log=None)
-        srv.connection_made(transport)
-        self.addCleanup(self.cleanup_server, srv)
-        srv.logger = unittest.mock.Mock()
-
-        message = unittest.mock.Mock()
-        message.headers = []
-        message.version = (1, 1)
-        srv.log_access(None, None, None, None)
-
-        self.assertFalse(srv.logger.error.called)
-
     def test_cancel_not_connected_handler(self):
         srv = server.ServerHttpProtocol(loop=self.loop)
         srv.cancel_slow_request()