Repository: aio-libs/aiosmtpd

Issue:
Possible Security Issue: AUTH credentials visible in log
I was fooling around with a branch, and decided to activate logging.

Found out that all `AUTH` commands are logged as-is, without sanitizing. Even if logging level set to `INFO`.

Not a problem if using multistep authentication (i.e., `AUTH mech` then wait for `334 ` then respond; the `_auth_interact` method does not log client response), but a problem if using one-step authentication (e.g., `AUTH PLAIN <encoded_username_and_password>`)

I fear this might cause to inadvertent leak of sensitive information. For example, if log is being forwarded to an external log collector (e.g., CloudWatch Log on AWS, DataDog, etc.)

I will submit an URGENT PR to fix this.


Comments:

Patch:
diff --git a/.gitignore b/.gitignore
index 995ec61..5295a9f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,4 @@ diffcov.html
 diffcov-*.html
 .python-version
 .pytype/
+~temp*
diff --git a/aiosmtpd/docs/NEWS.rst b/aiosmtpd/docs/NEWS.rst
index 260d382..eb47070 100644
--- a/aiosmtpd/docs/NEWS.rst
+++ b/aiosmtpd/docs/NEWS.rst
@@ -2,6 +2,17 @@
  NEWS for aiosmtpd
 ===================
 
+1.2.4 (aiosmtpd-next)
+=====================
+
+Added
+-----
+* Optional (default-disabled) logging of ``AUTH`` interaction -- with severe warnings
+
+Fixed/Improved
+--------------
+* ``AUTH`` command line now sanitized before logging (Closes #233)
+
 
 1.2.4 (aiosmtpd-next)
 =====================
diff --git a/aiosmtpd/smtp.py b/aiosmtpd/smtp.py
index da9c7a9..a5c5d39 100644
--- a/aiosmtpd/smtp.py
+++ b/aiosmtpd/smtp.py
@@ -31,7 +31,8 @@ from warnings import warn
 # region #### Custom Data Types #######################################################
 
 class _Missing:
-    pass
+    def __repr__(self):
+        return "MISSING"
 
 
 class _AuthMechAttr(NamedTuple):
@@ -60,7 +61,7 @@ __all__ = [
     "AuthMechanismType",
     "MISSING",
 ]  # Will be added to by @public
-__version__ = '1.2.4a1'
+__version__ = '1.2.4a2'
 __ident__ = 'Python SMTP {}'.format(__version__)
 log = logging.getLogger('mail.log')
 
@@ -77,6 +78,19 @@ VALID_AUTHMECH = re.compile(r"[A-Z0-9_-]+\Z")
 # https://tools.ietf.org/html/rfc3207.html#page-3
 ALLOWED_BEFORE_STARTTLS = {"NOOP", "EHLO", "STARTTLS", "QUIT"}
 
+# Auth hiding regexes
+CLIENT_AUTH_B = re.compile(
+    # Matches "AUTH" <mechanism> <whitespace_but_not_\r_nor_\n>
+    br"(?P<authm>\s*AUTH\s+\S+[^\S\r\n]+)"
+    # Param to AUTH <mechanism>. We only need to sanitize if param is given, which
+    # for some mechanisms contain sensitive info. If no param is given, then we
+    # can skip (match fails)
+    br"(\S+)"
+    # Optional bCRLF at end. Why optional? Because we also want to sanitize the
+    # stripped line. If no bCRLF, then this group will be b""
+    br"(?P<crlf>(?:\r\n)?)", re.IGNORECASE
+)
+
 # endregion
 
 
@@ -163,6 +177,23 @@ class TLSSetupException(Exception):
     pass
 
 
+@public
+def sanitize(text: bytes) -> bytes:
+    m = CLIENT_AUTH_B.match(text)
+    if m:
+        return m.group("authm") + b"********" + m.group("crlf")
+    return text
+
+
+@public
+def sanitized_log(func: Callable, msg: AnyStr, *args, **kwargs):
+    sanitized_args = [
+        sanitize(a) if isinstance(a, bytes) else a
+        for a in args
+    ]
+    func(msg, *sanitized_args, **kwargs)
+
+
 @public
 class SMTP(asyncio.StreamReaderProtocol):
     command_size_limit = 512
@@ -436,10 +467,10 @@ class SMTP(asyncio.StreamReaderProtocol):
                     # send error response and read the next command line.
                     await self.push('500 Command line too long')
                     continue
-                log.debug('_handle_client readline: %r', line)
+                sanitized_log(log.debug, '_handle_client readline: %r', line)
                 # XXX this rstrip may not completely preserve old behavior.
                 line = line.rstrip(b'\r\n')
-                log.info('%r >> %r', self.session.peer, line)
+                sanitized_log(log.info, '%r >> %r', self.session.peer, line)
                 if not line:
                     await self.push('500 Error: bad syntax')
                     continue
@@ -732,6 +763,7 @@ class SMTP(asyncio.StreamReaderProtocol):
             self,
             challenge: AnyStr,
             encode_to_b64: bool = True,
+            log_client_response: bool = False,
     ) -> Union[_Missing, bytes]:
         """
         Send challenge during authentication. "334 " will be prefixed, so do NOT
@@ -739,6 +771,9 @@ class SMTP(asyncio.StreamReaderProtocol):
 
         :param challenge: Challenge to send to client. If str, will be utf8-encoded.
         :param encode_to_b64: If true, then perform Base64 encoding on challenge
+        :param log_client_response: Perform logging of client's response.
+            WARNING: Might cause leak of sensitive information! Do not turn on
+            unless _absolutely_ necessary!
         :return: Response from client, or MISSING
         """
         challenge = (
@@ -750,9 +785,13 @@ class SMTP(asyncio.StreamReaderProtocol):
         #   - https://tools.ietf.org/html/rfc4954#page-4 ¶ 5
         #   - https://tools.ietf.org/html/rfc4954#page-13 "continue-req"
         challenge = b"334 " + (b64encode(challenge) if encode_to_b64 else challenge)
-        log.debug("Send challenge to %r: %r", self.session.peer, challenge)
+        log.debug("%r << challenge: %r", self.session.peer, challenge)
         await self.push(challenge)
         line = await self._reader.readline()
+        if log_client_response:
+            warn("AUTH interaction logging is enabled!")
+            warn("Sensitive information might be leaked!")
+            log.debug("%r >> %r", self.session.peer, line)
         blob: bytes = line.strip()
         # '*' handling in accordance with RFC4954
         if blob == b"*":


Test patch:
diff --git a/aiosmtpd/testing/helpers.py b/aiosmtpd/testing/helpers.py
index a02568e..614760b 100644
--- a/aiosmtpd/testing/helpers.py
+++ b/aiosmtpd/testing/helpers.py
@@ -66,24 +66,15 @@ def start(plugin):
 
 
 def assert_auth_success(testcase: TestCase, *response):
-    testcase.assertEqual(
-        (235, b"2.7.0 Authentication successful"),
-        response
-    )
+    assert response == (235, b"2.7.0 Authentication successful")
 
 
 def assert_auth_invalid(testcase: TestCase, *response):
-    testcase.assertEqual(
-        (535, b"5.7.8 Authentication credentials invalid"),
-        response
-    )
+    assert response == (535, b"5.7.8 Authentication credentials invalid")
 
 
 def assert_auth_required(testcase: TestCase, *response):
-    testcase.assertEqual(
-        (530, b"5.7.0 Authentication required"),
-        response
-    )
+    assert response == (530, b"5.7.0 Authentication required")
 
 
 SUPPORTED_COMMANDS_TLS: bytes = (
diff --git a/aiosmtpd/tests/test_smtp.py b/aiosmtpd/tests/test_smtp.py
index c2eed49..d225303 100644
--- a/aiosmtpd/tests/test_smtp.py
+++ b/aiosmtpd/tests/test_smtp.py
@@ -1,8 +1,10 @@
 """Test the SMTP protocol."""
 
+import os
 import time
 import socket
 import asyncio
+import logging
 import unittest
 import warnings
 
@@ -36,6 +38,7 @@ BCRLF = b'\r\n'
 
 
 ModuleResources = ExitStack()
+mail_logger = logging.getLogger('mail.log')
 
 
 def setUpModule():
@@ -43,6 +46,24 @@ def setUpModule():
     # and oftentimes (not always, though) leads to Error
     ModuleResources.enter_context(patch("socket.getfqdn", return_value="localhost"))
 
+    loglevel = int(os.environ.get("AIOSMTPD_TESTLOGLEVEL", logging.INFO))
+    mail_logger.setLevel(loglevel)
+
+    if "AIOSMTPD_TESTLOGFILE" in os.environ:
+        fhandler = logging.FileHandler(
+            os.environ["AIOSMTPD_TESTLOGFILE"],
+            mode="a",
+            encoding="latin1",
+        )
+        fhandler.setLevel(1)
+        fhandler.setFormatter(
+            logging.Formatter(
+                u"{asctime} - {name} - {levelname} - {message}",
+                style="{",
+            )
+        )
+        mail_logger.addHandler(fhandler)
+
 
 def tearDownModule():
     ModuleResources.close()
@@ -80,7 +101,14 @@ class PeekerHandler:
     ):
         return MISSING
 
-    async def auth_WITH_UNDERSCORE(self, server, args):
+    async def auth_WITH_UNDERSCORE(self, server: Server, args):
+        """
+        Be careful when using this AUTH mechanism; log_client_response is set to
+        True, and this will raise some severe warnings.
+        """
+        await server.challenge_auth(
+            "challenge", encode_to_b64=False, log_client_response=True
+        )
         return "250 OK"
 
     @auth_mechanism("with-dash")
@@ -241,6 +269,7 @@ class TestProtocol(unittest.TestCase):
     def setUp(self):
         self.transport = Mock()
         self.transport.write = self._write
+        self.transport.get_extra_info.return_value = "MockedPeer"
         self.responses = []
         self._old_loop = asyncio.get_event_loop()
         self.loop = asyncio.new_event_loop()
@@ -941,6 +970,26 @@ class TestSMTP(unittest.TestCase):
             )
             assert_auth_success(self, code, response)
 
+    def test_authplain_goodcreds_sanitized_log(self):
+        oldlevel = mail_logger.getEffectiveLevel()
+        mail_logger.setLevel(logging.DEBUG)
+        with SMTP(*self.address) as client:
+            client.ehlo('example.com')
+            with self.assertLogs(mail_logger, level="DEBUG") as cm:
+                code, response = client.docmd(
+                    'AUTH PLAIN ' +
+                    b64encode(b'\0goodlogin\0goodpasswd').decode()
+                )
+        mail_logger.setLevel(oldlevel)
+        interestings = [
+            msg for msg in cm.output if "AUTH PLAIN" in msg
+        ]
+        assert len(interestings) == 2
+        assert interestings[0].startswith("DEBUG:")
+        assert interestings[0].endswith("b'AUTH PLAIN ********\\r\\n'")
+        assert interestings[1].startswith("INFO:")
+        assert interestings[1].endswith("b'AUTH PLAIN ********'")
+
     def test_auth_plain_null(self):
         with SMTP(*self.address) as client:
             client.ehlo('example.com')
@@ -1110,6 +1159,19 @@ class TestSMTPAuth(unittest.TestCase):
                 code, mesg = client.docmd("AAA=")
                 assert_auth_success(self, code, mesg)
 
+    def test_auth_loginteract_warning(self):
+        """AUTH state of different clients must be independent"""
+        with SMTP(*self.address) as client:
+            client.ehlo("example.com")
+            resp = client.docmd("AUTH WITH_UNDERSCORE")
+            assert resp == (334, b"challenge")
+            with warnings.catch_warnings(record=True) as w:
+                code, mesg = client.docmd('=')
+            assert_auth_success(self, code, mesg)
+            assert len(w) > 0
+            assert str(w[0].message) == "AUTH interaction logging is enabled!"
+            assert str(w[1].message) == "Sensitive information might be leaked!"
+
 
 class TestRequiredAuthentication(unittest.TestCase):
     def setUp(self):