Repository: agronholm/smtpproto

Issue:
SMTPUTF8 mail command handling
It looks like smtpproto correctly handles the message body when the SMTPUTF8 extension is available. 

However, [per the spec](https://tools.ietf.org/html/rfc6531#section-3.4), the `MAIL FROM` command should include the `SMTPUTF8` option in that case. Currently there's no way to pass an option via the `mail` method. This could be solved by adding an options argument to that method.

It also seems like all output is always ASCII encoded. However, if SMTPUTF8 is supported and being used for the message, the `MAIL FROM` command (and subsequent `RCPT` commands) should be sent UTF-8 encoded.


Discussion:

Patch:
diff --git a/docs/versionhistory.rst b/docs/versionhistory.rst
index 2bc4d25..459ca8b 100644
--- a/docs/versionhistory.rst
+++ b/docs/versionhistory.rst
@@ -7,6 +7,8 @@ This library adheres to `Semantic Versioning 2.0 <http://semver.org/>`_.
 
 - Added missing ``authorization_id`` parameter to ``PlainAuthenticator`` (also fixes ``PLAIN``
   authentication not working since this field was missing from the encoded output)
+- Fixed sender/recipient addresses (in ``MAIL``/``RCPT`` commands) not being UTF-8 encoded in the
+  presence of the ``SMTPUTF8`` extension
 
 **1.0.0**
 
diff --git a/setup.cfg b/setup.cfg
index 98ba6a2..32510b1 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -18,6 +18,7 @@ classifiers =
     Programming Language :: Python :: 3.6
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
 
 [options]
 package_dir=
@@ -26,7 +27,7 @@ packages = find:
 python_requires = >= 3.6.2
 zip_safe = False
 install_requires =
-    anyio ~= 2.0.0b2
+    anyio ~= 2.0
     dataclasses; python_version < "3.7"
 
 [options.packages.find]
diff --git a/src/smtpproto/protocol.py b/src/smtpproto/protocol.py
index 4f5dcc2..e4ce01f 100644
--- a/src/smtpproto/protocol.py
+++ b/src/smtpproto/protocol.py
@@ -64,7 +64,7 @@ class SMTPClientProtocol:
     _response_code: Optional[int] = field(init=False, default=None)
     _response_lines: List[str] = field(init=False, default_factory=list)
     _command_sent: Optional[str] = field(init=False, default=None)
-    _args_sent: Optional[Tuple[str, ...]] = field(init=False, default=None)
+    _args_sent: Optional[Tuple[bytes, ...]] = field(init=False, default=None)
     _extensions: FrozenSet[str] = field(init=False, default_factory=frozenset)
     _auth_mechanisms: FrozenSet[str] = field(init=False, default_factory=frozenset)
     _max_message_size: Optional[int] = field(init=False, default=None)
@@ -85,18 +85,36 @@ class SMTPClientProtocol:
             raise SMTPUnsupportedAuthMechanism(
                 f'{mechanism} is not a supported authentication mechanism on this server')
 
-    def _send_command(self, command: str, *args: str) -> None:
+    def _encode_address(self, address: Union[str, Address]) -> bytes:
+        if isinstance(address, Address):
+            address_str = f'{address.username}@{address.domain}'
+        else:
+            address_str = address
+
+        if 'SMTPUTF8' in self._extensions:
+            return address_str.encode('utf-8')
+
+        # If SMPTUTF8 is not supported, the address must be ASCII compatible
+        try:
+            return address_str.encode('ascii')
+        except UnicodeEncodeError:
+            raise SMTPProtocolViolation(
+                f'The address {address_str!r} requires UTF-8 encoding but the server does not '
+                f'support the SMTPUTF8 extension')
+
+    def _send_command(self, command: str, *args: Union[str, bytes]) -> None:
         if self._command_sent is not None:
             raise SMTPProtocolViolation('Tried to send a command before the previous one received '
                                         'a response')
 
-        line = command
-        if args:
-            line += ' ' + ' '.join(args)
+        line = command.encode('ascii')
+        args_encoded = tuple(arg.encode('ascii') if isinstance(arg, str) else arg for arg in args)
+        if args_encoded:
+            line += b' ' + b' '.join(args_encoded)
 
-        self._out_buffer += line.encode('ascii') + b'\r\n'
+        self._out_buffer += line + b'\r\n'
         self._command_sent = command
-        self._args_sent = args
+        self._args_sent = args_encoded
 
     def _parse_extensions(self, lines: Iterable[str]) -> None:
         auth_mechanisms: List[str] = []
@@ -291,9 +309,10 @@ class SMTPClientProtocol:
         args = []
         if '8BITMIME' in self._extensions:
             args.append('BODY=8BITMIME')
+        if 'SMTPUTF8' in self._extensions:
+            args.append('SMTPUTF8')
 
-        address = sender.addr_spec if isinstance(sender, Address) else sender
-        self._send_command('MAIL', 'FROM:<' + address + '>', *args)
+        self._send_command('MAIL', b'FROM:<' + self._encode_address(sender) + b'>', *args)
 
     def recipient(self, recipient: Union[str, Address]) -> None:
         """
@@ -305,8 +324,7 @@ class SMTPClientProtocol:
 
         """
         self._require_state(ClientState.mailtx, ClientState.recipient_sent)
-        address = recipient.addr_spec if isinstance(recipient, Address) else recipient
-        self._send_command('RCPT', f'TO:<{address}>')
+        self._send_command('RCPT', b'TO:<' + self._encode_address(recipient) + b'>')
 
     def start_data(self) -> None:
         """


Test patch:
diff --git a/.github/workflows/codeqa-test.yml b/.github/workflows/codeqa-test.yml
index 45ff699..764333a 100644
--- a/.github/workflows/codeqa-test.yml
+++ b/.github/workflows/codeqa-test.yml
@@ -34,7 +34,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.6, 3.7, 3.8, 3.9]
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
diff --git a/tests/test_protocol.py b/tests/test_protocol.py
index ae63149..14cccff 100644
--- a/tests/test_protocol.py
+++ b/tests/test_protocol.py
@@ -65,7 +65,12 @@ def exchange_greetings(protocol, esmtp=True):
 
 def start_mail_tx(protocol):
     # Start a mail transaction
-    extra_args = b' BODY=8BITMIME' if '8BITMIME' in protocol.extensions else b''
+    extra_args = b''
+    if '8BITMIME' in protocol.extensions:
+        extra_args += b' BODY=8BITMIME'
+    if 'SMTPUTF8' in protocol.extensions:
+        extra_args += b' SMTPUTF8'
+
     call_protocol_method(protocol, lambda: protocol.mail('foo@bar.com'),
                          b'MAIL FROM:<foo@bar.com>' + extra_args + b'\r\n')
     feed_bytes(protocol, b'250 OK\r\n', 250, 'OK', ClientState.mailtx)
@@ -87,6 +92,14 @@ def start_mail_tx(protocol):
                'Start mail input; end with <CRLF>.<CRLF>', ClientState.send_data)
 
 
+@pytest.fixture(params=[
+    'héllö@example.org',
+    Address('Héllö World', 'héllö', 'example.org')
+], ids=['str', 'object'])
+def unicode_address(request):
+    return request.param
+
+
 @pytest.fixture
 def protocol():
     proto = SMTPClientProtocol()
@@ -100,7 +113,7 @@ def protocol():
     pytest.param(False, 'base64', 'This is a =?utf-8?q?subj=C3=ABct?=',
                  'VGhpcyBpcyDDpCB0ZXN0IG1lc3NhZ2UuCg==', id='7bit')
 ])
-def test_send_mail_utf8(protocol, esmtp, expected_cte, expected_subject, expected_body):
+def test_send_mail_utf8_content(protocol, esmtp, expected_cte, expected_subject, expected_body):
     exchange_greetings(protocol, esmtp=esmtp)
     start_mail_tx(protocol)
 
@@ -116,6 +129,29 @@ def test_send_mail_utf8(protocol, esmtp, expected_cte, expected_subject, expecte
     feed_bytes(protocol, b'250 OK\r\n', 250, 'OK', ClientState.ready)
 
 
+def test_send_mail_utf8_addresses(protocol, unicode_address):
+    exchange_greetings(protocol)
+    protocol.mail(unicode_address)
+    feed_bytes(protocol, b'250 OK\r\n', 250, 'OK', ClientState.mailtx)
+    protocol.recipient(unicode_address)
+    feed_bytes(protocol, b'250 OK\r\n', 250, 'OK', ClientState.recipient_sent)
+
+
+def test_send_mail_unicode_sender_encoding_error(protocol, unicode_address):
+    exchange_greetings(protocol, esmtp=False)
+    exc = pytest.raises(SMTPProtocolViolation, protocol.mail, unicode_address)
+    exc.match("^The address 'héllö@example.org' requires UTF-8")
+
+
+def test_send_mail_unicode_recipient_encoding_error(protocol, unicode_address):
+    exchange_greetings(protocol, esmtp=False)
+    protocol.mail('hello@example.org')
+    feed_bytes(protocol, b'250 OK\r\n', 250, 'OK', ClientState.mailtx)
+
+    exc = pytest.raises(SMTPProtocolViolation, protocol.recipient, unicode_address)
+    exc.match("^The address 'héllö@example.org' requires UTF-8")
+
+
 def test_send_mail_escape_dots(protocol):
     exchange_greetings(protocol)
     start_mail_tx(protocol)
@@ -158,7 +194,7 @@ def test_double_command(protocol):
 def test_authentication_required(protocol):
     exchange_greetings(protocol)
     call_protocol_method(protocol, lambda: protocol.mail('foo@bar.com'),
-                         b'MAIL FROM:<foo@bar.com> BODY=8BITMIME\r\n')
+                         b'MAIL FROM:<foo@bar.com> BODY=8BITMIME SMTPUTF8\r\n')
     feed_bytes(protocol, b'530 Authentication required\r\n', 530, 'Authentication required')
 
 
@@ -277,7 +313,7 @@ def test_helo_error(protocol, error_code):
 def test_mail_error(protocol, error_code):
     exchange_greetings(protocol)
     call_protocol_method(protocol, lambda: protocol.mail('foo@bar'),
-                         b'MAIL FROM:<foo@bar> BODY=8BITMIME\r\n')
+                         b'MAIL FROM:<foo@bar> BODY=8BITMIME SMTPUTF8\r\n')
     feed_bytes(protocol, f'{error_code} Error\r\n'.encode(), error_code, 'Error')
 
 
@@ -285,7 +321,7 @@ def test_mail_error(protocol, error_code):
 def test_rcpt_error(protocol, error_code):
     exchange_greetings(protocol)
     call_protocol_method(protocol, lambda: protocol.mail('foo@bar'),
-                         b'MAIL FROM:<foo@bar> BODY=8BITMIME\r\n')
+                         b'MAIL FROM:<foo@bar> BODY=8BITMIME SMTPUTF8\r\n')
     feed_bytes(protocol, b'250 OK\r\n', 250, 'OK')
     call_protocol_method(protocol, lambda: protocol.recipient('foo@bar'), b'RCPT TO:<foo@bar>\r\n')
     feed_bytes(protocol, f'{error_code} Error\r\n'.encode(), error_code, 'Error')
@@ -295,7 +331,7 @@ def test_rcpt_error(protocol, error_code):
 def test_start_data_error(protocol, error_code):
     exchange_greetings(protocol)
     call_protocol_method(protocol, lambda: protocol.mail('foo@bar'),
-                         b'MAIL FROM:<foo@bar> BODY=8BITMIME\r\n')
+                         b'MAIL FROM:<foo@bar> BODY=8BITMIME SMTPUTF8\r\n')
     feed_bytes(protocol, b'250 OK\r\n', 250, 'OK')
     call_protocol_method(protocol, lambda: protocol.recipient('foo@bar'), b'RCPT TO:<foo@bar>\r\n')
     feed_bytes(protocol, b'250 OK\r\n', 250, 'OK')