Repository: awdeorio/mailmerge

Issue:
Doesn't use charset=utf-8 when using markdown
Sending a message with special characters gives good results.

```
TO: {{email}}
SUBJECT: Testing mailmerge
FROM: test@example.com

Hi, {{name}},
æøå
```
outputs
```
>>> encoding utf-8
>>> message 0
TO: mail@larskarbo.no
SUBJECT: Testing mailmerge
FROM: test@example.com
MIME-Version: 1.0
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: base64
Date: Fri, 13 Dec 2019 20:41:08 -0000

SGksIE15c2VsZiwKw6bDuMOl
```

Notice that `charset` here is set ut `utf-8` and message renders well in email client.

But when specifying markdown:
```
TO: {{email}}
SUBJECT: Testing mailmerge
FROM: test@example.com
CONTENT-TYPE: text/markdown

Hi, {{name}},
æøå
```
It outputs
```
>>> encoding utf-8
>>> message 0
MIME-Version: 1.0
SUBJECT: Testing mailmerge
Date: Fri, 13 Dec 2019 20:42:22 -0000
TO: mail@larskarbo.no
FROM: test@example.com
MIME-Version: 1.0
Content-Transfer-Encoding: base64
Content-Type: multipart/alternative; boundary="===============3629053266709230733=="

--===============3629053266709230733==
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit

SGksIE15c2VsZiwKw6bDuMOl

--===============3629053266709230733==
Content-Type: text/html; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 7bit

<html><body><p>SGksIE15c2VsZiwKw6bDuMOl</p></body></html>
--===============3629053266709230733==--
```

Notice that `charset` here is set ut `us-ascii`, and message shows up as SGksIE15c2VsZiwKw6bDuMOl in the client.

Discussion:

Patch:
diff --git a/mailmerge/template_message.py b/mailmerge/template_message.py
index 0366d6e..a442cfc 100644
--- a/mailmerge/template_message.py
+++ b/mailmerge/template_message.py
@@ -93,38 +93,67 @@ class TemplateMessage(object):
 
     def _make_message_multipart(self):
         """Convert a message into a multipart message."""
-        if not self._message.is_multipart():
-            multipart_message = email.mime.multipart.MIMEMultipart(
-                'alternative')
-            for header_key in set(self._message.keys()):
-                # Preserve duplicate headers
-                values = self._message.get_all(header_key, failobj=[])
-                for value in values:
-                    multipart_message[header_key] = value
-            original_text = self._message.get_payload()
-            multipart_message.attach(email.mime.text.MIMEText(original_text))
-            self._message = multipart_message
+        # Do nothing if message already multipart
+        if self._message.is_multipart():
+            return
+
+        # Create empty multipart message
+        multipart_message = email.mime.multipart.MIMEMultipart('alternative')
+
+        # Copy headers, preserving duplicate headers
+        for header_key in set(self._message.keys()):
+            values = self._message.get_all(header_key, failobj=[])
+            for value in values:
+                multipart_message[header_key] = value
+
+        # Copy text, preserving original encoding
+        original_text = self._message.get_payload(decode=True)
+        original_encoding = str(self._message.get_charset())
+        multipart_message.attach(email.mime.text.MIMEText(
+            original_text,
+            _charset=original_encoding,
+        ))
+
+        # Replace original message with multipart message
+        self._message = multipart_message
 
     def _transform_markdown(self):
         """Convert markdown in message text to HTML."""
+        # Do nothing if Content-Type is not text/markdown
         if not self._message['Content-Type'].startswith("text/markdown"):
             return
 
+        # Remove the markdown Content-Type header, it's non-standard for email
         del self._message['Content-Type']
-        # Convert the text from markdown and then make the message multipart
+
+        # Make sure the message is multipart.  We need a multipart message so
+        # that we can add an HTML part containing rendered Markdown.
         self._make_message_multipart()
-        for payload_item in set(self._message.get_payload()):
-            # Assume the plaintext item is formatted with markdown.
-            # Add corresponding HTML version of the item as the last part of
-            # the multipart message (as per RFC 2046)
-            if payload_item['Content-Type'].startswith('text/plain'):
-                original_text = payload_item.get_payload()
-                html_text = markdown.markdown(original_text)
-                html_payload = future.backports.email.mime.text.MIMEText(
-                    "<html><body>{}</body></html>".format(html_text),
-                    "html",
-                )
-                self._message.attach(html_payload)
+
+        # Extract unrendered text and encoding.  We assume that the first
+        # plaintext payload is formatted with Markdown.
+        for mimetext in self._message.get_payload():
+            if mimetext['Content-Type'].startswith('text/plain'):
+                encoding = str(mimetext.get_charset())
+                text = mimetext.get_payload(decode=True).decode(encoding)
+                break
+        assert encoding
+        assert text
+
+        # Render Markdown to HTML and add the HTML as the last part of the
+        # multipart message as per RFC 2046.
+        #
+        # Note: We need to use u"..." to ensure that unicode string
+        # substitution works properly in Python 2.
+        #
+        # https://docs.python.org/3/library/email.mime.html#email.mime.text.MIMEText
+        html = markdown.markdown(text)
+        payload = future.backports.email.mime.text.MIMEText(
+            u"<html><body>{}</body></html>".format(html),
+            _subtype="html",
+            _charset=encoding,
+        )
+        self._message.attach(payload)
 
     def _transform_attachments(self):
         """Parse Attachment headers and add attachments."""


Test patch:
diff --git a/tests/test_template_message.py b/tests/test_template_message.py
index 1dc59f1..0baa86b 100644
--- a/tests/test_template_message.py
+++ b/tests/test_template_message.py
@@ -74,19 +74,59 @@ def test_markdown():
 
     # Ensure that the first part is plaintext and the last part
     # is HTML (as per RFC 2046)
-    plaintext_contenttype = payload[0]['Content-Type']
-    assert plaintext_contenttype.startswith("text/plain")
-    plaintext = payload[0].get_payload()
-    html_contenttype = payload[1]['Content-Type']
-    assert html_contenttype.startswith("text/html")
+    plaintext_part = payload[0]
+    assert plaintext_part['Content-Type'].startswith("text/plain")
+    plaintext_encoding = str(plaintext_part.get_charset())
+    plaintext = plaintext_part.get_payload(decode=True) \
+                              .decode(plaintext_encoding)
+
+    html_part = payload[1]
+    assert html_part['Content-Type'].startswith("text/html")
+    html_encoding = str(html_part.get_charset())
+    htmltext = html_part.get_payload(decode=True) \
+                        .decode(html_encoding)
 
     # Verify rendered Markdown
-    htmltext = payload[1].get_payload()
     rendered = markdown.markdown(plaintext)
     htmltext_correct = "<html><body>{}</body></html>".format(rendered)
     assert htmltext.strip() == htmltext_correct.strip()
 
 
+def test_markdown_encoding():
+    """Verify encoding is preserved when rendering a Markdown template.
+
+    See Issue #59 for a detailed explanation
+    https://github.com/awdeorio/mailmerge/issues/59
+    """
+    template_message = mailmerge.template_message.TemplateMessage(
+        utils.TESTDATA/"markdown_template_utf8.txt"
+    )
+    _, _, message = template_message.render({
+        "email": "myself@mydomain.com",
+        "name": "Myself",
+    })
+
+    # Message should contain an unrendered Markdown plaintext part and a
+    # rendered Markdown HTML part
+    plaintext_part, html_part = message.get_payload()
+
+    # Verify encodings
+    assert str(plaintext_part.get_charset()) == "utf-8"
+    assert str(html_part.get_charset()) == "utf-8"
+    assert plaintext_part["Content-Transfer-Encoding"] == "base64"
+    assert html_part["Content-Transfer-Encoding"] == "base64"
+
+    # Verify content, which is base64 encoded
+    plaintext = plaintext_part.get_payload().strip()
+    htmltext = html_part.get_payload().strip()
+    assert plaintext == "SGksIE15c2VsZiwKw6bDuMOl"
+    assert htmltext == (
+        "PGh0bWw+PGJvZHk+PHA+"
+        "SGksIE15c2VsZiwKw6bDuMOl"
+        "PC9wPjwvYm9keT48L2h0bWw+"
+    )
+
+
 def test_attachment():
     """Attachments should be sent as part of the email."""
     template_message = mailmerge.template_message.TemplateMessage(
@@ -165,7 +205,17 @@ def test_utf8_template():
     # NOTE: to decode a base46-encoded string:
     # print((str(base64.b64decode(payload), "utf-8")))
     payload = message.get_payload().replace("\n", "")
-    assert payload == 'RnJvbSB0aGUgVGFnZWxpZWQgb2YgV29sZnJhbSB2b24gRXNjaGVuYmFjaCAoTWlkZGxlIEhpZ2ggR2VybWFuKToKClPDrm5lIGtsw6J3ZW4gZHVyaCBkaWUgd29sa2VuIHNpbnQgZ2VzbGFnZW4sCmVyIHN0w65nZXQgw7tmIG1pdCBncsO0emVyIGtyYWZ0LAppY2ggc2loIGluIGdyw6J3ZW4gdMOkZ2Vsw65jaCBhbHMgZXIgd2lsIHRhZ2VuLApkZW4gdGFjLCBkZXIgaW0gZ2VzZWxsZXNjaGFmdAplcndlbmRlbiB3aWwsIGRlbSB3ZXJkZW4gbWFuLApkZW4gaWNoIG1pdCBzb3JnZW4gw65uIHZlcmxpZXouCmljaCBicmluZ2UgaW4gaGlubmVuLCBvYiBpY2gga2FuLgpzw65uIHZpbCBtYW5lZ2l1IHR1Z2VudCBtaWNoeiBsZWlzdGVuIGhpZXouCgpodHRwOi8vd3d3LmNvbHVtYmlhLmVkdS9+ZmRjL3V0Zjgv'  # noqa: E501 pylint: disable=line-too-long
+    assert payload == (
+        "RnJvbSB0aGUgVGFnZWxpZWQgb2YgV29sZnJhbSB2b24gRXNjaGVuYmFjaCAo"
+        "TWlkZGxlIEhpZ2ggR2VybWFuKToKClPDrm5lIGtsw6J3ZW4gZHVyaCBkaWUg"
+        "d29sa2VuIHNpbnQgZ2VzbGFnZW4sCmVyIHN0w65nZXQgw7tmIG1pdCBncsO0"
+        "emVyIGtyYWZ0LAppY2ggc2loIGluIGdyw6J3ZW4gdMOkZ2Vsw65jaCBhbHMg"
+        "ZXIgd2lsIHRhZ2VuLApkZW4gdGFjLCBkZXIgaW0gZ2VzZWxsZXNjaGFmdApl"
+        "cndlbmRlbiB3aWwsIGRlbSB3ZXJkZW4gbWFuLApkZW4gaWNoIG1pdCBzb3Jn"
+        "ZW4gw65uIHZlcmxpZXouCmljaCBicmluZ2UgaW4gaGlubmVuLCBvYiBpY2gg"
+        "a2FuLgpzw65uIHZpbCBtYW5lZ2l1IHR1Z2VudCBtaWNoeiBsZWlzdGVuIGhp"
+        "ZXouCgpodHRwOi8vd3d3LmNvbHVtYmlhLmVkdS9+ZmRjL3V0Zjgv"
+    )
 
 
 def test_utf8_database():
diff --git a/tests/testdata/markdown_template_utf8.txt b/tests/testdata/markdown_template_utf8.txt
new file mode 100644
index 0000000..aa8b14d
--- /dev/null
+++ b/tests/testdata/markdown_template_utf8.txt
@@ -0,0 +1,7 @@
+TO: {{email}}
+SUBJECT: Testing mailmerge
+FROM: test@example.com
+CONTENT-TYPE: text/markdown
+
+Hi, {{name}},
+æøå