Repository: awdeorio/mailmerge

Issue:
Terminal output with utf8
Decode utf-8 messages printed to stdout.  The current behavior is to print base64-encoded messages, which are hard to verify.


## Example
`mailmerge_template.txt` with unicode characters, including emoji.
```
TO: to@test.com
FROM: from@test.com

Laȝamon 😀 klâwen
```

`mailmerge_database.csv`:
```
email
myself@mydomain.com
```

`mailmerge_server.conf`:
```yaml
[smtp_server]
host = open-smtp.example.com
port = 25
```

Running mailmerge results in the following stdout:
```console
$ mailmerge
>>> message 0
TO: to@test.com
FROM: from@test.com
MIME-Version: 1.0
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: base64
Date: Wed, 18 Dec 2019 15:02:04 -0000

TGHInWFtb24ga2zDondlbg==

>>> sent message 0
>>> Limit was 1 messages.  To remove the limit, use the --no-limit option.
>>> This was a dry run.  To send messages, use the --no-dry-run option.
```

Ideally, the output would look like this:
```console
Running mailmerge results in the following stdout:
```console
$ mailmerge
>>> message 0
TO: to@test.com
FROM: from@test.com
MIME-Version: 1.0
Content-Type: text/plain; charset="utf-8"
Content-Transfer-Encoding: base64
Date: Wed, 18 Dec 2019 15:02:04 -0000

Laȝamon 😀 klâwen

>>> sent message 0
>>> Limit was 1 messages.  To remove the limit, use the --no-limit option.
>>> This was a dry run.  To send messages, use the --no-dry-run option.
```


## Testing
- Update existing `test_main.py::test_stdout_utf8`
- Don't print attachment content, just filename.  This is already the case.
- Non-multipart message
- HTML and plaintext multipart
- UTF8 multipart message
- Attachment
- CC
- BCC



Discussion:

Patch:
diff --git a/.gitignore b/.gitignore
index 81682ce..2ebcb05 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,5 +25,5 @@ build/
 # Test tools
 /.tox/
 /tmp/
-/.coverage
+/.coverage*
 *,cover
diff --git a/mailmerge/__main__.py b/mailmerge/__main__.py
index 935d862..b59bb64 100644
--- a/mailmerge/__main__.py
+++ b/mailmerge/__main__.py
@@ -5,6 +5,7 @@ Andrew DeOrio <awdeorio@umich.edu>
 """
 from __future__ import print_function
 import sys
+import codecs
 import textwrap
 import click
 from .template_message import TemplateMessage
@@ -23,6 +24,11 @@ try:
 except ImportError:
     import csv
 
+# Python 2 UTF8 file redirection
+# http://www.macfreek.nl/memory/Encoding_of_Python_stdout
+if sys.stdout.encoding != 'UTF-8' and not hasattr(sys.stdout, "buffer"):
+    sys.stdout = codecs.getwriter('utf-8')(sys.stdout, 'strict')
+
 
 @click.command(context_settings={"help_option_names": ['-h', '--help']})
 @click.version_option()  # Auto detect version from setup.py
@@ -66,8 +72,15 @@ except ImportError:
     type=click.Path(),
     help="server configuration (mailmerge_server.conf)",
 )
+@click.option(
+    "--output-format", "output_format",
+    default="colorized",
+    type=click.Choice(["colorized", "text", "raw"]),
+    help="Output format (colorized).",
+)
 def main(sample, dry_run, limit, no_limit, resume,
-         template_path, database_path, config_path):
+         template_path, database_path, config_path,
+         output_format):
     """
     Mailmerge is a simple, command line mail merge tool.
 
@@ -101,11 +114,17 @@ def main(sample, dry_run, limit, no_limit, resume,
         for _, row in enumerate_range(csv_database, start, stop):
             sender, recipients, message = template_message.render(row)
             sendmail_client.sendmail(sender, recipients, message)
-            print(">>> message {}".format(message_num))
-            print(message.as_string())
-            for filename in get_attachment_filenames(message):
-                print(">>> attached {}".format(filename))
-            print(">>> sent message {}".format(message_num))
+            print_bright_white_on_cyan(
+                ">>> message {message_num}"
+                .format(message_num=message_num),
+                output_format,
+            )
+            print_message(message, output_format)
+            print_bright_white_on_cyan(
+                ">>> message {message_num} sent"
+                .format(message_num=message_num),
+                output_format,
+            )
             message_num += 1
     except MailmergeError as error:
         sys.exit(
@@ -269,20 +288,64 @@ def enumerate_range(iterable, start=0, stop=None):
         yield i, value
 
 
-def get_attachment_filenames(message):
-    """Return a list of attachment filenames."""
-    if message.get_content_maintype() != "multipart":
-        return []
+def print_cyan(string, output_format):
+    """Print string to stdout, optionally enabling color."""
+    if output_format == "colorized":
+        string = "\x1b[36m" + string + "\x1b(B\x1b[m"
+    print(string)
+
+
+def print_bright_white_on_cyan(string, output_format):
+    """Print string to stdout, optionally enabling color."""
+    if output_format == "colorized":
+        string = "\x1b[7m\x1b[1m\x1b[36m" + string + "\x1b(B\x1b[m"
+    print(string)
+
+
+def print_message(message, output_format):
+    """Print a message with colorized output."""
+    assert output_format in ["colorized", "text", "raw"]
 
-    filenames = []
+    if output_format == "raw":
+        print(message.as_string())
+        return
+
+    for header, value in message.items():
+        print("{header}: {value}".format(header=header, value=value))
+    print()
     for part in message.walk():
         if part.get_content_maintype() == "multipart":
-            continue
-        if part.get_content_maintype() == "text":
-            continue
-        if part.get("Content-Disposition") == "inline":
-            continue
-        if part.get("Content-Disposition") is None:
-            continue
-        filenames.append(part.get_filename())
-    return filenames
+            pass
+        elif part.get_content_maintype() == "text":
+            if message.is_multipart():
+                # Only print message part dividers for multipart messages
+                print_cyan(
+                    ">>> message part: {content_type}"
+                    .format(content_type=part.get_content_type()),
+                    output_format,
+                )
+            charset = str(part.get_charset())
+            print(part.get_payload(decode=True).decode(charset))
+            print()
+        elif is_attachment(part):
+            print_cyan(
+                ">>> message part: attachment {filename}"
+                .format(filename=part.get_filename()),
+                output_format,
+            )
+        else:
+            print_cyan(
+                ">>> message part: {content_type}"
+                .format(content_type=part.get_content_type()),
+                output_format,
+            )
+
+
+def is_attachment(part):
+    """Return True if message part looks like an attachment."""
+    return (
+        part.get_content_maintype() != "multipart" and
+        part.get_content_maintype() != "text" and
+        part.get("Content-Disposition") != "inline" and
+        part.get("Content-Disposition") is not None
+    )


Test patch:
diff --git a/tests/test_main.py b/tests/test_main.py
index 0ee9df9..e1b0d26 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -23,134 +23,6 @@ except ImportError:
 # pylint: disable=no-member
 
 
-def test_stdout(tmpdir):
-    """Verify stdout and stderr with dry run on simple input files."""
-    # Simple template
-    template_path = Path(tmpdir/"template.txt")
-    template_path.write_text(textwrap.dedent(u"""\
-        TO: {{email}}
-        SUBJECT: Testing mailmerge
-        FROM: My Self <myself@mydomain.com>
-
-        Hi, {{name}},
-
-        Your number is {{number}}.
-    """))
-
-    # Simple database
-    database_path = Path(tmpdir/"database.csv")
-    database_path.write_text(textwrap.dedent(u"""\
-        email,name,number
-        myself@mydomain.com,"Myself",17
-        bob@bobdomain.com,"Bob",42
-    """))
-
-    # Simple unsecure server config
-    config_path = Path(tmpdir/"server.conf")
-    config_path.write_text(textwrap.dedent(u"""\
-        [smtp_server]
-        host = open-smtp.example.com
-        port = 25
-    """))
-
-    # Run mailmerge
-    output = sh.mailmerge(
-        "--template", template_path,
-        "--database", database_path,
-        "--config", config_path,
-        "--no-limit",
-        "--dry-run",
-    )
-
-    # Verify mailmerge output.  We'll filter out the Date header because it
-    # won't match exactly.
-    stdout = output.stdout.decode("utf-8")
-    stderr = output.stderr.decode("utf-8")
-    assert stderr == ""
-    assert "Date:" in stdout
-    stdout = re.sub(r"Date.*\n", "", stdout)
-    assert stdout == textwrap.dedent(u"""\
-        >>> message 1
-        TO: myself@mydomain.com
-        SUBJECT: Testing mailmerge
-        FROM: My Self <myself@mydomain.com>
-        MIME-Version: 1.0
-        Content-Type: text/plain; charset="us-ascii"
-        Content-Transfer-Encoding: 7bit
-
-        Hi, Myself,
-
-        Your number is 17.
-        >>> sent message 1
-        >>> message 2
-        TO: bob@bobdomain.com
-        SUBJECT: Testing mailmerge
-        FROM: My Self <myself@mydomain.com>
-        MIME-Version: 1.0
-        Content-Type: text/plain; charset="us-ascii"
-        Content-Transfer-Encoding: 7bit
-
-        Hi, Bob,
-
-        Your number is 42.
-        >>> sent message 2
-        >>> This was a dry run.  To send messages, use the --no-dry-run option.
-        """)
-
-
-def test_stdout_utf8(tmpdir):
-    """Verify human-readable output when template contains utf-8."""
-    # Simple template
-    template_path = Path(tmpdir/"mailmerge_template.txt")
-    template_path.write_text(textwrap.dedent(u"""\
-        TO: to@test.com
-        FROM: from@test.com
-
-        Laȝamon 😀 klâwen
-    """))
-
-    # Simple database
-    database_path = Path(tmpdir/"mailmerge_database.csv")
-    database_path.write_text(textwrap.dedent(u"""\
-        email
-        myself@mydomain.com
-    """))
-
-    # Simple unsecure server config
-    config_path = Path(tmpdir/"mailmerge_server.conf")
-    config_path.write_text(textwrap.dedent(u"""\
-        [smtp_server]
-        host = open-smtp.example.com
-        port = 25
-    """))
-
-    # Run mailmerge with defaults, which includes dry-run
-    with tmpdir.as_cwd():
-        output = sh.mailmerge()
-
-    # Verify mailmerge output.  We'll filter out the Date header because it
-    # won't match exactly.
-    stdout = output.stdout.decode("utf-8")
-    stderr = output.stderr.decode("utf-8")
-    assert stderr == ""
-    assert "Date:" in stdout
-    stdout = re.sub(r"Date.*\n", "", stdout)
-    assert stdout == textwrap.dedent(u"""\
-        >>> message 1
-        TO: to@test.com
-        FROM: from@test.com
-        MIME-Version: 1.0
-        Content-Type: text/plain; charset="utf-8"
-        Content-Transfer-Encoding: base64
-
-        TGHInWFtb24g8J+YgCBrbMOid2Vu
-
-        >>> sent message 1
-        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
-        >>> This was a dry run.  To send messages, use the --no-dry-run option.
-    """)  # noqa: E501
-
-
 def test_no_options(tmpdir):
     """Verify help message when called with no options.
 
@@ -218,7 +90,7 @@ def test_defaults(tmpdir):
         sh.mailmerge("--sample")
         output = sh.mailmerge()
     assert output.stderr.decode("utf-8") == ""
-    assert "sent message 1" in output
+    assert "message 1 sent" in output
     assert "Limit was 1 message" in output
     assert "This was a dry run" in output
 
@@ -288,8 +160,8 @@ def test_limit_combo(tmpdir):
     with tmpdir.as_cwd():
         output = sh.mailmerge("--no-limit", "--limit", "1")
     assert output.stderr.decode("utf-8") == ""
-    assert "sent message 1" in output
-    assert "sent message 2" in output
+    assert "message 1 sent" in output
+    assert "message 2 sent" in output
     assert "Limit was 1" not in output
 
 
@@ -491,15 +363,14 @@ def test_attachment(tmpdir):
 
     # Run mailmerge
     with tmpdir.as_cwd():
-        output = sh.mailmerge()
+        output = sh.mailmerge("--output-format", "text")
 
     # Verify output
     assert output.stderr.decode("utf-8") == ""
+    assert ">>> message part: text/plain" in output
     assert "Hello world" in output  # message
-    assert "SGVsbG8gd29ybGQK" in output  # attachment1
-    assert "SGVsbG8gbWFpbG1lcmdlCg" in output  # attachment2
-    assert "attached attachment1.txt" in output
-    assert "attached attachment2.txt" in output
+    assert ">>> message part: attachment attachment1.txt" in output
+    assert ">>> message part: attachment attachment2.txt" in output
 
 
 def test_utf8_template(tmpdir):
@@ -534,11 +405,30 @@ def test_utf8_template(tmpdir):
         "--database", database_path,
         "--config", config_path,
         "--dry-run",
+        "--output-format", "text",
     )
 
+    # Remove the Date string, which will be different each time
+    stdout = output.stdout.decode("utf-8")
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+
     # Verify output
     assert output.stderr.decode("utf-8") == ""
-    assert u"TGHInWFtb24g8J+YgCBrbMOid2Vu" in output
+    assert stdout == textwrap.dedent(u"""\
+        >>> message 1
+        TO: to@test.com
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="utf-8"
+        Content-Transfer-Encoding: base64
+        Date: REDACTED
+
+        Laȝamon 😀 klâwen
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
 
 
 def test_utf8_database(tmpdir):
@@ -569,11 +459,29 @@ def test_utf8_database(tmpdir):
 
     # Run mailmerge
     with tmpdir.as_cwd():
-        output = sh.mailmerge()
+        output = sh.mailmerge("--output-format", "text")
+
+    # Remove the Date string, which will be different each time
+    stdout = output.stdout.decode("utf-8")
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
 
     # Verify output
     assert output.stderr.decode("utf-8") == ""
-    assert u"TGHInWFtb24g8J+YgCBrbMOid2Vu" in output
+    assert stdout == textwrap.dedent(u"""\
+        >>> message 1
+        TO: to@test.com
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="utf-8"
+        Content-Transfer-Encoding: base64
+        Date: REDACTED
+
+        Laȝamon 😀 klâwen
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
 
 
 def test_complicated(tmpdir):
@@ -644,7 +552,10 @@ def test_complicated(tmpdir):
 
     # Run mailmerge in tmpdir with defaults, which includes dry run
     with tmpdir.as_cwd():
-        output = sh.mailmerge("--no-limit")
+        output = sh.mailmerge(
+            "--no-limit",
+            "--output-format", "raw",
+        )
 
     # Decode output and remove date
     stdout = output.stdout.decode("utf-8")
@@ -717,9 +628,7 @@ def test_complicated(tmpdir):
         aGVsbG8sbWFpbG1lcmdlCg==
 
         --boundary--
-        >>> attached attachment1.txt
-        >>> attached attachment2.csv
-        >>> sent message 1
+        >>> message 1 sent
         >>> message 2
         TO: Lazamon<two@test.com>
         FROM: from@test.com
@@ -763,48 +672,11 @@ def test_complicated(tmpdir):
         aGVsbG8sbWFpbG1lcmdlCg==
 
         --boundary--
-        >>> attached attachment1.txt
-        >>> attached attachment2.csv
-        >>> sent message 2
+        >>> message 2 sent
         >>> This was a dry run.  To send messages, use the --no-dry-run option.
     """)
 
 
-def test_english(tmpdir):
-    """Verify correct English, message vs. messages."""
-    # Blank message
-    template_path = Path(tmpdir/"mailmerge_template.txt")
-    template_path.write_text(textwrap.dedent(u"""\
-        TO: to@test.com
-        FROM: from@test.com
-    """))
-
-    # Database with 2 entries
-    database_path = Path(tmpdir/"mailmerge_database.csv")
-    database_path.write_text(textwrap.dedent(u"""\
-        dummy
-        1
-        2
-    """))
-
-    # Simple unsecure server config
-    config_path = Path(tmpdir/"mailmerge_server.conf")
-    config_path.write_text(textwrap.dedent(u"""\
-        [smtp_server]
-        host = open-smtp.example.com
-        port = 25
-    """))
-
-    # Run mailmerge with several limits
-    with tmpdir.as_cwd():
-        output = sh.mailmerge("--limit", "0")
-        assert "Limit was 0 messages." in output
-        output = sh.mailmerge("--limit", "1")
-        assert "Limit was 1 message." in output
-        output = sh.mailmerge("--limit", "2")
-        assert "Limit was 2 messages." in output
-
-
 def test_resume(tmpdir):
     """Verify --resume option starts "in the middle" of the database."""
     # Simple template
@@ -841,9 +713,9 @@ def test_resume(tmpdir):
     stderr = output.stderr.decode("utf-8")
     assert stderr == ""
     assert "hello" not in stdout
-    assert "sent message 1" not in stdout
+    assert "message 1 sent" not in stdout
     assert "world" in stdout
-    assert "sent message 2" in stdout
+    assert "message 2 sent" in stdout
 
 
 def test_resume_too_small(tmpdir):
@@ -995,3 +867,68 @@ def test_resume_hint_on_csv_error(tmpdir):
     stderr = error.value.stderr.decode("utf-8")
     assert stdout == ""
     assert "--resume 2" in stderr
+
+
+def test_other_mime_type(tmpdir):
+    """Verify output with a MIME type that's not text or an attachment."""
+    # Template containing a pdf
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: {{email}}
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: multipart/alternative; boundary="boundary"
+
+        --boundary
+        Content-Type: text/plain; charset=us-ascii
+
+        Hello world
+
+        --boundary
+        Content-Type: application/pdf
+
+        DUMMY
+    """))
+
+    # Simple database with two entries
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        email
+        one@test.com
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge
+    with tmpdir.as_cwd():
+        output = sh.mailmerge()
+
+    # Verify output
+    stdout = output.stdout.decode("utf-8")
+    stderr = output.stderr.decode("utf-8")
+    assert stderr == ""
+    stdout = stdout.replace("\n\n\n\n", "\n\n\n")
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+    assert stdout == textwrap.dedent(u"""\
+        \x1b[7m\x1b[1m\x1b[36m>>> message 1\x1b(B\x1b[m
+        TO: one@test.com
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: multipart/alternative; boundary="boundary"
+        Date: REDACTED
+
+        \x1b[36m>>> message part: text/plain\x1b(B\x1b[m
+        Hello world
+
+
+        \x1b[36m>>> message part: application/pdf\x1b(B\x1b[m
+        \x1b[7m\x1b[1m\x1b[36m>>> message 1 sent\x1b(B\x1b[m
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
diff --git a/tests/test_main_output.py b/tests/test_main_output.py
new file mode 100644
index 0000000..781a5db
--- /dev/null
+++ b/tests/test_main_output.py
@@ -0,0 +1,444 @@
+# coding=utf-8
+# Python 2 source containing unicode https://www.python.org/dev/peps/pep-0263/
+"""
+System tests focused on CLI output.
+
+Andrew DeOrio <awdeorio@umich.edu>
+
+pytest tmpdir docs:
+http://doc.pytest.org/en/latest/tmpdir.html#the-tmpdir-fixture
+"""
+import os
+import re
+import textwrap
+import sh
+import pytest
+
+# Python 2 pathlib support requires backport
+try:
+    from pathlib2 import Path
+except ImportError:
+    from pathlib import Path
+
+# The sh library triggers lot of false no-member errors
+# pylint: disable=no-member
+
+
+def test_stdout(tmpdir):
+    """Verify stdout and stderr with dry run on simple input files."""
+    # Simple template
+    template_path = Path(tmpdir/"template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: {{email}}
+        SUBJECT: Testing mailmerge
+        FROM: My Self <myself@mydomain.com>
+
+        Hi, {{name}},
+
+        Your number is {{number}}.
+    """))
+
+    # Simple database
+    database_path = Path(tmpdir/"database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        email,name,number
+        myself@mydomain.com,"Myself",17
+        bob@bobdomain.com,"Bob",42
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge
+    output = sh.mailmerge(
+        "--template", template_path,
+        "--database", database_path,
+        "--config", config_path,
+        "--no-limit",
+        "--dry-run",
+        "--output-format", "text",
+    )
+
+    # Verify mailmerge output.  We'll filter out the Date header because it
+    # won't match exactly.
+    stdout = output.stdout.decode("utf-8")
+    stderr = output.stderr.decode("utf-8")
+    assert stderr == ""
+    assert "Date:" in stdout
+    stdout = re.sub(r"Date.*\n", "", stdout)
+    assert stdout == textwrap.dedent(u"""\
+        >>> message 1
+        TO: myself@mydomain.com
+        SUBJECT: Testing mailmerge
+        FROM: My Self <myself@mydomain.com>
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="us-ascii"
+        Content-Transfer-Encoding: 7bit
+
+        Hi, Myself,
+
+        Your number is 17.
+
+        >>> message 1 sent
+        >>> message 2
+        TO: bob@bobdomain.com
+        SUBJECT: Testing mailmerge
+        FROM: My Self <myself@mydomain.com>
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="us-ascii"
+        Content-Transfer-Encoding: 7bit
+
+        Hi, Bob,
+
+        Your number is 42.
+
+        >>> message 2 sent
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+        """)
+
+
+def test_stdout_utf8(tmpdir):
+    """Verify human-readable output when template contains utf-8."""
+    # Simple template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: to@test.com
+        FROM: from@test.com
+
+        Laȝamon 😀 klâwen
+    """))
+
+    # Simple database
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        email
+        myself@mydomain.com
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge with defaults, which includes dry-run
+    with tmpdir.as_cwd():
+        output = sh.mailmerge("--output-format", "text")
+
+    # Verify mailmerge output.  We'll filter out the Date header because it
+    # won't match exactly.
+    stdout = output.stdout.decode("utf-8")
+    stderr = output.stderr.decode("utf-8")
+    assert stderr == ""
+    assert "Date:" in stdout
+    stdout = re.sub(r"Date.*\n", "", stdout)
+    assert stdout == textwrap.dedent(u"""\
+        >>> message 1
+        TO: to@test.com
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="utf-8"
+        Content-Transfer-Encoding: base64
+
+        Laȝamon 😀 klâwen
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
+
+
+def test_stdout_utf8_redirect(tmpdir):
+    """Verify utf-8 output is properly encoded when redirected.
+
+    UTF-8 print fails when redirecting stdout under Pythnon 2
+    http://blog.mathieu-leplatre.info/python-utf-8-print-fails-when-redirecting-stdout.html
+    """
+    # Simple template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: to@test.com
+        FROM: from@test.com
+
+        Laȝamon 😀 klâwen
+    """))
+
+    # Simple database
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        email
+        myself@mydomain.com
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge.  We only care that no exceptions occur.  Note that we
+    # can't use the sh.mailmerge() here, even with its redirection utility,
+    # because it doesn't accurately recreate the conditions of the bug where
+    # the redirect destination lacks utf-8 encoding.
+    with tmpdir.as_cwd():
+        exit_code = os.system("mailmerge > mailmerge.out")
+    assert exit_code == 0
+
+
+def test_english(tmpdir):
+    """Verify correct English, message vs. messages."""
+    # Blank message
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: to@test.com
+        FROM: from@test.com
+    """))
+
+    # Database with 2 entries
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        dummy
+        1
+        2
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge with several limits
+    with tmpdir.as_cwd():
+        output = sh.mailmerge("--limit", "0")
+        assert "Limit was 0 messages." in output
+        output = sh.mailmerge("--limit", "1")
+        assert "Limit was 1 message." in output
+        output = sh.mailmerge("--limit", "2")
+        assert "Limit was 2 messages." in output
+
+
+def test_output_format_bad(tmpdir):
+    """Verify bad output format."""
+    with tmpdir.as_cwd(), pytest.raises(sh.ErrorReturnCode_2) as error:
+        sh.mailmerge("--output-format", "bad")
+    stdout = error.value.stdout.decode("utf-8")
+    stderr = error.value.stderr.decode("utf-8")
+    assert stdout == ""
+    assert 'Invalid value for "--output-format"' in stderr
+    assert "invalid choice: bad." in stderr
+
+
+def test_output_format_raw(tmpdir):
+    """Verify raw output format."""
+    # Attachment
+    attachment_path = Path(tmpdir/"attachment.txt")
+    attachment_path.write_text(u"Hello world\n")
+
+    # Simple template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: {{email}}
+        FROM: from@test.com
+
+        Laȝamon 😀 klâwen
+    """))
+
+    # Simple database
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        email
+        to@test.com
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge
+    with tmpdir.as_cwd():
+        output = sh.mailmerge("--output-format", "raw")
+    stdout = output.stdout.decode("utf-8")
+    stderr = output.stderr.decode("utf-8")
+
+    # Remove the Date string, which will be different each time
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+
+    # Verify output
+    assert stderr == ""
+    assert stdout == textwrap.dedent("""\
+        >>> message 1
+        TO: to@test.com
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="utf-8"
+        Content-Transfer-Encoding: base64
+        Date: REDACTED
+
+        TGHInWFtb24g8J+YgCBrbMOid2Vu
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
+
+
+def test_output_format_text(tmpdir):
+    """Verify text output format."""
+    # Attachment
+    attachment_path = Path(tmpdir/"attachment.txt")
+    attachment_path.write_text(u"Hello world\n")
+
+    # Simple template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: {{email}}
+        FROM: from@test.com
+
+        Laȝamon 😀 klâwen
+    """))
+
+    # Simple database
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        email
+        to@test.com
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge
+    with tmpdir.as_cwd():
+        output = sh.mailmerge("--output-format", "text")
+    stdout = output.stdout.decode("utf-8")
+    stderr = output.stderr.decode("utf-8")
+
+    # Remove the Date string, which will be different each time
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+
+    # Verify output
+    assert stderr == ""
+    assert stdout == textwrap.dedent(u"""\
+        >>> message 1
+        TO: to@test.com
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="utf-8"
+        Content-Transfer-Encoding: base64
+        Date: REDACTED
+
+        Laȝamon 😀 klâwen
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
+
+
+def test_output_format_colorized(tmpdir):
+    """Verify colorized output format."""
+    # Attachment
+    attachment_path = Path(tmpdir/"attachment.txt")
+    attachment_path.write_text(u"Hello world\n")
+
+    # HTML template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent(u"""\
+        TO: {{email}}
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: multipart/alternative; boundary="boundary"
+
+        This is a MIME-encoded message. If you are seeing this, your mail
+        reader is old.
+
+        --boundary
+        Content-Type: text/plain; charset=us-ascii
+
+        Laȝamon 😀 klâwen
+
+        --boundary
+        Content-Type: text/html; charset=us-ascii
+
+        <html>
+          <body>
+            <p>Laȝamon 😀 klâwen</p>
+          </body>
+        </html>
+    """))
+
+    # Simple database
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_path.write_text(textwrap.dedent(u"""\
+        email
+        to@test.com
+    """))
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent(u"""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge
+    with tmpdir.as_cwd():
+        output = sh.mailmerge("--output-format", "colorized")
+    stdout = output.stdout.decode("utf-8")
+    stderr = output.stderr.decode("utf-8")
+
+    # Remove the Date string, which will be different each time
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+
+    # The long output string below is the correct answer with Python 3.  With
+    # Python 2, we get a few differences in newlines.  We'll just query-replace
+    # those known mismatches so that the equality test passes.
+    stdout = stdout.replace("\n\n\n\n", "\n\n\n")
+
+    # Verify output.  The funny looking character sequences are colors.
+    assert stderr == ""
+    assert stdout == textwrap.dedent(u"""\
+        \x1b[7m\x1b[1m\x1b[36m>>> message 1\x1b(B\x1b[m
+        TO: to@test.com
+        FROM: from@test.com
+        MIME-Version: 1.0
+        Content-Type: multipart/alternative; boundary="boundary"
+        Date: REDACTED
+
+        \x1b[36m>>> message part: text/plain\x1b(B\x1b[m
+        La\u021damon \U0001f600 kl\xe2wen
+
+
+        \x1b[36m>>> message part: text/html\x1b(B\x1b[m
+        <html>
+          <body>
+            <p>La\u021damon \U0001f600 kl\xe2wen</p>
+          </body>
+        </html>
+
+        \x1b[7m\x1b[1m\x1b[36m>>> message 1 sent\x1b(B\x1b[m
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501