Repository: awdeorio/mailmerge

Issue:
CSV with BOM
It looks like Excel will sometimes save a file with Byte Order Mark (BOM).  When the mailmerge database contains a BOM, it can't seem to find the first header key.

`mailmerge_template.txt`:
```
TO: {{email}}
FROM: My Self <myself@mydomain.com>

Hello {{name}}
```

`mailmerge_database.csv`:
```
name,email
My Name,to@test.com
```

```console
$ file mailmerge_database.csv 
mailmerge_database.csv: UTF-8 Unicode (with BOM) text, with CRLF line terminators
```


[mailmerge_database_with_BOM.csv.txt](https://github.com/awdeorio/mailmerge/files/4621656/mailmerge_database_with_BOM.csv.txt)


Discussion:

Patch:
diff --git a/mailmerge/__main__.py b/mailmerge/__main__.py
index 2f24cb1..a364548 100644
--- a/mailmerge/__main__.py
+++ b/mailmerge/__main__.py
@@ -261,13 +261,18 @@ def read_csv_database(database_path):
     We'll use a class to modify the csv library's default dialect ('excel') to
     enable strict syntax checking.  This will trigger errors for things like
     unclosed quotes.
+
+    We open the file with the utf-8-sig encoding, which skips a byte order mark
+    (BOM), if any.  Sometimes Excel will save CSV files with a BOM.  See Issue
+    #93 https://github.com/awdeorio/mailmerge/issues/93
+
     """
     class StrictExcel(csv.excel):
         # Our helper class is really simple
         # pylint: disable=too-few-public-methods, missing-class-docstring
         strict = True
 
-    with database_path.open() as database_file:
+    with database_path.open(encoding="utf-8-sig") as database_file:
         reader = csv.DictReader(database_file, dialect=StrictExcel)
         try:
             for row in reader:


Test patch:
diff --git a/tests/test_main.py b/tests/test_main.py
index 60c01cd..cab716e 100644
--- a/tests/test_main.py
+++ b/tests/test_main.py
@@ -7,11 +7,13 @@ pytest tmpdir docs:
 http://doc.pytest.org/en/latest/tmpdir.html#the-tmpdir-fixture
 """
 import copy
+import shutil
 import re
 from pathlib import Path
 import textwrap
 import click.testing
 from mailmerge.__main__ import main
+from . import utils
 
 
 def test_no_options(tmpdir):
@@ -799,3 +801,61 @@ def test_other_mime_type(tmpdir):
         >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
         >>> This was a dry run.  To send messages, use the --no-dry-run option.
     """)  # noqa: E501
+
+
+def test_database_bom(tmpdir):
+    """Bug fix CSV with a byte order mark (BOM).
+
+    It looks like Excel will sometimes save a file with Byte Order Mark
+    (BOM). When the mailmerge database contains a BOM, it can't seem to find
+    the first header key.
+    https://github.com/awdeorio/mailmerge/issues/93
+
+    """
+    # Simple template
+    template_path = Path(tmpdir/"mailmerge_template.txt")
+    template_path.write_text(textwrap.dedent("""\
+        TO: {{email}}
+        FROM: My Self <myself@mydomain.com>
+
+        Hello {{name}}
+    """))
+
+    # Copy database containing a BOM
+    database_path = Path(tmpdir/"mailmerge_database.csv")
+    database_with_bom = utils.TESTDATA/"mailmerge_database_with_BOM.csv"
+    shutil.copyfile(database_with_bom, database_path)
+
+    # Simple unsecure server config
+    config_path = Path(tmpdir/"mailmerge_server.conf")
+    config_path.write_text(textwrap.dedent("""\
+        [smtp_server]
+        host = open-smtp.example.com
+        port = 25
+    """))
+
+    # Run mailmerge
+    runner = click.testing.CliRunner()
+    with tmpdir.as_cwd():
+        result = runner.invoke(main, ["--output-format", "text"])
+    assert not result.exception
+    assert result.exit_code == 0
+
+    # Verify output
+    stdout = copy.deepcopy(result.output)
+    stdout = re.sub(r"Date:.+", "Date: REDACTED", stdout, re.MULTILINE)
+    assert stdout == textwrap.dedent("""\
+        >>> message 1
+        TO: to@test.com
+        FROM: My Self <myself@mydomain.com>
+        MIME-Version: 1.0
+        Content-Type: text/plain; charset="us-ascii"
+        Content-Transfer-Encoding: 7bit
+        Date: REDACTED
+
+        Hello My Name
+
+        >>> message 1 sent
+        >>> Limit was 1 message.  To remove the limit, use the --no-limit option.
+        >>> This was a dry run.  To send messages, use the --no-dry-run option.
+    """)  # noqa: E501
diff --git a/tests/testdata/mailmerge_database_with_BOM.csv b/tests/testdata/mailmerge_database_with_BOM.csv
new file mode 100644
index 0000000..fb9c879
--- /dev/null
+++ b/tests/testdata/mailmerge_database_with_BOM.csv
@@ -0,0 +1,2 @@
+﻿name,email
+My Name,to@test.com