Repository: conda/conda-build

Issue:
Patch with non-ascii characters causes UnicodeDecodeError
I'm build a package for conda-forge on appveyor and it fails when applying a patch with non-ascii characters:

```
Downloading source to cache: ftfy-5.1.tar.gz
Downloading https://pypi.io/packages/source/f/ftfy/ftfy-5.1.tar.gz
Success
Applying patch: 'C:\\projects\\ftfy-feedstock\\recipe\\tests-fix-linesep.patch'
Traceback (most recent call last):
  File "C:\Miniconda35\Scripts\conda-build-script.py", line 5, in <module>
    sys.exit(conda_build.cli.main_build.main())
  File "C:\Miniconda35\lib\site-packages\conda_build\cli\main_build.py", line 334, in main
    execute(sys.argv[1:])
  File "C:\Miniconda35\lib\site-packages\conda_build\cli\main_build.py", line 325, in execute
    noverify=args.no_verify)
  File "C:\Miniconda35\lib\site-packages\conda_build\api.py", line 97, in build
    need_source_download=need_source_download, config=config)
  File "C:\Miniconda35\lib\site-packages\conda_build\build.py", line 1510, in build_tree
    config=config)
  File "C:\Miniconda35\lib\site-packages\conda_build\build.py", line 1058, in build
    config=config)
  File "C:\Miniconda35\lib\site-packages\conda_build\render.py", line 86, in parse_or_try_download
    source.provide(metadata, config=config)
  File "C:\Miniconda35\lib\site-packages\conda_build\source.py", line 544, in provide
    apply_patch(src_dir, join(metadata.path, patch), config, git)
  File "C:\Miniconda35\lib\site-packages\conda_build\source.py", line 499, in apply_patch
    patch_args[-1] = _ensure_unix_line_endings(path)
  File "C:\Miniconda35\lib\site-packages\conda_build\source.py", line 420, in _ensure_unix_line_endings
    for line in inputfile:
  File "C:\Miniconda35\lib\encodings\cp1252.py", line 23, in decode
    return codecs.charmap_decode(input,self.errors,decoding_table)[0]
UnicodeDecodeError: 'charmap' codec can't decode byte 0x90 in position 331: character maps to <undefined>
Command exited with code 1
```

I tried to set the envvar `PYTHONIOENCODING=UTF-8` but had no success.

Full log: https://ci.appveyor.com/project/conda-forge/ftfy-feedstock/build/1.0.11/job/soe86qk54ln43gtd
Patch file: https://github.com/rolando-contrib/ftfy-feedstock/blob/fix-win-build/recipe/tests-fix-linesep.patch


Discussion:
Thanks for reporting this issue.  We've had another recent report of the unix line ending conversion breaking things.  I'll look into it soon.
Sorry, I tinkered with this a bit, trying to get BeautifulSoup's UnicodeDammit to work.  I have no idea what I'm doing.

The best idea I have for a fallback path is to just try the line ending conversion, but if it fails, show a warning and return the original patch file unchanged.  Does this seem like it would work for you?

Resolving patch:
diff --git a/conda_build/source.py b/conda_build/source.py
--- a/conda_build/source.py
+++ b/conda_build/source.py
@@ -434,10 +434,10 @@ def get_repository_info(recipe_path):
 def _ensure_unix_line_endings(path):
     """Replace windows line endings with Unix.  Return path to modified file."""
     out_path = path + "_unix"
-    with open(path) as inputfile:
-        with open(out_path, "w") as outputfile:
+    with open(path, "rb") as inputfile:
+        with open(out_path, "wb") as outputfile:
             for line in inputfile:
-                outputfile.write(line.replace("\r\n", "\n"))
+                outputfile.write(line.replace(b"\r\n", b"\n"))
     return out_path
 
 
@@ -516,6 +516,7 @@ def apply_patch(src_dir, path, config, git=None):
         patch_strip_level = _guess_patch_strip_level(files, src_dir)
         patch_args = ['-p%d' % patch_strip_level, '-i', path]
         if sys.platform == 'win32':
+            patch_args.insert(0, '--binary')
             patch_args[-1] = _ensure_unix_line_endings(path)
         check_call_env([patch] + patch_args, cwd=src_dir)
         if sys.platform == 'win32' and os.path.exists(patch_args[-1]):


Test patch:
diff --git a/tests/test_patch.py b/tests/test_patch.py
--- a/tests/test_patch.py
+++ b/tests/test_patch.py
@@ -1,6 +1,9 @@
 import os
+import tempfile
 
-from conda_build.source import _guess_patch_strip_level, apply_patch
+from conda_build.source import (
+    _ensure_unix_line_endings, _guess_patch_strip_level, apply_patch,
+)
 
 
 def test_patch_strip_level(testing_workdir, monkeypatch):
@@ -54,3 +57,16 @@ def test_patch(testing_workdir, testing_config):
         with open('file-modification.txt', 'r') as modified:
             lines = modified.readlines()
         assert lines[0] == '43770\n'
+
+
+def test_ensure_unix_line_endings_with_nonutf8_characters():
+    in_path = tempfile.mktemp()
+    with open(in_path, "wb") as fp:
+        fp.write(b"\xf1\r\n")  # tilde-n encoded in latin1
+
+    out_path = _ensure_unix_line_endings(in_path)
+    with open(out_path, "rb") as fp:
+        assert fp.read() == b"\xf1\n"
+
+    os.remove(in_path)
+    os.remove(out_path)
diff --git a/tests/test_source.py b/tests/test_source.py
--- a/tests/test_source.py
+++ b/tests/test_source.py
@@ -1,5 +1,6 @@
 import os
 import subprocess
+import tempfile
 
 import pytest