Repository: jazzband/pip-tools

Issue:
UnicodeDecodeError 'ascii' codec can't decode byte 0xe3 in position 66: ordinal not in range(128)
Running pip-compile causes a UnicodeDecodeError

##### Environment Versions

1. Debian Jessie Docker
1. Python version: Python 2.7.14
1. pip version:  pip 9.0.1 from /usr/local/lib/python2.7/site-packages (python 2.7)
1. pip-tools version: pip-compile, version 1.9.0

##### Steps to replicate

1. `pip-compile -U`

##### Expected result

requirements.txt updated

##### Actual result

```
Traceback (most recent call last):
  File "/usr/local/bin/pip-compile", line 11, in <module>
    sys.exit(cli())
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/usr/local/lib/python2.7/site-packages/piptools/scripts/compile.py", line 184, in cli
    results = resolver.resolve(max_rounds=max_rounds)
  File "/usr/local/lib/python2.7/site-packages/piptools/resolver.py", line 107, in resolve
    has_changed, best_matches = self._resolve_one_round()
  File "/usr/local/lib/python2.7/site-packages/piptools/resolver.py", line 195, in _resolve_one_round
    for dep in self._iter_dependencies(best_match):
  File "/usr/local/lib/python2.7/site-packages/piptools/resolver.py", line 274, in _iter_dependencies
    dependencies = self.repository.get_dependencies(ireq)
  File "/usr/local/lib/python2.7/site-packages/piptools/repositories/pypi.py", line 145, in get_dependencies
    self._dependencies_cache[ireq] = reqset._prepare_file(self.finder, ireq)
  File "/usr/local/lib/python2.7/site-packages/pip/req/req_set.py", line 620, in _prepare_file
    session=self.session, hashes=hashes)
  File "/usr/local/lib/python2.7/site-packages/pip/download.py", line 821, in unpack_url
    hashes=hashes
  File "/usr/local/lib/python2.7/site-packages/pip/download.py", line 663, in unpack_http_url
    unpack_file(from_path, location, content_type, link)
  File "/usr/local/lib/python2.7/site-packages/pip/utils/__init__.py", line 605, in unpack_file
    untar_file(filename, location)
  File "/usr/local/lib/python2.7/site-packages/pip/utils/__init__.py", line 551, in untar_file
    path = os.path.join(location, fn)
  File "/usr/local/lib/python2.7/posixpath.py", line 73, in join
    path += '/' + b
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe3 in position 66: ordinal not in range(128)
```


Discussion:
Same here 
in my case the error I found is coming from ansible, putting a print just before the line shows these filenames:

```
/tmp/tmpf9NBhMbuild/ansible test/integration/targets/unarchive/meta
/tmp/tmpf9NBhMbuild/ansible test/integration/targets/unarchive/meta/main.yml
/tmp/tmpf9NBhMbuild/ansible test/integration/targets/unarchive/tasks
/tmp/tmpf9NBhMbuild/ansible test/integration/targets/unarchive/tasks/main.yml
/tmp/tmpf9NBhMbuild/ansible test/integration/targets/unarchive/files
/tmp/tmpf9NBhMbuild/ansible test/integration/targets/unarchive/files/foo.txt
/tmp/tmpf9NBhMbuild/ansible test/integration/targets/unarchive/files/test-unarchive-nonascii-くくららととみみ.tar.gz
Traceback (most recent call last):
  File "/usr/local/bin/pip-compile", line 11, in <module>
    sys.exit(cli())
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/usr/local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/usr/local/lib/python2.7/site-packages/piptools/scripts/compile.py", line 184, in cli
    results = resolver.resolve(max_rounds=max_rounds)
  File "/usr/local/lib/python2.7/site-packages/piptools/resolver.py", line 107, in resolve
    has_changed, best_matches = self._resolve_one_round()
  File "/usr/local/lib/python2.7/site-packages/piptools/resolver.py", line 195, in _resolve_one_round
    for dep in self._iter_dependencies(best_match):
  File "/usr/local/lib/python2.7/site-packages/piptools/resolver.py", line 274, in _iter_dependencies
    dependencies = self.repository.get_dependencies(ireq)
  File "/usr/local/lib/python2.7/site-packages/piptools/repositories/pypi.py", line 145, in get_dependencies
    self._dependencies_cache[ireq] = reqset._prepare_file(self.finder, ireq)
  File "/usr/local/lib/python2.7/site-packages/pip/req/req_set.py", line 620, in _prepare_file
    session=self.session, hashes=hashes)
  File "/usr/local/lib/python2.7/site-packages/pip/download.py", line 821, in unpack_url
    hashes=hashes
  File "/usr/local/lib/python2.7/site-packages/pip/download.py", line 663, in unpack_http_url
    unpack_file(from_path, location, content_type, link)
  File "/usr/local/lib/python2.7/site-packages/pip/utils/__init__.py", line 606, in unpack_file
    untar_file(filename, location)
  File "/usr/local/lib/python2.7/site-packages/pip/utils/__init__.py", line 552, in untar_file
    path = os.path.join(location, fn)
  File "/usr/local/lib/python2.7/posixpath.py", line 73, in join
    path += '/' + b
UnicodeDecodeError: 'ascii' codec can't decode byte 0xe3 in position 66: ordinal not in range(128)
```

* it's non ascii filenames in the package causing it to error
* unicode filenames should be valid
* it's definitely pip-tools issue since pip install -U ansible works
* seems to be similar to #433 but the fix provided there does not fix this
work around was to upgrade ansible manually and then continue using pip-tools
I confirm that this problem can be reproduced by installing ansible via pip-compile. If ansible is already installed manually the problem does not happen.

Resolving patch:
diff --git a/piptools/_compat/tempfile.py b/piptools/_compat/tempfile.py
--- a/piptools/_compat/tempfile.py
+++ b/piptools/_compat/tempfile.py
@@ -1,6 +1,5 @@
 # coding: utf-8
-from __future__ import (absolute_import, division, print_function,
-                        unicode_literals)
+from __future__ import absolute_import, division, print_function
 
 import os as _os
 import sys as _sys
diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py
--- a/piptools/repositories/pypi.py
+++ b/piptools/repositories/pypi.py
@@ -17,7 +17,7 @@
 
 from ..cache import CACHE_DIR
 from ..exceptions import NoCandidateFound
-from ..utils import (is_pinned_requirement, lookup_table,
+from ..utils import (fs_str, is_pinned_requirement, lookup_table,
                      make_install_requirement, pip_version_info)
 from .base import BaseRepository
 
@@ -83,16 +83,16 @@ def __init__(self, pip_options, session):
 
         # Setup file paths
         self.freshen_build_caches()
-        self._download_dir = os.path.join(CACHE_DIR, 'pkgs')
-        self._wheel_download_dir = os.path.join(CACHE_DIR, 'wheels')
+        self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs'))
+        self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels'))
 
     def freshen_build_caches(self):
         """
         Start with fresh build/source caches.  Will remove any old build
         caches from disk automatically.
         """
-        self._build_dir = TemporaryDirectory('build')
-        self._source_dir = TemporaryDirectory('source')
+        self._build_dir = TemporaryDirectory(fs_str('build'))
+        self._source_dir = TemporaryDirectory(fs_str('source'))
 
     @property
     def build_dir(self):
diff --git a/piptools/utils.py b/piptools/utils.py
--- a/piptools/utils.py
+++ b/piptools/utils.py
@@ -58,7 +58,9 @@ def make_install_requirement(name, version, extras, constraint=False):
         # Sort extras for stability
         extras_string = "[{}]".format(",".join(sorted(extras)))
 
-    return InstallRequirement.from_line('{}{}=={}'.format(name, extras_string, str(version)), constraint=constraint)
+    return InstallRequirement.from_line(
+        str('{}{}=={}'.format(name, extras_string, version)),
+        constraint=constraint)
 
 
 def format_requirement(ireq, marker=None):
@@ -208,3 +210,30 @@ def dedup(iterable):
     order-reserved.
     """
     return iter(OrderedDict.fromkeys(iterable))
+
+
+def fs_str(string):
+    """
+    Convert given string to a correctly encoded filesystem string.
+
+    On Python 2, if the input string is unicode, converts it to bytes
+    encoded with the filesystem encoding.
+
+    On Python 3 returns the string as is, since Python 3 uses unicode
+    paths and the input string shouldn't be bytes.
+
+    >>> fs_str(u'some path component/Something')
+    'some path component/Something'
+    >>> assert isinstance(fs_str('whatever'), str)
+    >>> assert isinstance(fs_str(u'whatever'), str)
+
+    :type string: str|unicode
+    :rtype: str
+    """
+    if isinstance(string, str):
+        return string
+    assert not isinstance(string, bytes)
+    return string.encode(_fs_encoding)
+
+
+_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()


Test patch:
diff --git a/tests/test_repositories.py b/tests/test_repositories.py
new file mode 100644
--- /dev/null
+++ b/tests/test_repositories.py
@@ -0,0 +1,62 @@
+from mock import MagicMock, patch
+from pip.index import PackageFinder
+from pip.req import InstallRequirement
+
+from piptools.repositories.pypi import PyPIRepository
+from piptools.scripts.compile import get_pip_command
+
+
+def test_pypirepo_build_dir_is_str():
+    assert isinstance(get_pypi_repository().build_dir, str)
+
+
+def test_pypirepo_source_dir_is_str():
+    assert isinstance(get_pypi_repository().source_dir, str)
+
+
+def test_pypirepo_calls_reqset_with_str_paths():
+    """
+    Make sure that paths passed to RequirementSet init are str.
+
+    Passing unicode paths on Python 2 could make pip fail later on
+    unpack, if the package contains non-ASCII file names, because
+    non-ASCII str and unicode paths cannot be combined.
+    """
+    with patch('piptools.repositories.pypi.RequirementSet') as mocked_init:
+        repo = get_pypi_repository()
+        ireq = InstallRequirement.from_line('ansible==2.4.0.0')
+
+        # Setup a mock object to be returned from the RequirementSet call
+        mocked_reqset = MagicMock()
+        mocked_init.return_value = mocked_reqset
+
+        # Do the call
+        repo.get_dependencies(ireq)
+
+        # Check that RequirementSet init is called with correct type arguments
+        assert mocked_init.call_count == 1
+        (init_call_args, init_call_kwargs) = mocked_init.call_args
+        assert isinstance(init_call_args[0], str)
+        assert isinstance(init_call_args[1], str)
+        assert isinstance(init_call_kwargs.get('download_dir'), str)
+        assert isinstance(init_call_kwargs.get('wheel_download_dir'), str)
+
+        # Check that _prepare_file is called correctly
+        assert mocked_reqset._prepare_file.call_count == 1
+        (pf_call_args, pf_call_kwargs) = mocked_reqset._prepare_file.call_args
+        (called_with_finder, called_with_ireq) = pf_call_args
+        assert isinstance(called_with_finder, PackageFinder)
+        assert called_with_ireq == ireq
+        assert not pf_call_kwargs
+
+
+def get_pypi_repository():
+    """
+    Get a PyPIRepository object for the tests.
+
+    :rtype: PyPIRepository
+    """
+    pip_command = get_pip_command()
+    pip_options = pip_command.parse_args([])[0]
+    session = pip_command._build_session(pip_options)
+    return PyPIRepository(pip_options, session)