Repository: jazzband/pip-tools

Issue:
NotADirectoryError when running --generate-hashes
Running `pip-compile` works flawless, but pip-compile `--generate-hashes` produces the following error:

> Traceback (most recent call last):
>   File "/home/ric/.local/bin/pip-compile", line 11, in <module>
>     sys.exit(cli())
>   File "/home/ric/.local/lib/python3.5/site-packages/click/core.py", line 722, in __call__
>     return self.main(*args, **kwargs)
>   File "/home/ric/.local/lib/python3.5/site-packages/click/core.py", line 697, in main
>     rv = self.invoke(ctx)
>   File "/home/ric/.local/lib/python3.5/site-packages/click/core.py", line 895, in invoke
>     return ctx.invoke(self.callback, **ctx.params)
>   File "/home/ric/.local/lib/python3.5/site-packages/click/core.py", line 535, in invoke
>     return callback(*args, **kwargs)
>   File "/home/ric/.local/lib/python3.5/site-packages/piptools/scripts/compile.py", line 186, in cli
>     hashes = resolver.resolve_hashes(results)
>   File "/home/ric/.local/lib/python3.5/site-packages/piptools/resolver.py", line 77, in resolve_hashes
>     return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs}
>   File "/home/ric/.local/lib/python3.5/site-packages/piptools/resolver.py", line 77, in <dictcomp>
>     return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs}
>   File "/home/ric/.local/lib/python3.5/site-packages/piptools/repositories/pypi.py", line 169, in get_hashes
>     for candidate in matching_candidates
>   File "/home/ric/.local/lib/python3.5/site-packages/piptools/repositories/pypi.py", line 169, in <setcomp>
>     for candidate in matching_candidates
>   File "/home/ric/.local/lib/python3.5/site-packages/piptools/repositories/pypi.py", line 176, in _get_file_hash
>     download_dir=tmpdir, only_download=True, session=self.session
>   File "/usr/lib/python3/dist-packages/pip/download.py", line 821, in unpack_url
>     hashes=hashes
>   File "/usr/lib/python3/dist-packages/pip/download.py", line 663, in unpack_http_url
>     unpack_file(from_path, location, content_type, link)
>   File "/usr/lib/python3/dist-packages/pip/utils/__init__.py", line 623, in unpack_file
>     untar_file(filename, location)
>   File "/usr/lib/python3/dist-packages/pip/utils/__init__.py", line 595, in untar_file
>     with open(path, 'wb') as destfp:
> NotADirectoryError: [Errno 20] Not a directory: '/tmp/tmprk7taktxbuild/LICENSE/LICENSE_STIX'
> 

##### Environment Versions

1. Ubuntu 16.04
1. Python 3.5.3
1. pip 9.0.1
1. pip-compile 1.9.0

This is my `requirements.in` file:

> 
> seaborn==0.7.1
> pandas==0.20.1
> numpy==1.12.1
> scipy==0.19.0
> tqdm==4.11.2
> matplotlib==2.0.2
> 


Discussion:
I encountered the same error – I believe it's `matplotlib` – see https://github.com/jheinen/matplotlib/blob/master/LICENSE/LICENSE_STIX .
Seems to be a duplicate of #438.

Resolving patch:
diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py
--- a/piptools/repositories/pypi.py
+++ b/piptools/repositories/pypi.py
@@ -4,9 +4,10 @@
 
 import hashlib
 import os
+from contextlib import contextmanager
 from shutil import rmtree
 
-from pip.download import unpack_url
+from pip.download import is_file_url, url_to_path
 from pip.index import PackageFinder
 from pip.req.req_set import RequirementSet
 from pip.wheel import Wheel
@@ -194,18 +195,38 @@ def get_hashes(self, ireq):
         }
 
     def _get_file_hash(self, location):
-        with TemporaryDirectory() as tmpdir:
-            unpack_url(
-                location, self.build_dir,
-                download_dir=tmpdir, only_download=True, session=self.session
-            )
-            files = os.listdir(tmpdir)
-            assert len(files) == 1
-            filename = os.path.abspath(os.path.join(tmpdir, files[0]))
-
-            h = hashlib.new(FAVORITE_HASH)
-            with open(filename, "rb") as fp:
-                for chunk in iter(lambda: fp.read(8096), b""):
-                    h.update(chunk)
-
+        h = hashlib.new(FAVORITE_HASH)
+        with open_local_or_remote_file(location, self.session) as fp:
+            for chunk in iter(lambda: fp.read(8096), b""):
+                h.update(chunk)
         return ":".join([FAVORITE_HASH, h.hexdigest()])
+
+
+@contextmanager
+def open_local_or_remote_file(link, session):
+    """
+    Open local or remote file for reading.
+
+    :type link: pip.index.Link
+    :type session: requests.Session
+    :raises ValueError: If link points to a local directory.
+    :return: a context manager to the opened file-like object
+    """
+    url = link.url_without_fragment
+
+    if is_file_url(link):
+        # Local URL
+        local_path = url_to_path(url)
+        if os.path.isdir(local_path):
+            raise ValueError("Cannot open directory for read: {}".format(url))
+        else:
+            with open(local_path, 'rb') as local_file:
+                yield local_file
+    else:
+        # Remote URL
+        headers = {"Accept-Encoding": "identity"}
+        response = session.get(url, headers=headers, stream=True)
+        try:
+            yield response.raw
+        finally:
+            response.close()


Test patch:
diff --git a/tests/test_repository_pypi.py b/tests/test_repository_pypi.py
--- a/tests/test_repository_pypi.py
+++ b/tests/test_repository_pypi.py
@@ -53,3 +53,12 @@ def test_generate_hashes_all_platforms(from_line):
     repository = PyPIRepository(pip_options, session)
     ireq = from_line('cffi==1.9.1')
     assert repository.get_hashes(ireq) == expected
+
+
+def test_generate_hashes_without_interfering_with_each_other(from_line):
+    pip_command = get_pip_command()
+    pip_options, _ = pip_command.parse_args([])
+    session = pip_command._build_session(pip_options)
+    repository = PyPIRepository(pip_options, session)
+    repository.get_hashes(from_line('cffi==1.9.1'))
+    repository.get_hashes(from_line('matplotlib==2.0.2'))