Repository: peterbe/hashin

Issue:
Rewrap 404 errors from pypi.org
E.g.:

```bash
▶ python hashin.py -r /tmp/r.txt requests djangu
Traceback (most recent call last):
  File "hashin.py", line 544, in <module>
    sys.exit(main())
  File "hashin.py", line 536, in main
    dry_run=args.dry_run,
  File "hashin.py", line 142, in run
    run_single_package(spec, requirements_file, *args, **kwargs)
  File "hashin.py", line 173, in run_single_package
    include_prereleases=include_prereleases,
  File "hashin.py", line 467, in get_package_hashes
    data = get_package_data(package, verbose)
  File "hashin.py", line 400, in get_package_data
    content = json.loads(_download(url))
  File "hashin.py", line 107, in _download
    r = urlopen(url)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 223, in urlopen
    return opener.open(url, data, timeout)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 532, in open
    response = meth(req, response)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 642, in http_response
    'http', request, response, code, msg, hdrs)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 564, in error
    result = self._call_chain(*args)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 504, in _call_chain
    result = func(*args)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 756, in http_error_302
    return self.parent.open(new, timeout=req.timeout)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 532, in open
    response = meth(req, response)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 642, in http_response
    'http', request, response, code, msg, hdrs)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 570, in error
    return self._call_chain(*args)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 504, in _call_chain
    result = func(*args)
  File "/usr/local/Cellar/python/3.6.5_1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/urllib/request.py", line 650, in http_error_default
    raise HTTPError(req.full_url, code, msg, hdrs, fp)
urllib.error.HTTPError: HTTP Error 404: Not Found
```

In this case, you don't know which of the packages that failed on you. Was it `requests` or `djangu`?

Discussion:

Patch:
diff --git a/README.rst b/README.rst
index d7d6ae3..e8684f3 100644
--- a/README.rst
+++ b/README.rst
@@ -254,6 +254,10 @@ Version History
 
 next
 
+  * In Python 3, if the package can't be found you get a more explicit exception
+    pointing out which package (URL) that failed.
+    See https://github.com/peterbe/hashin/issues/87
+
   * New flag ``--update-all`` (alias ``-u``) will parse the requirements file,
     ignore the version, and update all packages that have new versions.
     See https://github.com/peterbe/hashin/pull/88
diff --git a/hashin.py b/hashin.py
index cd471c1..98d5c44 100755
--- a/hashin.py
+++ b/hashin.py
@@ -20,6 +20,7 @@ from packaging.version import parse
 
 if sys.version_info >= (3,):
     from urllib.request import urlopen
+    from urllib.error import HTTPError
 else:
     from urllib import urlopen
 
@@ -99,12 +100,23 @@ class NoVersionsError(Exception):
     """When there are no valid versions found."""
 
 
+class PackageNotFoundError(Exception):
+    """When the package can't be found on pypi.org."""
+
+
 def _verbose(*args):
     print("* " + " ".join(args))
 
 
 def _download(url, binary=False):
-    r = urlopen(url)
+    try:
+        r = urlopen(url)
+    except HTTPError as exception:
+        status_code = exception.getcode()
+        if status_code == 404:
+            raise PackageNotFoundError(url)
+        raise PackageError("Download error. {0} on {1}".format(status_code, url))
+
     # Note that urlopen will, by default, follow redirects.
     status_code = r.getcode()
 
@@ -115,8 +127,10 @@ def _download(url, binary=False):
                 "No 'Location' header on {0} ({1})".format(url, status_code)
             )
         return _download(location)
+    elif status_code == 404:
+        raise PackageNotFoundError(url)
     elif status_code != 200:
-        raise PackageError("Package not found. {0} on {1}".format(status_code, url))
+        raise PackageError("Download error. {0} on {1}".format(status_code, url))
     if binary:
         return r.read()
     _, params = cgi.parse_header(r.headers.get("Content-Type", ""))


Test patch:
diff --git a/tests/test_cli.py b/tests/test_cli.py
index cae9004..6cd1d07 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -17,12 +17,15 @@ import hashin
 if sys.version_info >= (3,):
     # As in, Python 3
     from io import StringIO
+    from urllib.error import HTTPError
 
     STR_TYPE = str
 else:  # Python 2
     from StringIO import StringIO
 
     FileNotFoundError = IOError  # ugly but necessary
+    # Python 2 does not have this exception.
+    HTTPError = None
 
 
 @contextmanager
@@ -1282,6 +1285,39 @@ selenium==2.53.1 \
 
         self.assertEqual(result, expected)
 
+    @cleanup_tmpdir("hashin*")
+    @mock.patch("hashin.urlopen")
+    def test_get_package_hashes_package_not_found(self, murlopen):
+        def mocked_get(url, **options):
+            if url == "https://pypi.org/pypi/gobblygook/json":
+                if HTTPError:
+                    raise HTTPError(url, 404, "Page not found", {}, None)
+                else:
+                    return _Response({}, status_code=404)
+
+            if url == "https://pypi.org/pypi/troublemaker/json":
+                if HTTPError:
+                    raise HTTPError(url, 500, "Something went wrong", {}, None)
+                else:
+                    return _Response({}, status_code=500)
+
+            raise NotImplementedError(url)
+
+        murlopen.side_effect = mocked_get
+
+        with self.assertRaises(hashin.PackageNotFoundError) as cm:
+            hashin.get_package_hashes(
+                package="gobblygook", version="0.10", algorithm="sha256"
+            )
+        exception = cm.exception
+        self.assertEqual(str(exception), "https://pypi.org/pypi/gobblygook/json")
+
+        # Errors left as is if not a 404
+        with self.assertRaises(hashin.PackageError):
+            hashin.get_package_hashes(
+                package="troublemaker", version="0.10", algorithm="sha256"
+            )
+
     @cleanup_tmpdir("hashin*")
     @mock.patch("hashin.urlopen")
     def test_get_package_hashes_unknown_algorithm(self, murlopen):