Repository: pypa/setuptools

Issue:
Include path in error message when get_metadata() raises e.g. UnicodeDecodeError
This issue is another manifestation of issue #1664 ("Include location of bad METADATA files in errors"), but for the `value.decode('utf-8')` line in `NullProvider.get_metadata()`: https://github.com/pypa/setuptools/blob/8aeff6b2c9a64e47ad2a22533d7e65c08cd4103f/pkg_resources/__init__.py#L1416-L1420

This came up recently in this pip issue (and probably other pip issues -- this is just one example): https://github.com/pypa/pip/issues/6566

Rather than re-opening #1664, it probably makes sense to have a new issue. It's unknown to me how many code paths in `pkg_resources` affect the larger issue. PR #1706 addressed the version-is-missing case.



Discussion:

Resolving patch:
diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py
--- a/pkg_resources/__init__.py
+++ b/pkg_resources/__init__.py
@@ -1416,8 +1416,17 @@ def has_metadata(self, name):
     def get_metadata(self, name):
         if not self.egg_info:
             return ""
-        value = self._get(self._fn(self.egg_info, name))
-        return value.decode('utf-8') if six.PY3 else value
+        path = self._get_metadata_path(name)
+        value = self._get(path)
+        if six.PY2:
+            return value
+        try:
+            return value.decode('utf-8')
+        except UnicodeDecodeError as exc:
+            # Include the path in the error message to simplify
+            # troubleshooting, and without changing the exception type.
+            exc.reason += ' in {} file at path: {}'.format(name, path)
+            raise
 
     def get_metadata_lines(self, name):
         return yield_lines(self.get_metadata(name))


Test patch:
diff --git a/pkg_resources/tests/test_pkg_resources.py b/pkg_resources/tests/test_pkg_resources.py
--- a/pkg_resources/tests/test_pkg_resources.py
+++ b/pkg_resources/tests/test_pkg_resources.py
@@ -18,6 +18,7 @@
     import mock
 
 from pkg_resources import DistInfoDistribution, Distribution, EggInfoDistribution
+from setuptools.extern import six
 from pkg_resources.extern.six.moves import map
 from pkg_resources.extern.six import text_type, string_types
 
@@ -191,6 +192,59 @@ def test_setuptools_not_imported(self):
         subprocess.check_call(cmd)
 
 
+def make_test_distribution(metadata_path, metadata):
+    """
+    Make a test Distribution object, and return it.
+
+    :param metadata_path: the path to the metadata file that should be
+        created. This should be inside a distribution directory that should
+        also be created. For example, an argument value might end with
+        "<project>.dist-info/METADATA".
+    :param metadata: the desired contents of the metadata file, as bytes.
+    """
+    dist_dir = os.path.dirname(metadata_path)
+    os.mkdir(dist_dir)
+    with open(metadata_path, 'wb') as f:
+        f.write(metadata)
+    dists = list(pkg_resources.distributions_from_metadata(dist_dir))
+    dist, = dists
+
+    return dist
+
+
+def test_get_metadata__bad_utf8(tmpdir):
+    """
+    Test a metadata file with bytes that can't be decoded as utf-8.
+    """
+    filename = 'METADATA'
+    # Convert the tmpdir LocalPath object to a string before joining.
+    metadata_path = os.path.join(str(tmpdir), 'foo.dist-info', filename)
+    # Encode a non-ascii string with the wrong encoding (not utf-8).
+    metadata = 'née'.encode('iso-8859-1')
+    dist = make_test_distribution(metadata_path, metadata=metadata)
+
+    if six.PY2:
+        # In Python 2, get_metadata() doesn't do any decoding.
+        actual = dist.get_metadata(filename)
+        assert actual == metadata
+        return
+
+    # Otherwise, we are in the Python 3 case.
+    with pytest.raises(UnicodeDecodeError) as excinfo:
+        dist.get_metadata(filename)
+
+    exc = excinfo.value
+    actual = str(exc)
+    expected = (
+        # The error message starts with "'utf-8' codec ..." However, the
+        # spelling of "utf-8" can vary (e.g. "utf8") so we don't include it
+        "codec can't decode byte 0xe9 in position 1: "
+        'invalid continuation byte in METADATA file at path: '
+    )
+    assert expected in actual, 'actual: {}'.format(actual)
+    assert actual.endswith(metadata_path), 'actual: {}'.format(actual)
+
+
 # TODO: remove this in favor of Path.touch() when Python 2 is dropped.
 def touch_file(path):
     """