Repository: pypa/bandersnatch (python)

Issue:
package json digest dict mapped to simple json hashes dict causes pip >23 to fail
blake2b_256 has been added to the digest dict in package json, pip does not know how to handle this hash name. Causing and exception when trying to install packages with pip23 or later. 

If you check https://pypi.org/simple/pip against https://pypi.org/simple/pip/23.0.1/json
the digest dict does not contain the same data as the hashes dict. 

```
$ pip install --force pip
Looking in indexes: https://<pypi mirror>/simple/
Collecting pip
  Using cached https://<pypi mirror>/packages/07/51/2c0959c5adf988c44d9e1e0d940f5b074516ecc87e96b1af25f59de9ba38/pip-23.0.1-py3-none-any.whl (2.1 MB)
ERROR: Unknown hash name: blake2b_256
```

This is caused by how bandersnatch discovers the hashes that should be applied to /simple/
https://github.com/pypa/bandersnatch/blob/f99542f17adcc5c35ddfbb62b03a84d43122793a/src/bandersnatch/simple.py#L192
```
def generate_json_simple_page(
        ...
        # Add release files into the JSON dict
        for r in release_files:
            package_json["files"].append(
                {
                    "filename": r["filename"],
                    "hashes": {
                        digest_name: digest_hash
                        for digest_name, digest_hash in r["digests"].items()
                    },
                    "requires-python": r.get("requires_python", ""),
                    "url": self._file_url_to_local_url(r["url"]),
                    "yanked": r.get("yanked", False),
                }
            )
```

Pull request:
Use the configurable digest_name for simple JSON
PEP 691 specifies that each file's hashes dictionary must contain only
hash algorithms that can be passed, without arguments, to hashlib.new().
PyPI has started using a "blake2b_256" key for digests, which is blake2b
with the digest size set to 32 bytes, and makes it so that contents of
"digests" can't just be reused for the hashes dictionary.

This commit uses the value of configuration option digest_name to pick
a single supported key/value pair from the digests for use in the hashes
dictionary.

Patch:
diff --git a/CHANGES.md b/CHANGES.md
index 27a083e..a3f6a67 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,3 +1,10 @@
+# Unreleased
+
+## Bug Fixes
+
+- Fix digest used for file hashes in PEP 691 simple JSON file output `PR #1442`
+  - The `digest_name` setting from configuration (default value: `sha256`) will now be used for both HTML and JSON files.
+
 # 6.2.0
 
 ## New Features
diff --git a/src/bandersnatch/configuration.py b/src/bandersnatch/configuration.py
index 00688fd..292f28c 100644
--- a/src/bandersnatch/configuration.py
+++ b/src/bandersnatch/configuration.py
@@ -7,7 +7,7 @@ import logging
 from pathlib import Path
 from typing import Any, Dict, List, NamedTuple, Optional, Type
 
-from .simple import SimpleFormat, get_format_value
+from .simple import SimpleDigest, SimpleFormat, get_digest_value, get_format_value
 
 logger = logging.getLogger("bandersnatch")
 
@@ -131,15 +131,17 @@ def validate_config_values(  # noqa: C901
     logger.info(f"Selected storage backend: {storage_backend_name}")
 
     try:
-        digest_name = config.get("mirror", "digest_name")
+        digest_name = get_digest_value(config.get("mirror", "digest_name"))
     except configparser.NoOptionError:
-        digest_name = "sha256"
-    if digest_name not in ("md5", "sha256"):
-        raise ValueError(
-            f"Supplied digest_name {digest_name} is not supported! Please "
-            + "update digest_name to one of ('sha256', 'md5') in the [mirror] "
-            + "section."
+        digest_name = SimpleDigest.SHA256
+        logger.debug(f"Using digest {digest_name} by default ...")
+    except ValueError as e:
+        logger.error(
+            f"Supplied digest_name {config.get('mirror', 'digest_name')} is "
+            + "not supported! Please update the digest_name in the [mirror] "
+            + "section of your config to a supported digest value."
         )
+        raise e
 
     try:
         cleanup = config.getboolean("mirror", "cleanup")
diff --git a/src/bandersnatch/simple.py b/src/bandersnatch/simple.py
index f003201..b5accd4 100644
--- a/src/bandersnatch/simple.py
+++ b/src/bandersnatch/simple.py
@@ -1,6 +1,7 @@
 import html
 import json
 import logging
+import sys
 from enum import Enum, auto
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Union
@@ -11,6 +12,11 @@ from .package import Package
 if TYPE_CHECKING:
     from .storage import Storage
 
+if sys.version_info >= (3, 11):
+    from enum import StrEnum
+else:
+    from .utils import StrEnum
+
 
 class SimpleFormats(NamedTuple):
     html: str
@@ -23,6 +29,16 @@ class SimpleFormat(Enum):
     JSON = auto()
 
 
+class SimpleDigests(NamedTuple):
+    sha256: str
+    md5: str
+
+
+class SimpleDigest(StrEnum):
+    SHA256 = "sha256"
+    MD5 = "md5"
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -32,17 +48,34 @@ class InvalidSimpleFormat(KeyError):
     pass
 
 
+class InvalidDigestFormat(ValueError):
+    """We don't have a valid digest choice from configuration"""
+
+    pass
+
+
 def get_format_value(format: str) -> SimpleFormat:
     try:
         return SimpleFormat[format.upper()]
     except KeyError:
-        valid_formats = [v.name for v in SimpleFormat].sort()
+        valid_formats = sorted([v.name for v in SimpleFormat])
         raise InvalidSimpleFormat(
             f"{format.upper()} is not a valid Simple API format. "
             + f"Valid Options: {valid_formats}"
         )
 
 
+def get_digest_value(digest: str) -> SimpleDigest:
+    try:
+        return SimpleDigest[digest.upper()]
+    except KeyError:
+        valid_digests = sorted([v.name for v in SimpleDigest])
+        raise InvalidDigestFormat(
+            f"{digest} is not a valid Simple API file hash digest. "
+            + f"Valid Options: {valid_digests}"
+        )
+
+
 class SimpleAPI:
     """Handle all Simple API file generation"""
 
@@ -56,12 +89,16 @@ class SimpleAPI:
         storage_backend: "Storage",
         format: Union[SimpleFormat, str],
         diff_file_list: List[Path],
-        digest_name: str,
+        digest_name: Union[SimpleDigest, str],
         hash_index: bool,
         root_uri: Optional[str],
     ) -> None:
         self.diff_file_list = diff_file_list
-        self.digest_name = digest_name
+        self.digest_name = (
+            get_digest_value(digest_name)
+            if isinstance(digest_name, str)
+            else digest_name
+        )
         self.format = get_format_value(format) if isinstance(format, str) else format
         self.hash_index = hash_index
         self.root_uri = root_uri
@@ -190,8 +227,7 @@ class SimpleAPI:
                 {
                     "filename": r["filename"],
                     "hashes": {
-                        digest_name: digest_hash
-                        for digest_name, digest_hash in r["digests"].items()
+                        self.digest_name: r["digests"][self.digest_name],
                     },
                     "requires-python": r.get("requires_python", ""),
                     "url": self._file_url_to_local_url(r["url"]),
diff --git a/src/bandersnatch/utils.py b/src/bandersnatch/utils.py
index e8deb4a..bff78f9 100644
--- a/src/bandersnatch/utils.py
+++ b/src/bandersnatch/utils.py
@@ -9,6 +9,7 @@ import shutil
 import sys
 import tempfile
 from datetime import datetime
+from enum import Enum
 from pathlib import Path
 from typing import IO, Any, Generator, List, Set, Union
 from urllib.parse import urlparse
@@ -37,6 +38,15 @@ USER_AGENT = user_agent()
 WINDOWS = bool(platform.system() == "Windows")
 
 
+class StrEnum(str, Enum):
+    """Enumeration class where members can be treated as strings."""
+
+    value: str
+
+    def __str__(self) -> str:
+        return self.value
+
+
 def make_time_stamp() -> str:
     """Helper function that returns a timestamp suitable for use
     in a filename on any OS"""


Test patch:
diff --git a/src/bandersnatch/tests/test_main.py b/src/bandersnatch/tests/test_main.py
index d69eebf..dcfed0a 100644
--- a/src/bandersnatch/tests/test_main.py
+++ b/src/bandersnatch/tests/test_main.py
@@ -128,7 +128,7 @@ def test_main_throws_exception_on_unsupported_digest_name(
     with pytest.raises(ValueError) as e:
         main(asyncio.new_event_loop())
 
-    assert "foobar is not supported" in str(e.value)
+    assert "foobar is not a valid" in str(e.value)
 
 
 @pytest.fixture
diff --git a/src/bandersnatch/tests/test_simple.py b/src/bandersnatch/tests/test_simple.py
index fb6b50b..9972530 100644
--- a/src/bandersnatch/tests/test_simple.py
+++ b/src/bandersnatch/tests/test_simple.py
@@ -6,8 +6,15 @@ from tempfile import TemporaryDirectory
 import pytest
 
 from bandersnatch import utils
+from bandersnatch.configuration import validate_config_values
 from bandersnatch.package import Package
-from bandersnatch.simple import InvalidSimpleFormat, SimpleAPI, SimpleFormat
+from bandersnatch.simple import (
+    InvalidDigestFormat,
+    InvalidSimpleFormat,
+    SimpleAPI,
+    SimpleDigest,
+    SimpleFormat,
+)
 from bandersnatch.storage import Storage
 from bandersnatch.tests.test_simple_fixtures import (
     EXPECTED_SIMPLE_GLOBAL_JSON_PRETTY,
@@ -20,16 +27,35 @@ from bandersnatch_storage_plugins.filesystem import FilesystemStorage
 
 def test_format_invalid() -> None:
     with pytest.raises(InvalidSimpleFormat):
-        SimpleAPI(Storage(), "l33t", [], "digest", False, None)
+        SimpleAPI(Storage(), "l33t", [], "sha256", False, None)
 
 
 def test_format_valid() -> None:
-    s = SimpleAPI(Storage(), "ALL", [], "digest", False, None)
+    s = SimpleAPI(Storage(), "ALL", [], "sha256", False, None)
     assert s.format == SimpleFormat.ALL
 
 
+def test_digest_invalid() -> None:
+    with pytest.raises(InvalidDigestFormat):
+        SimpleAPI(Storage(), "ALL", [], "digest", False, None)
+
+
+def test_digest_valid() -> None:
+    s = SimpleAPI(Storage(), "ALL", [], "md5", False, None)
+    assert s.digest_name == SimpleDigest.MD5
+
+
+def test_digest_config_default() -> None:
+    c = ConfigParser()
+    c.add_section("mirror")
+    config = validate_config_values(c)
+    s = SimpleAPI(Storage(), "ALL", [], config.digest_name, False, None)
+    assert config.digest_name.upper() in [v.name for v in SimpleDigest]
+    assert s.digest_name == SimpleDigest.SHA256
+
+
 def test_json_package_page() -> None:
-    s = SimpleAPI(Storage(), SimpleFormat.JSON, [], "digest", False, None)
+    s = SimpleAPI(Storage(), SimpleFormat.JSON, [], SimpleDigest.SHA256, False, None)
     p = Package("69")
     p._metadata = SIXTYNINE_METADATA
     assert EXPECTED_SIMPLE_SIXTYNINE_JSON == s.generate_json_simple_page(p)
@@ -44,7 +70,7 @@ def test_json_index_page() -> None:
     c.add_section("mirror")
     c["mirror"]["workers"] = "1"
     s = SimpleAPI(
-        FilesystemStorage(config=c), SimpleFormat.ALL, [], "digest", False, None
+        FilesystemStorage(config=c), SimpleFormat.ALL, [], "sha256", False, None
     )
     with TemporaryDirectory() as td:
         td_path = Path(td)
diff --git a/src/bandersnatch/tests/test_simple_fixtures.py b/src/bandersnatch/tests/test_simple_fixtures.py
index 7bf356b..bc7058c 100644
--- a/src/bandersnatch/tests/test_simple_fixtures.py
+++ b/src/bandersnatch/tests/test_simple_fixtures.py
@@ -109,7 +109,7 @@ SIXTYNINE_METADATA = {
 }
 
 EXPECTED_SIMPLE_SIXTYNINE_JSON = """\
-{"files": [{"filename": "69-0.69.tar.gz", "hashes": {"md5": "4328d962656395fbd3e730c9d30bb48c", "sha256": "5c11f48399f9b1bca802751513f1f97bff6ce97e6facb576b7729e1351453c10"}, "requires-python": ">=3.6", "url": "../../packages/d3/cc/95dc5434362bd333a1fec275231775d748315b26edf1e7e568e6f8660238/69-0.69.tar.gz", "yanked": false}, {"filename": "69-6.9.tar.gz", "hashes": {"md5": "ff4bf804ef3722a1fd8853a8a32513d4", "sha256": "0c8deb7c8574787283c3fc08b714ee63fd6752a38d13515a9d8508798d428597"}, "requires-python": ">=3.6", "url": "../../packages/7b/6e/7c4ce77c6ca092e94e19b78282b459e7f8270362da655cbc6a75eeb9cdd7/69-6.9.tar.gz", "yanked": false}], "meta": {"api-version": "1.0", "_last-serial": "10333928"}, "name": "69"}\
+{"files": [{"filename": "69-0.69.tar.gz", "hashes": {"sha256": "5c11f48399f9b1bca802751513f1f97bff6ce97e6facb576b7729e1351453c10"}, "requires-python": ">=3.6", "url": "../../packages/d3/cc/95dc5434362bd333a1fec275231775d748315b26edf1e7e568e6f8660238/69-0.69.tar.gz", "yanked": false}, {"filename": "69-6.9.tar.gz", "hashes": {"sha256": "0c8deb7c8574787283c3fc08b714ee63fd6752a38d13515a9d8508798d428597"}, "requires-python": ">=3.6", "url": "../../packages/7b/6e/7c4ce77c6ca092e94e19b78282b459e7f8270362da655cbc6a75eeb9cdd7/69-6.9.tar.gz", "yanked": false}], "meta": {"api-version": "1.0", "_last-serial": "10333928"}, "name": "69"}\
 """
 
 EXPECTED_SIMPLE_SIXTYNINE_JSON_PRETTY = """\
@@ -118,7 +118,6 @@ EXPECTED_SIMPLE_SIXTYNINE_JSON_PRETTY = """\
         {
             "filename": "69-0.69.tar.gz",
             "hashes": {
-                "md5": "4328d962656395fbd3e730c9d30bb48c",
                 "sha256": "5c11f48399f9b1bca802751513f1f97bff6ce97e6facb576b7729e1351453c10"
             },
             "requires-python": ">=3.6",
@@ -128,7 +127,6 @@ EXPECTED_SIMPLE_SIXTYNINE_JSON_PRETTY = """\
         {
             "filename": "69-6.9.tar.gz",
             "hashes": {
-                "md5": "ff4bf804ef3722a1fd8853a8a32513d4",
                 "sha256": "0c8deb7c8574787283c3fc08b714ee63fd6752a38d13515a9d8508798d428597"
             },
             "requires-python": ">=3.6",