Repository: pypi/warehouse

Issue:
Support package upload with a SHA-256 checksum
Currently package upload respects `md5_digest` as a form parameter and [validates the digest](https://github.com/pypa/warehouse/blob/master/warehouse/legacy/api/pypi.py#L646).

Discussion took place earlier this year (or last year) about migrating to using SHA-256 as the digest algorithm instead. We should begin to validate that and support it so tools (e.g., twine) can start uploading packages using this digest algorithm.



Discussion:

Resolving patch:
diff --git a/warehouse/legacy/api/pypi.py b/warehouse/legacy/api/pypi.py
--- a/warehouse/legacy/api/pypi.py
+++ b/warehouse/legacy/api/pypi.py
@@ -342,9 +342,19 @@ class MetadataForm(forms.Form):
     comment = wtforms.StringField(validators=[wtforms.validators.Optional()])
     md5_digest = wtforms.StringField(
         validators=[
-            wtforms.validators.DataRequired(),
+            wtforms.validators.Optional(),
         ],
     )
+    sha256_digest = wtforms.StringField(
+        validators=[
+            wtforms.validators.Optional(),
+            wtforms.validators.Regexp(
+                r"^[A-F0-9]{64}$",
+                re.IGNORECASE,
+                message="Must be a valid, hex encoded, SHA256 message digest.",
+            ),
+        ]
+    )
 
     # Legacy dependency information
     requires = ListField(
@@ -417,6 +427,12 @@ def full_validate(self):
                     "The only valid Python version for a sdist is 'source'."
                 )
 
+        # We *must* have at least one digest to verify against.
+        if not self.md5_digest.data and not self.sha256_digest.data:
+            raise wtforms.validators.ValidationError(
+                "Must include at least one message digest.",
+            )
+
 
 _safe_zipnames = re.compile(r"(purelib|platlib|headers|scripts|data).+", re.I)
 
@@ -741,15 +757,21 @@ def file_upload(request):
                 for hasher in file_hashes.values():
                     hasher.update(chunk)
 
-        # Actually verify that the md5 hash of the file matches the expected
-        # md5 hash. We probably don't actually need to use hmac.compare_digest
-        # here since both the md5_digest and the file whose file_hash we've
-        # computed comes from the remote user, however better safe than sorry.
-        if not hmac.compare_digest(
-                form.md5_digest.data, file_hashes["md5"].hexdigest()):
+        # Actually verify the digests that we've gotten. We're going to use
+        # hmac.compare_digest even though we probably don't actually need to
+        # because it's better safe than sorry. In the case of multiple digests
+        # we expect them all to be given.
+        if not all([
+            hmac.compare_digest(
+                getattr(form, "{}_digest".format(digest_name)).data.lower(),
+                digest_value.hexdigest().lower(),
+            )
+            for digest_name, digest_value in file_hashes.items()
+            if getattr(form, "{}_digest".format(digest_name)).data
+        ]):
             raise _exc_with_message(
                 HTTPBadRequest,
-                "The MD5 digest supplied does not match a digest calculated "
+                "The digest supplied does not match a digest calculated "
                 "from the uploaded file."
             )
 


Test patch:
diff --git a/tests/unit/legacy/api/test_pypi.py b/tests/unit/legacy/api/test_pypi.py
--- a/tests/unit/legacy/api/test_pypi.py
+++ b/tests/unit/legacy/api/test_pypi.py
@@ -297,8 +297,25 @@ class TestMetadataForm:
     @pytest.mark.parametrize(
         "data",
         [
-            {"filetype": "sdist"},
-            {"filetpye": "bdist_wheel", "pyversion": "3.4"},
+            {"filetype": "sdist", "md5_digest": "bad"},
+            {
+                "filetpye": "bdist_wheel",
+                "pyversion": "3.4",
+                "md5_digest": "bad",
+            },
+            {"filetype": "sdist", "sha256_digest": "bad"},
+            {
+                "filetpye": "bdist_wheel",
+                "pyversion": "3.4",
+                "sha256_digest": "bad",
+            },
+            {"filetype": "sdist", "md5_digest": "bad", "sha256_digest": "bad"},
+            {
+                "filetpye": "bdist_wheel",
+                "pyversion": "3.4",
+                "md5_digest": "bad",
+                "sha256_digest": "bad",
+            },
         ],
     )
     def test_full_validate_valid(self, data):
@@ -489,6 +506,7 @@ def test_fails_invalid_version(self, pyramid_config, pyramid_request,
                     "metadata_version": "1.2",
                     "name": "example",
                     "version": "1.0",
+                    "md5_digest": "bad",
                 },
                 "filetype: This field is required.",
             ),
@@ -509,6 +527,7 @@ def test_fails_invalid_version(self, pyramid_config, pyramid_request,
                     "version": "1.0",
                     "filetype": "bdist_wat",
                     "pyversion": "1.0",
+                    "md5_digest": "bad",
                 },
                 "filetype: Unknown type of file.",
             ),
@@ -524,7 +543,7 @@ def test_fails_invalid_version(self, pyramid_config, pyramid_request,
                 "'source'.",
             ),
 
-            # md5_digest errors.
+            # digest errors.
             (
                 {
                     "metadata_version": "1.2",
@@ -532,7 +551,7 @@ def test_fails_invalid_version(self, pyramid_config, pyramid_request,
                     "version": "1.0",
                     "filetype": "sdist",
                 },
-                "md5_digest: This field is required.",
+                "__all__: Must include at least one message digest.",
             ),
 
             # summary errors
@@ -625,9 +644,53 @@ def test_upload_cleans_unknown_values(self, pyramid_config, db_request):
 
         assert "name" not in db_request.POST
 
-    @pytest.mark.parametrize("has_signature", [True, False])
+    @pytest.mark.parametrize(
+        ("has_signature", "digests"),
+        [
+            (True, {"md5_digest": "335c476dc930b959dda9ec82bd65ef19"}),
+            (
+                True,
+                {
+                    "sha256_digest": (
+                        "4a8422abcc484a4086bdaa618c65289f749433b07eb433c51c4e3"
+                        "77143ff5fdb"
+                    ),
+                },
+            ),
+            (False, {"md5_digest": "335c476dc930b959dda9ec82bd65ef19"}),
+            (
+                False,
+                {
+                    "sha256_digest": (
+                        "4a8422abcc484a4086bdaa618c65289f749433b07eb433c51c4e3"
+                        "77143ff5fdb"
+                    ),
+                },
+            ),
+            (
+                True,
+                {
+                    "md5_digest": "335c476dc930b959dda9ec82bd65ef19",
+                    "sha256_digest": (
+                        "4a8422abcc484a4086bdaa618c65289f749433b07eb433c51c4e3"
+                        "77143ff5fdb"
+                    ),
+                },
+            ),
+            (
+                False,
+                {
+                    "md5_digest": "335c476dc930b959dda9ec82bd65ef19",
+                    "sha256_digest": (
+                        "4a8422abcc484a4086bdaa618c65289f749433b07eb433c51c4e3"
+                        "77143ff5fdb"
+                    ),
+                },
+            ),
+        ],
+    )
     def test_successful_upload(self, tmpdir, monkeypatch, pyramid_config,
-                               db_request, has_signature):
+                               db_request, has_signature, digests):
         monkeypatch.setattr(tempfile, "tempdir", str(tmpdir))
 
         pyramid_config.testing_securitypolicy(userid=1)
@@ -650,7 +713,6 @@ def test_successful_upload(self, tmpdir, monkeypatch, pyramid_config,
             "version": release.version,
             "filetype": "sdist",
             "pyversion": "source",
-            "md5_digest": "335c476dc930b959dda9ec82bd65ef19",
             "content": pretend.stub(
                 filename=filename,
                 file=io.BytesIO(b"A fake file."),
@@ -660,6 +722,7 @@ def test_successful_upload(self, tmpdir, monkeypatch, pyramid_config,
         db_request.POST.extend([
             ("classifiers", "Environment :: Other Environment"),
         ])
+        db_request.POST.update(digests)
 
         if has_signature:
             db_request.POST["gpg_signature"] = pretend.stub(
@@ -861,7 +924,41 @@ def test_upload_fails_with_invalid_classifier(self, pyramid_config,
             "valid choice for this field"
         )
 
-    def test_upload_fails_with_invalid_hash(self, pyramid_config, db_request):
+    @pytest.mark.parametrize(
+        "digests",
+        [
+            {"md5_digest": "bad"},
+            {
+                "sha256_digest": (
+                    "badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbad"
+                    "badbadb"
+                ),
+            },
+            {
+                "md5_digest": "bad",
+                "sha256_digest": (
+                    "badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbad"
+                    "badbadb"
+                ),
+            },
+            {
+                "md5_digest": "335c476dc930b959dda9ec82bd65ef19",
+                "sha256_digest": (
+                    "badbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbadbad"
+                    "badbadb"
+                ),
+            },
+            {
+                "md5_digest": "bad",
+                "sha256_digest": (
+                    "4a8422abcc484a4086bdaa618c65289f749433b07eb433c51c4e37714"
+                    "3ff5fdb"
+                ),
+            },
+        ],
+    )
+    def test_upload_fails_with_invalid_digest(self, pyramid_config, db_request,
+                                              digests):
         pyramid_config.testing_securitypolicy(userid=1)
 
         user = UserFactory.create()
@@ -876,13 +973,13 @@ def test_upload_fails_with_invalid_hash(self, pyramid_config, db_request):
             "name": project.name,
             "version": release.version,
             "filetype": "sdist",
-            "md5_digest": "nope!",
             "content": pretend.stub(
                 filename=filename,
                 file=io.BytesIO(b"A fake file."),
                 type="application/tar",
             ),
         })
+        db_request.POST.update(digests)
 
         with pytest.raises(HTTPBadRequest) as excinfo:
             pypi.file_upload(db_request)
@@ -891,7 +988,7 @@ def test_upload_fails_with_invalid_hash(self, pyramid_config, db_request):
 
         assert resp.status_code == 400
         assert resp.status == (
-            "400 The MD5 digest supplied does not match a digest calculated "
+            "400 The digest supplied does not match a digest calculated "
             "from the uploaded file."
         )