Repository: pypi/warehouse

Issue:
Invalid API Token: invalid macaroon!r for url
**Describe the bug**
Possibly related to #6259
When using the same token generated from pypi a couple of months ago, now I got the error:
```
HTTPError: 403 Client Error: Invalid API Token: invalid macaroon!r for url: https://upload.pypi.org/legacy/
```
I figured it was based on an old key and re-generated a token, but I get the same error.

I verified that the version of `twine` did not change between the successful execution and the failure.

**Expected behavior**
No error on upload.

**To Reproduce**

TWINE_USERNAME=`__token__`
TWINE_PASSWORD=`<token generated at pypi.org>`

twine upload ...

**My Platform**
```
Build dist: xenial
Runtime kernel version: 4.15.0-1077-gcp
```
**Additional context**
https://travis-ci.com/github/pyproj4/pyproj-wheels/jobs/384840987


Discussion:
Hi @snowman2, this should be resolved by #8555 (and the error message improved by #8561), can you confirm?
Thanks for the fix - restarted build: https://travis-ci.com/github/pyproj4/pyproj-wheels/builds/184304607
It worked, thanks :+1:
Just got a similar error. Was working fine a week ago.

https://travis-ci.org/github/twilio/twilio-python/jobs/729123230
We also just got a similar error:
https://github.com/newrelic/newrelic-python-agent/runs/1146467559?check_suite_focus=true#step:9:14
Thanks, looks like https://github.com/pypa/warehouse/pull/8562 has re-introduced this (cc @ewjoachim).

Resolving patch:
diff --git a/warehouse/macaroons/caveats.py b/warehouse/macaroons/caveats.py
--- a/warehouse/macaroons/caveats.py
+++ b/warehouse/macaroons/caveats.py
@@ -33,7 +33,7 @@ def __call__(self, predicate):
 
 
 class V1Caveat(Caveat):
-    def verify_projects(self, projects) -> None:
+    def verify_projects(self, projects):
         # First, ensure that we're actually operating in
         # the context of a package.
         if not isinstance(self.verifier.context, Project):
@@ -43,13 +43,13 @@ def verify_projects(self, projects) -> None:
 
         project = self.verifier.context
         if project.normalized_name in projects:
-            return
+            return True
 
         raise InvalidMacaroon(
             f"project-scoped token is not valid for project '{project.name}'"
         )
 
-    def verify(self, predicate) -> None:
+    def verify(self, predicate):
         try:
             data = json.loads(predicate)
         except ValueError:
@@ -64,13 +64,13 @@ def verify(self, predicate) -> None:
 
         if permissions == "user":
             # User-scoped tokens behave exactly like a user's normal credentials.
-            return
+            return True
 
         projects = permissions.get("projects")
         if projects is None:
             raise InvalidMacaroon("invalid projects in predicate")
 
-        self.verify_projects(projects)
+        return self.verify_projects(projects)
 
 
 class Verifier:
@@ -81,19 +81,13 @@ def __init__(self, macaroon, context, principals, permission):
         self.permission = permission
         self.verifier = pymacaroons.Verifier()
 
-    def verify(self, key: str) -> None:
+    def verify(self, key):
         self.verifier.satisfy_general(V1Caveat(self))
-        self.verify_signature(key=key)
 
-    def verify_signature(self, key: str) -> None:
         try:
-            result = self.verifier.verify(self.macaroon, key)
+            return self.verifier.verify(self.macaroon, key)
         except (
             pymacaroons.exceptions.MacaroonInvalidSignatureException,
             Exception,  # https://github.com/ecordell/pymacaroons/issues/50
         ):
             raise InvalidMacaroon("invalid macaroon signature")
-
-        # This is dead code, the only hardcoded thing verify() can return is True
-        if not result:
-            raise InvalidMacaroon("invalid macaroon")
diff --git a/warehouse/macaroons/interfaces.py b/warehouse/macaroons/interfaces.py
--- a/warehouse/macaroons/interfaces.py
+++ b/warehouse/macaroons/interfaces.py
@@ -43,14 +43,6 @@ def verify(raw_macaroon, context, principals, permission):
         Raises InvalidMacaroon if the macaroon is not valid.
         """
 
-    def check_if_macaroon_exists(raw_macaroon):
-        """
-        Returns the database macaroon if the given raw (serialized) macaroon is
-        an existing valid macaroon, whatever its permissions.
-
-        Raises InvalidMacaroon otherwise.
-        """
-
     def create_macaroon(location, user_id, description, caveats):
         """
         Returns a new raw (serialized) macaroon. The description provided
diff --git a/warehouse/macaroons/services.py b/warehouse/macaroons/services.py
--- a/warehouse/macaroons/services.py
+++ b/warehouse/macaroons/services.py
@@ -97,11 +97,10 @@ def find_userid(self, raw_macaroon):
 
         return dm.user.id
 
-    def verify(self, raw_macaroon, context, principals, permission) -> None:
+    def verify(self, raw_macaroon, context, principals, permission):
         """
-        Passes if the given raw (serialized) macaroon is
+        Returns True if the given raw (serialized) macaroon is
         valid for the context, principals, and requested permission.
-        Updates the last_used date for the macaroon.
 
         Raises InvalidMacaroon if the macaroon is not valid.
         """
@@ -112,35 +111,11 @@ def verify(self, raw_macaroon, context, principals, permission) -> None:
             raise InvalidMacaroon("deleted or nonexistent macaroon")
 
         verifier = Verifier(m, context, principals, permission)
-        verifier.verify(dm.key)
-        dm.last_used = datetime.datetime.now()
+        if verifier.verify(dm.key):
+            dm.last_used = datetime.datetime.now()
+            return True
 
-    def check_if_macaroon_exists(self, raw_macaroon):
-        """
-        Returns the database macaroon if the given raw (serialized) macaroon is
-        an existing valid macaroon, whatever its permissions.
-
-        Raises InvalidMacaroon otherwise.
-        """
-        raw_macaroon = self._extract_raw_macaroon(raw_macaroon)
-        if raw_macaroon is None:
-            raise InvalidMacaroon("malformed or nonexistent macaroon")
-
-        try:
-            m = pymacaroons.Macaroon.deserialize(raw_macaroon)
-        except MacaroonDeserializationException:
-            raise InvalidMacaroon("malformed macaroon")
-
-        dm = self.find_macaroon(m.identifier.decode())
-
-        if dm is None:
-            raise InvalidMacaroon("deleted or nonexistent macaroon")
-
-        verifier = Verifier(m, context=None, principals=None, permission=None)
-        # Will raise InvalidMacaroon if necessary
-        verifier.verify_signature(dm.key)
-
-        return dm
+        raise InvalidMacaroon("invalid macaroon")
 
     def create_macaroon(self, location, user_id, description, caveats):
         """


Test patch:
diff --git a/tests/unit/macaroons/test_caveats.py b/tests/unit/macaroons/test_caveats.py
--- a/tests/unit/macaroons/test_caveats.py
+++ b/tests/unit/macaroons/test_caveats.py
@@ -55,7 +55,7 @@ def test_verify_valid_predicate(self):
         caveat = V1Caveat(verifier)
         predicate = '{"permissions": "user", "version": 1}'
 
-        caveat(predicate)
+        assert caveat(predicate) is True
 
     def test_verify_project_invalid_context(self):
         verifier = pretend.stub(context=pretend.stub())
@@ -90,9 +90,9 @@ def test_verify_project(self, db_request):
         project = ProjectFactory.create(name="foobar")
         verifier = pretend.stub(context=project)
         caveat = V1Caveat(verifier)
-        predicate = {"version": 1, "permissions": {"projects": ["foobar"]}}
 
-        caveat(json.dumps(predicate))
+        predicate = {"version": 1, "permissions": {"projects": ["foobar"]}}
+        assert caveat(json.dumps(predicate)) is True
 
 
 class TestVerifier:
@@ -108,7 +108,7 @@ def test_creation(self):
         assert verifier.principals is principals
         assert verifier.permission is permission
 
-    def test_verify_raises(self, monkeypatch):
+    def test_verify(self, monkeypatch):
         verify = pretend.call_recorder(
             pretend.raiser(MacaroonInvalidSignatureException)
         )
@@ -123,33 +123,3 @@ def test_verify_raises(self, monkeypatch):
         with pytest.raises(InvalidMacaroon):
             verifier.verify(key)
         assert verify.calls == [pretend.call(macaroon, key)]
-
-    def test_verify_works(self, monkeypatch):
-        verify = pretend.call_recorder(lambda x, y: True)
-        macaroon = pretend.stub()
-        context = pretend.stub()
-        principals = pretend.stub()
-        permission = pretend.stub()
-        key = pretend.stub()
-        verifier = Verifier(macaroon, context, principals, permission)
-
-        monkeypatch.setattr(verifier.verifier, "verify", verify)
-
-        verifier.verify(key)
-        assert verify.calls == [pretend.call(macaroon, key)]
-
-    def test_verify_false(self, monkeypatch):
-        # This will not happen in real life, but in case pymacaroon's verify returns
-        # False, we need to raise.
-        verify = pretend.call_recorder(lambda x, y: False)
-        macaroon = pretend.stub()
-        context = pretend.stub()
-        principals = pretend.stub()
-        permission = pretend.stub()
-        key = pretend.stub()
-        verifier = Verifier(macaroon, context, principals, permission)
-
-        monkeypatch.setattr(verifier.verifier, "verify", verify)
-
-        with pytest.raises(InvalidMacaroon):
-            verifier.verify(key)
diff --git a/tests/unit/macaroons/test_services.py b/tests/unit/macaroons/test_services.py
--- a/tests/unit/macaroons/test_services.py
+++ b/tests/unit/macaroons/test_services.py
@@ -72,16 +72,13 @@ def test_find_macaroon(self, user_service, macaroon_service):
     def test_find_userid_no_macaroon(self, macaroon_service):
         assert macaroon_service.find_userid(None) is None
 
-    @pytest.fixture
-    def raw_macaroon(self):
-        return pymacaroons.Macaroon(
+    def test_find_userid_invalid_macaroon(self, macaroon_service):
+        raw_macaroon = pymacaroons.Macaroon(
             location="fake location",
             identifier=str(uuid4()),
             key=b"fake key",
             version=pymacaroons.MACAROON_V2,
         ).serialize()
-
-    def test_find_userid_invalid_macaroon(self, macaroon_service, raw_macaroon):
         raw_macaroon = f"pypi-{raw_macaroon}"
 
         assert macaroon_service.find_userid(raw_macaroon) is None
@@ -105,13 +102,26 @@ def test_find_userid(self, macaroon_service):
 
         assert user.id == user_id
 
-    def test_verify_unprefixed_macaroon(self, macaroon_service, raw_macaroon):
+    def test_verify_unprefixed_macaroon(self, macaroon_service):
+        raw_macaroon = pymacaroons.Macaroon(
+            location="fake location",
+            identifier=str(uuid4()),
+            key=b"fake key",
+            version=pymacaroons.MACAROON_V2,
+        ).serialize()
+
         with pytest.raises(services.InvalidMacaroon):
             macaroon_service.verify(
                 raw_macaroon, pretend.stub(), pretend.stub(), pretend.stub()
             )
 
-    def test_verify_no_macaroon(self, macaroon_service, raw_macaroon):
+    def test_verify_no_macaroon(self, macaroon_service):
+        raw_macaroon = pymacaroons.Macaroon(
+            location="fake location",
+            identifier=str(uuid4()),
+            key=b"fake key",
+            version=pymacaroons.MACAROON_V2,
+        ).serialize()
         raw_macaroon = f"pypi-{raw_macaroon}"
 
         with pytest.raises(services.InvalidMacaroon):
@@ -125,7 +135,7 @@ def test_verify_invalid_macaroon(self, monkeypatch, user_service, macaroon_servi
             "fake location", user.id, "fake description", {"fake": "caveats"}
         )
 
-        verifier_obj = pretend.stub(verify=pretend.raiser(services.InvalidMacaroon))
+        verifier_obj = pretend.stub(verify=pretend.call_recorder(lambda k: False))
         verifier_cls = pretend.call_recorder(lambda *a: verifier_obj)
         monkeypatch.setattr(services, "Verifier", verifier_cls)
 
@@ -193,7 +203,7 @@ def test_verify_valid_macaroon(self, monkeypatch, macaroon_service):
         principals = pretend.stub()
         permissions = pretend.stub()
 
-        macaroon_service.verify(raw_macaroon, context, principals, permissions)
+        assert macaroon_service.verify(raw_macaroon, context, principals, permissions)
         assert verifier_cls.calls == [
             pretend.call(mock.ANY, context, principals, permissions)
         ]
@@ -228,50 +238,3 @@ def test_get_macaroon_by_description(self, macaroon_service):
             macaroon_service.get_macaroon_by_description(user.id, macaroon.description)
             == dm
         )
-
-    def test_check_if_macaroon_exists_unprefixed_macaroon(
-        self, macaroon_service, raw_macaroon
-    ):
-        with pytest.raises(services.InvalidMacaroon):
-            macaroon_service.check_if_macaroon_exists(raw_macaroon)
-
-    def test_check_if_macaroon_exists_no_macaroon(self, macaroon_service, raw_macaroon):
-        raw_macaroon = f"pypi-{raw_macaroon}"
-
-        with pytest.raises(services.InvalidMacaroon):
-            macaroon_service.check_if_macaroon_exists(raw_macaroon)
-
-    def test_check_if_macaroon_exists_invalid_macaroon(
-        self, monkeypatch, user_service, macaroon_service
-    ):
-        user = UserFactory.create()
-        raw_macaroon, _ = macaroon_service.create_macaroon(
-            "fake location", user.id, "fake description", {"fake": "caveats"}
-        )
-
-        verifier_obj = pretend.stub(
-            verify_signature=pretend.raiser(services.InvalidMacaroon)
-        )
-        verifier_cls = pretend.call_recorder(lambda *a, **k: verifier_obj)
-        monkeypatch.setattr(services, "Verifier", verifier_cls)
-
-        with pytest.raises(services.InvalidMacaroon):
-            macaroon_service.check_if_macaroon_exists(raw_macaroon)
-
-    def test_check_if_macaroon_exists_malformed_macaroon(self, macaroon_service):
-        with pytest.raises(services.InvalidMacaroon):
-            macaroon_service.check_if_macaroon_exists("pypi-thiswillnotdeserialize")
-
-    def test_check_if_macaroon_exists_valid_macaroon(
-        self, monkeypatch, macaroon_service
-    ):
-        user = UserFactory.create()
-        raw_macaroon, data_macaroon = macaroon_service.create_macaroon(
-            "fake location", user.id, "fake description", {"fake": "caveats"}
-        )
-
-        verifier_obj = pretend.stub(verify_signature=lambda k: None)
-        verifier_cls = pretend.call_recorder(lambda *a, **k: verifier_obj)
-        monkeypatch.setattr(services, "Verifier", verifier_cls)
-
-        assert macaroon_service.check_if_macaroon_exists(raw_macaroon) is data_macaroon