Repository: pypi/warehouse

Issue:
CDN Purging is broken, stale JSON responses are served instead
In https://github.com/pypi/warehouse/pull/11920 we removed traversal for our legacy JSON views, however our purging mechanisms depend on traversal to determine when to issue purges. As a result, PyPI is currently serving stale data for all JSON endpoints for which responses existed before the merge of pypi/warehouse#11920, as no purges have been issued for these when something changes which would update the responses here (e.g., publishing a new release).

(ref: https://github.com/pypi/infra/issues/104, https://github.com/pypi/warehouse/issues/11936, https://github.com/pypi/warehouse/issues/11937, https://github.com/pypi/warehouse/issues/11939, https://github.com/pypi/warehouse/issues/11940)



Discussion:

Resolving patch:
diff --git a/warehouse/legacy/api/json.py b/warehouse/legacy/api/json.py
--- a/warehouse/legacy/api/json.py
+++ b/warehouse/legacy/api/json.py
@@ -166,12 +166,7 @@ def _json_data(request, project, release, *, all_releases):
     return data
 
 
-@view_config(
-    route_name="legacy.api.json.project",
-    renderer="json",
-    decorator=_CACHE_DECORATOR,
-)
-def json_project(request):
+def latest_release_factory(request):
     normalized_name = canonicalize_name(request.matchdict["name"])
 
     try:
@@ -202,6 +197,17 @@ def json_project(request):
         .filter(Release.id == latest.id)
         .one()
     )
+
+    return release
+
+
+@view_config(
+    route_name="legacy.api.json.project",
+    context=Release,
+    renderer="json",
+    decorator=_CACHE_DECORATOR,
+)
+def json_project(release, request):
     project = release.project
 
     if project.normalized_name != request.matchdict["name"]:
@@ -224,19 +230,15 @@ def json_project(request):
 
 @view_config(
     route_name="legacy.api.json.project_slash",
+    context=Release,
     renderer="json",
     decorator=_CACHE_DECORATOR,
 )
-def json_project_slash(request):
-    return json_project(request)
+def json_project_slash(release, request):
+    return json_project(release, request)
 
 
-@view_config(
-    route_name="legacy.api.json.release",
-    renderer="json",
-    decorator=_CACHE_DECORATOR,
-)
-def json_release(request):
+def release_factory(request):
     normalized_name = canonicalize_name(request.matchdict["name"])
     version = request.matchdict["version"]
     canonical_version = canonicalize_version(version)
@@ -269,6 +271,16 @@ def json_release(request):
     except NoResultFound:
         return HTTPNotFound(headers=_CORS_HEADERS)
 
+    return release
+
+
+@view_config(
+    route_name="legacy.api.json.release",
+    context=Release,
+    renderer="json",
+    decorator=_CACHE_DECORATOR,
+)
+def json_release(release, request):
     project = release.project
 
     if project.normalized_name != request.matchdict["name"]:
@@ -289,8 +301,9 @@ def json_release(request):
 
 @view_config(
     route_name="legacy.api.json.release_slash",
+    context=Release,
     renderer="json",
     decorator=_CACHE_DECORATOR,
 )
-def json_release_slash(request):
-    return json_release(request)
+def json_release_slash(release, request):
+    return json_release(release, request)
diff --git a/warehouse/routes.py b/warehouse/routes.py
--- a/warehouse/routes.py
+++ b/warehouse/routes.py
@@ -442,12 +442,14 @@ def includeme(config):
     config.add_route(
         "legacy.api.json.project",
         "/pypi/{name}/json",
+        factory="warehouse.legacy.api.json.latest_release_factory",
         read_only=True,
         domain=warehouse,
     )
     config.add_route(
         "legacy.api.json.project_slash",
         "/pypi/{name}/json/",
+        factory="warehouse.legacy.api.json.latest_release_factory",
         read_only=True,
         domain=warehouse,
     )
@@ -455,12 +457,14 @@ def includeme(config):
     config.add_route(
         "legacy.api.json.release",
         "/pypi/{name}/{version}/json",
+        factory="warehouse.legacy.api.json.release_factory",
         read_only=True,
         domain=warehouse,
     )
     config.add_route(
         "legacy.api.json.release_slash",
         "/pypi/{name}/{version}/json/",
+        factory="warehouse.legacy.api.json.release_factory",
         read_only=True,
         domain=warehouse,
     )


Test patch:
diff --git a/tests/unit/legacy/api/test_json.py b/tests/unit/legacy/api/test_json.py
--- a/tests/unit/legacy/api/test_json.py
+++ b/tests/unit/legacy/api/test_json.py
@@ -40,33 +40,11 @@ def _assert_has_cors_headers(headers):
     assert headers["Access-Control-Expose-Headers"] == "X-PyPI-Last-Serial"
 
 
-class TestJSONProject:
-    def test_normalizing_redirects(self, db_request):
-        project = ProjectFactory.create()
-        ReleaseFactory.create(project=project, version="1.0")
-
-        name = project.name.lower()
-        if name == project.normalized_name:
-            name = project.name.upper()
-
-        db_request.matchdict = {"name": name}
-        db_request.current_route_path = pretend.call_recorder(
-            lambda name: "/project/the-redirect/"
-        )
-
-        resp = json.json_project(db_request)
-
-        assert isinstance(resp, HTTPMovedPermanently)
-        assert resp.headers["Location"] == "/project/the-redirect/"
-        _assert_has_cors_headers(resp.headers)
-        assert db_request.current_route_path.calls == [
-            pretend.call(name=project.normalized_name)
-        ]
-
+class TestLatestReleaseFactory:
     def test_missing_release(self, db_request):
         project = ProjectFactory.create()
         db_request.matchdict = {"name": project.normalized_name}
-        resp = json.json_project(db_request)
+        resp = json.latest_release_factory(db_request)
         assert isinstance(resp, HTTPNotFound)
         _assert_has_cors_headers(resp.headers)
 
@@ -78,20 +56,8 @@ def test_with_prereleases(self, monkeypatch, db_request):
         ReleaseFactory.create(project=project, version="4.0.dev0")
 
         release = ReleaseFactory.create(project=project, version="3.0")
-
-        data = pretend.stub()
-        json_data = pretend.call_recorder(
-            lambda request, project, release, *, all_releases: data
-        )
-        monkeypatch.setattr(json, "_json_data", json_data)
         db_request.matchdict = {"name": project.normalized_name}
-
-        rvalue = json.json_project(db_request)
-
-        assert rvalue is data
-        assert json_data.calls == [
-            pretend.call(db_request, project, release, all_releases=True)
-        ]
+        assert json.latest_release_factory(db_request) == release
 
     def test_only_prereleases(self, monkeypatch, db_request):
         project = ProjectFactory.create()
@@ -100,20 +66,8 @@ def test_only_prereleases(self, monkeypatch, db_request):
         ReleaseFactory.create(project=project, version="2.0.dev0")
 
         release = ReleaseFactory.create(project=project, version="3.0.dev0")
-
-        data = pretend.stub()
-        json_data = pretend.call_recorder(
-            lambda request, project, release, *, all_releases: data
-        )
-        monkeypatch.setattr(json, "_json_data", json_data)
         db_request.matchdict = {"name": project.normalized_name}
-
-        rvalue = json.json_project(db_request)
-
-        assert rvalue is data
-        assert json_data.calls == [
-            pretend.call(db_request, project, release, all_releases=True)
-        ]
+        assert json.latest_release_factory(db_request) == release
 
     def test_all_releases_yanked(self, monkeypatch, db_request):
         """
@@ -128,20 +82,8 @@ def test_all_releases_yanked(self, monkeypatch, db_request):
         ReleaseFactory.create(project=project, version="4.0.dev0", yanked=True)
 
         release = ReleaseFactory.create(project=project, version="3.0", yanked=True)
-
-        data = pretend.stub()
-        json_data = pretend.call_recorder(
-            lambda request, project, release, *, all_releases: data
-        )
-        monkeypatch.setattr(json, "_json_data", json_data)
         db_request.matchdict = {"name": project.normalized_name}
-
-        rvalue = json.json_project(db_request)
-
-        assert rvalue is data
-        assert json_data.calls == [
-            pretend.call(db_request, project, release, all_releases=True)
-        ]
+        assert json.latest_release_factory(db_request) == release
 
     def test_latest_release_yanked(self, monkeypatch, db_request):
         """
@@ -156,20 +98,8 @@ def test_latest_release_yanked(self, monkeypatch, db_request):
         ReleaseFactory.create(project=project, version="3.0.dev0")
 
         release = ReleaseFactory.create(project=project, version="2.0")
-
-        data = pretend.stub()
-        json_data = pretend.call_recorder(
-            lambda request, project, release, *, all_releases: data
-        )
-        monkeypatch.setattr(json, "_json_data", json_data)
         db_request.matchdict = {"name": project.normalized_name}
-
-        rvalue = json.json_project(db_request)
-
-        assert rvalue is data
-        assert json_data.calls == [
-            pretend.call(db_request, project, release, all_releases=True)
-        ]
+        assert json.latest_release_factory(db_request) == release
 
     def test_all_non_prereleases_yanked(self, monkeypatch, db_request):
         """
@@ -185,19 +115,31 @@ def test_all_non_prereleases_yanked(self, monkeypatch, db_request):
         ReleaseFactory.create(project=project, version="3.0.dev0", yanked=True)
 
         release = ReleaseFactory.create(project=project, version="2.0.dev0")
+        db_request.matchdict = {"name": project.normalized_name}
+        assert json.latest_release_factory(db_request) == release
+
+
+class TestJSONProject:
+    def test_normalizing_redirects(self, db_request):
+        project = ProjectFactory.create()
+        release = ReleaseFactory.create(project=project, version="1.0")
 
-        data = pretend.stub()
-        json_data = pretend.call_recorder(
-            lambda request, project, release, *, all_releases: data
+        name = project.name.lower()
+        if name == project.normalized_name:
+            name = project.name.upper()
+
+        db_request.matchdict = {"name": name}
+        db_request.current_route_path = pretend.call_recorder(
+            lambda name: "/project/the-redirect/"
         )
-        monkeypatch.setattr(json, "_json_data", json_data)
-        db_request.matchdict = {"name": project.normalized_name}
 
-        rvalue = json.json_project(db_request)
+        resp = json.json_project(release, db_request)
 
-        assert rvalue is data
-        assert json_data.calls == [
-            pretend.call(db_request, project, release, all_releases=True)
+        assert isinstance(resp, HTTPMovedPermanently)
+        assert resp.headers["Location"] == "/project/the-redirect/"
+        _assert_has_cors_headers(resp.headers)
+        assert db_request.current_route_path.calls == [
+            pretend.call(name=project.normalized_name)
         ]
 
     def test_renders(self, pyramid_config, db_request, db_session):
@@ -264,7 +206,7 @@ def test_renders(self, pyramid_config, db_request, db_session):
         db_request.route_url = pretend.call_recorder(lambda *args, **kw: url)
         db_request.matchdict = {"name": project.normalized_name}
 
-        result = json.json_project(db_request)
+        result = json.json_project(releases[-1], db_request)
 
         assert set(db_request.route_url.calls) == {
             pretend.call("packaging.file", path=files[0].path),
@@ -414,7 +356,7 @@ def test_renders(self, pyramid_config, db_request, db_session):
 class TestJSONProjectSlash:
     def test_normalizing_redirects(self, db_request):
         project = ProjectFactory.create()
-        ReleaseFactory.create(project=project, version="1.0")
+        release = ReleaseFactory.create(project=project, version="1.0")
 
         name = project.name.lower()
         if name == project.normalized_name:
@@ -425,7 +367,7 @@ def test_normalizing_redirects(self, db_request):
             lambda name: "/project/the-redirect/"
         )
 
-        resp = json.json_project_slash(db_request)
+        resp = json.json_project_slash(release, db_request)
 
         assert isinstance(resp, HTTPMovedPermanently)
         assert resp.headers["Location"] == "/project/the-redirect/"
@@ -435,33 +377,11 @@ def test_normalizing_redirects(self, db_request):
         ]
 
 
-class TestJSONRelease:
-    def test_normalizing_redirects(self, db_request):
-        project = ProjectFactory.create()
-        release = ReleaseFactory.create(project=project, version="3.0")
-
-        name = release.project.name.lower()
-        if name == release.project.normalized_name:
-            name = release.project.name.upper()
-
-        db_request.matchdict = {"name": name, "version": "3.0"}
-        db_request.current_route_path = pretend.call_recorder(
-            lambda name: "/project/the-redirect/3.0/"
-        )
-
-        resp = json.json_release(db_request)
-
-        assert isinstance(resp, HTTPMovedPermanently)
-        assert resp.headers["Location"] == "/project/the-redirect/3.0/"
-        _assert_has_cors_headers(resp.headers)
-        assert db_request.current_route_path.calls == [
-            pretend.call(name=release.project.normalized_name)
-        ]
-
+class TestReleaseFactory:
     def test_missing_release(self, db_request):
         project = ProjectFactory.create()
         db_request.matchdict = {"name": project.normalized_name, "version": "3.0"}
-        resp = json.json_release(db_request)
+        resp = json.release_factory(db_request)
         assert isinstance(resp, HTTPNotFound)
         _assert_has_cors_headers(resp.headers)
 
@@ -470,7 +390,7 @@ def test_missing_release_with_multiple_canonical(self, db_request):
         ReleaseFactory.create(project=project, version="3.0.0")
         ReleaseFactory.create(project=project, version="3.0.0.0")
         db_request.matchdict = {"name": project.normalized_name, "version": "3.0"}
-        resp = json.json_release(db_request)
+        resp = json.release_factory(db_request)
         assert isinstance(resp, HTTPNotFound)
         _assert_has_cors_headers(resp.headers)
 
@@ -482,15 +402,51 @@ def test_missing_release_with_multiple_canonical(self, db_request):
             (["0.1", "1.0", "2.0", "3.0.0"], "3.0.0.0.0", "3.0.0.0.0"),
         ],
     )
-    def test_detail_renders(
-        self,
-        pyramid_config,
-        db_request,
-        db_session,
-        other_versions,
-        the_version,
-        lookup_version,
+    def test_lookup_release(
+        self, db_request, other_versions, the_version, lookup_version
     ):
+        project = ProjectFactory.create()
+        releases = [
+            ReleaseFactory.create(project=project, version=v) for v in other_versions
+        ]
+        releases += [ReleaseFactory.create(project=project, version=the_version)]
+
+        user = UserFactory.create()
+        JournalEntryFactory.reset_sequence()
+        JournalEntryFactory.create(name=project.name, submitted_by=user)
+
+        db_request.matchdict = {
+            "name": project.normalized_name,
+            "version": lookup_version,
+        }
+
+        assert json.release_factory(db_request) == releases[-1]
+
+
+class TestJSONRelease:
+    def test_normalizing_redirects(self, db_request):
+        project = ProjectFactory.create()
+        release = ReleaseFactory.create(project=project, version="3.0")
+
+        name = release.project.name.lower()
+        if name == release.project.normalized_name:
+            name = release.project.name.upper()
+
+        db_request.matchdict = {"name": name, "version": "3.0"}
+        db_request.current_route_path = pretend.call_recorder(
+            lambda name: "/project/the-redirect/3.0/"
+        )
+
+        resp = json.json_release(release, db_request)
+
+        assert isinstance(resp, HTTPMovedPermanently)
+        assert resp.headers["Location"] == "/project/the-redirect/3.0/"
+        _assert_has_cors_headers(resp.headers)
+        assert db_request.current_route_path.calls == [
+            pretend.call(name=release.project.normalized_name)
+        ]
+
+    def test_detail_renders(self, pyramid_config, db_request, db_session):
         project = ProjectFactory.create(has_docs=True)
         description_content_type = "text/x-rst"
         url = "/the/fake/url/"
@@ -514,12 +470,13 @@ def test_detail_renders(
         expected_urls = dict(tuple(expected_urls))
 
         releases = [
-            ReleaseFactory.create(project=project, version=v) for v in other_versions
+            ReleaseFactory.create(project=project, version=v)
+            for v in ["0.1", "1.0", "2.0"]
         ]
         releases += [
             ReleaseFactory.create(
                 project=project,
-                version=the_version,
+                version="3.0",
                 description=DescriptionFactory.create(
                     content_type=description_content_type
                 ),
@@ -553,10 +510,10 @@ def test_detail_renders(
         db_request.route_url = pretend.call_recorder(lambda *args, **kw: url)
         db_request.matchdict = {
             "name": project.normalized_name,
-            "version": lookup_version,
+            "version": "3.0",
         }
 
-        result = json.json_release(db_request)
+        result = json.json_release(releases[-1], db_request)
 
         assert set(db_request.route_url.calls) == {
             pretend.call("packaging.file", path=files[-1].path),
@@ -597,7 +554,7 @@ def test_detail_renders(
                 "summary": None,
                 "yanked": False,
                 "yanked_reason": None,
-                "version": the_version,
+                "version": "3.0",
             },
             "urls": [
                 {
@@ -647,7 +604,7 @@ def test_minimal_renders(self, pyramid_config, db_request):
             "version": release.canonical_version,
         }
 
-        result = json.json_release(db_request)
+        result = json.json_release(release, db_request)
 
         assert set(db_request.route_url.calls) == {
             pretend.call("packaging.file", path=file.path),
@@ -733,7 +690,7 @@ def test_vulnerabilities_renders(self, pyramid_config, db_request):
             "version": release.canonical_version,
         }
 
-        result = json.json_release(db_request)
+        result = json.json_release(release, db_request)
 
         assert result["vulnerabilities"] == [
             {
@@ -762,7 +719,7 @@ def test_normalizing_redirects(self, db_request):
             lambda name: "/project/the-redirect/3.0/"
         )
 
-        resp = json.json_release_slash(db_request)
+        resp = json.json_release_slash(release, db_request)
 
         assert isinstance(resp, HTTPMovedPermanently)
         assert resp.headers["Location"] == "/project/the-redirect/3.0/"
diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py
--- a/tests/unit/test_routes.py
+++ b/tests/unit/test_routes.py
@@ -444,24 +444,28 @@ def add_policy(name, filename):
         pretend.call(
             "legacy.api.json.project",
             "/pypi/{name}/json",
+            factory="warehouse.legacy.api.json.latest_release_factory",
             read_only=True,
             domain=warehouse,
         ),
         pretend.call(
             "legacy.api.json.project_slash",
             "/pypi/{name}/json/",
+            factory="warehouse.legacy.api.json.latest_release_factory",
             read_only=True,
             domain=warehouse,
         ),
         pretend.call(
             "legacy.api.json.release",
             "/pypi/{name}/{version}/json",
+            factory="warehouse.legacy.api.json.release_factory",
             read_only=True,
             domain=warehouse,
         ),
         pretend.call(
             "legacy.api.json.release_slash",
             "/pypi/{name}/{version}/json/",
+            factory="warehouse.legacy.api.json.release_factory",
             read_only=True,
             domain=warehouse,
         ),