Repository: pypi/warehouse

Issue:
Document how to purge the HTTP cache when deploying the design
It'd be useful to document that you can do `curl -XPURGE https://warehouse.python.org/.../` when the design has been deployed to see the changes if they have been cached by Fastly.



Discussion:
Another option we could maybe do here is tag all of the pages with which template(s) they depend on and whenever a deployment happens, do an automatic purge of any pages that had their templates changed. That will be more effort than just documenting the `curl -XPURGE` thing though.


Resolving patch:
diff --git a/warehouse/cache/origin/__init__.py b/warehouse/cache/origin/__init__.py
--- a/warehouse/cache/origin/__init__.py
+++ b/warehouse/cache/origin/__init__.py
@@ -18,6 +18,7 @@
 from sqlalchemy.orm.session import Session
 
 from warehouse import db
+from warehouse.cache.origin.derivers import html_cache_deriver
 from warehouse.cache.origin.interfaces import IOriginCache
 
 
@@ -75,7 +76,7 @@ def wrapped(context, request):
                 request.add_response_callback(
                     functools.partial(
                         cacher.cache,
-                        sorted(context_keys + keys),
+                        context_keys + keys,
                         seconds=seconds,
                         stale_while_revalidate=stale_while_revalidate,
                         stale_if_error=stale_if_error,
@@ -152,6 +153,7 @@ def includeme(config):
             cache_class.create_service,
             IOriginCache,
         )
+        config.add_view_deriver(html_cache_deriver)
 
     config.add_directive(
         "register_origin_cache_keys",
diff --git a/warehouse/cache/origin/derivers.py b/warehouse/cache/origin/derivers.py
new file mode 100644
--- /dev/null
+++ b/warehouse/cache/origin/derivers.py
@@ -0,0 +1,35 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import functools
+
+from warehouse.cache.origin.interfaces import IOriginCache
+
+
+def html_cache_deriver(view, info):
+    renderer = info.options.get('renderer')
+    if renderer and renderer.name.endswith('.html'):
+        def wrapper_view(context, request):
+            try:
+                cacher = request.find_service(IOriginCache)
+            except ValueError:
+                pass
+            else:
+                request.add_response_callback(
+                    functools.partial(
+                        cacher.cache,
+                        ['all-html', renderer.name],
+                    )
+                )
+            return view(context, request)
+        return wrapper_view
+    return view
diff --git a/warehouse/cache/origin/fastly.py b/warehouse/cache/origin/fastly.py
--- a/warehouse/cache/origin/fastly.py
+++ b/warehouse/cache/origin/fastly.py
@@ -11,6 +11,7 @@
 # limitations under the License.
 
 import urllib.parse
+import weakref
 
 import requests
 
@@ -45,6 +46,7 @@ def __init__(self, *, api_key, service_id, purger):
         self.api_key = api_key
         self.service_id = service_id
         self._purger = purger
+        self.keys = weakref.WeakKeyDictionary()
 
     @classmethod
     def create_service(cls, context, request):
@@ -56,7 +58,15 @@ def create_service(cls, context, request):
 
     def cache(self, keys, request, response, *, seconds=None,
               stale_while_revalidate=None, stale_if_error=None):
-        response.headers["Surrogate-Key"] = " ".join(keys)
+
+        if request in self.keys:
+            self.keys[request].update(keys)
+        else:
+            self.keys[request] = set(keys)
+
+        response.headers["Surrogate-Key"] = " ".join(
+            sorted(self.keys[request])
+        )
 
         values = []
 


Test patch:
diff --git a/tests/unit/cache/origin/test_derivers.py b/tests/unit/cache/origin/test_derivers.py
new file mode 100644
--- /dev/null
+++ b/tests/unit/cache/origin/test_derivers.py
@@ -0,0 +1,89 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pretend
+
+from warehouse.cache.origin.derivers import html_cache_deriver
+
+
+def test_no_renderer():
+    view = pretend.stub()
+    info = pretend.stub(options={})
+
+    assert html_cache_deriver(view, info) == view
+
+
+def test_non_html_renderer():
+    view = pretend.stub()
+    renderer = pretend.stub(name='foo.txt')
+    info = pretend.stub(options={'renderer': renderer})
+
+    assert html_cache_deriver(view, info) == view
+
+
+def test_no_origin_cache_found():
+    view_result = pretend.stub()
+    view = pretend.call_recorder(lambda context, request: view_result)
+    renderer = pretend.stub(name='foo.html')
+    info = pretend.stub(options={'renderer': renderer})
+    context = pretend.stub()
+
+    def raise_valueerror(*a):
+        raise ValueError
+
+    request = pretend.stub(
+        find_service=raise_valueerror,
+        add_response_callback=pretend.call_recorder(lambda a: None)
+    )
+
+    assert html_cache_deriver(view, info)(context, request) == view_result
+    assert request.add_response_callback.calls == []
+
+
+def test_response_hook():
+
+    class Cache:
+
+        @staticmethod
+        @pretend.call_recorder
+        def cache(keys, request, response):
+            pass
+
+    response = pretend.stub()
+
+    @pretend.call_recorder
+    def view(context, request):
+        return response
+
+    context = pretend.stub()
+    cacher = Cache()
+    callbacks = []
+    request = pretend.stub(
+        find_service=lambda iface: cacher,
+        add_response_callback=callbacks.append,
+    )
+    info = pretend.stub(options={'renderer': pretend.stub(name='foo.html')})
+    derived_view = html_cache_deriver(view, info)
+
+    assert derived_view(context, request) is response
+    assert view.calls == [pretend.call(context, request)]
+    assert len(callbacks) == 1
+
+    callbacks[0](request, response)
+
+    assert cacher.cache.calls == [
+        pretend.call(
+            ['all-html', 'foo.html'],
+            request,
+            response,
+        ),
+    ]
diff --git a/tests/unit/cache/origin/test_fastly.py b/tests/unit/cache/origin/test_fastly.py
--- a/tests/unit/cache/origin/test_fastly.py
+++ b/tests/unit/cache/origin/test_fastly.py
@@ -138,6 +138,35 @@ def test_adds_surrogate_control(self):
             ),
         }
 
+    def test_multiple_calls_to_cache_dont_overwrite_surrogate_keys(self):
+        request = pretend.stub()
+        response = pretend.stub(headers={})
+
+        cacher = fastly.FastlyCache(api_key=None, service_id=None, purger=None)
+        cacher.cache(["abc"], request, response)
+        cacher.cache(["defg"], request, response)
+
+        assert response.headers == {
+            "Surrogate-Key": "abc defg",
+        }
+
+    def test_multiple_calls_with_different_requests(self):
+        request_a = pretend.stub()
+        request_b = pretend.stub()
+        response_a = pretend.stub(headers={})
+        response_b = pretend.stub(headers={})
+
+        cacher = fastly.FastlyCache(api_key=None, service_id=None, purger=None)
+        cacher.cache(["abc"], request_a, response_a)
+        cacher.cache(["defg"], request_b, response_b)
+
+        assert response_a.headers == {
+            "Surrogate-Key": "abc",
+        }
+        assert response_b.headers == {
+            "Surrogate-Key": "defg",
+        }
+
     def test_purge(self, monkeypatch):
         purge_delay = pretend.call_recorder(lambda *a, **kw: None)
         cacher = fastly.FastlyCache(
diff --git a/tests/unit/cache/origin/test_init.py b/tests/unit/cache/origin/test_init.py
--- a/tests/unit/cache/origin/test_init.py
+++ b/tests/unit/cache/origin/test_init.py
@@ -14,6 +14,7 @@
 import pytest
 
 from warehouse.cache import origin
+from warehouse.cache.origin.derivers import html_cache_deriver
 from warehouse.cache.origin.interfaces import IOriginCache
 
 
@@ -189,7 +190,7 @@ def view(context, request):
 
         assert cacher.cache.calls == [
             pretend.call(
-                sorted(["one", "two"] + ([] if keys is None else keys)),
+                ["one", "two"] + ([] if keys is None else keys),
                 request,
                 response,
                 seconds=seconds,
@@ -306,6 +307,7 @@ def test_includeme_with_origin_cache():
     cache_class = pretend.stub(create_service=pretend.stub())
     config = pretend.stub(
         add_directive=pretend.call_recorder(lambda name, func: None),
+        add_view_deriver=pretend.call_recorder(lambda deriver: None),
         registry=pretend.stub(
             settings={
                 "origin_cache.backend":
@@ -324,6 +326,9 @@ def test_includeme_with_origin_cache():
             origin.register_origin_cache_keys,
         ),
     ]
+    assert config.add_view_deriver.calls == [
+        pretend.call(html_cache_deriver),
+    ]
     assert config.maybe_dotted.calls == [
         pretend.call("warehouse.cache.origin.fastly.FastlyCache"),
     ]