Repository: pypi/warehouse

Issue:
Enable Read Only Transactions (via factory wrapper)
If a route isn't going to modify data, then we can use a read only deferrable transaction so that we don't need to take SIRead locks and we skip some of the overhead of having a serializable transaction.

Closes #623



Discussion:

Resolving patch:
diff --git a/warehouse/db.py b/warehouse/db.py
--- a/warehouse/db.py
+++ b/warehouse/db.py
@@ -28,6 +28,24 @@
 __all__ = ["includeme", "metadata", "ModelBase"]
 
 
+# We'll add a basic predicate that won't do anything except allow marking a
+# route as read only (or not).
+class ReadOnlyPredicate:
+
+    def __init__(self, val, config):
+        self.val = val
+
+    def text(self):
+        return "read_only = {!r}".format(self.val)
+
+    phash = text
+
+    # This predicate doesn't actually participate in the route selection
+    # process, so we'll just always return True.
+    def __call__(self, info, request):
+        return True
+
+
 class ModelBase:
 
     def __repr__(self):
@@ -84,6 +102,16 @@ def _create_session(request):
     # Create our session
     session = Session(bind=request.registry["sqlalchemy.engine"])
 
+    # Set our transaction to read only if the route has been marked as read
+    # only.
+    for predicate in request.matched_route.predicates:
+        if isinstance(predicate, ReadOnlyPredicate) and predicate.val:
+            session.execute(
+                """ SET TRANSACTION
+                    ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE
+                """
+            )
+
     # Register only this particular session with zope.sqlalchemy
     zope.sqlalchemy.register(session, transaction_manager=request.tm)
 
@@ -103,3 +131,6 @@ def includeme(config):
 
     # Register our request.db property
     config.add_request_method(_create_session, name="db", reify=True)
+
+    # Add a route predicate to mark a route as read only.
+    config.add_route_predicate("read_only", ReadOnlyPredicate)
diff --git a/warehouse/routes.py b/warehouse/routes.py
--- a/warehouse/routes.py
+++ b/warehouse/routes.py
@@ -12,7 +12,7 @@
 
 
 def includeme(config):
-    config.add_route("index", "/")
+    config.add_route("index", "/", read_only=True)
 
     # Accounts
     config.add_route(
@@ -20,6 +20,7 @@ def includeme(config):
         "/user/{username}/",
         factory="warehouse.accounts.models:UserFactory",
         traverse="/{username}",
+        read_only=True,
     )
     config.add_route("accounts.login", "/account/login/")
     config.add_route("accounts.logout", "/account/logout/")
@@ -30,34 +31,39 @@ def includeme(config):
         "/project/{name}/",
         factory="warehouse.packaging.models:ProjectFactory",
         traverse="/{name}",
+        read_only=True,
     )
     config.add_route(
         "packaging.release",
         "/project/{name}/{version}/",
         factory="warehouse.packaging.models:ProjectFactory",
         traverse="/{name}/{version}",
+        read_only=True,
     )
-    config.add_route("packaging.file", "/packages/{path:.*}")
+    config.add_route("packaging.file", "/packages/{path:.*}", read_only=True)
 
     # Legacy URLs
-    config.add_route("legacy.api.simple.index", "/simple/")
+    config.add_route("legacy.api.simple.index", "/simple/", read_only=True)
     config.add_route(
         "legacy.api.simple.detail",
         "/simple/{name}/",
         factory="warehouse.packaging.models:ProjectFactory",
         traverse="/{name}/",
+        read_only=True,
     )
     config.add_route(
         "legacy.api.json.project",
         "/pypi/{name}/json",
         factory="warehouse.packaging.models:ProjectFactory",
         traverse="/{name}",
+        read_only=True,
     )
     config.add_route(
         "legacy.api.json.release",
         "/pypi/{name}/{version}/json",
         factory="warehouse.packaging.models:ProjectFactory",
         traverse="/{name}/{version}",
+        read_only=True,
     )
 
     # Legacy Action URLs
@@ -75,6 +81,7 @@ def includeme(config):
         "pypi",
         pattern="/pypi",
         header="Content-Type:text/xml",
+        read_only=True,
     )
 
     # Legacy Documentation


Test patch:
diff --git a/tests/unit/test_db.py b/tests/unit/test_db.py
--- a/tests/unit/test_db.py
+++ b/tests/unit/test_db.py
@@ -14,6 +14,7 @@
 
 import alembic.config
 import pretend
+import pytest
 import sqlalchemy
 import venusian
 import zope.sqlalchemy
@@ -87,7 +88,15 @@ def config_cls():
     ]
 
 
-def test_create_session(monkeypatch):
+@pytest.mark.parametrize(
+    "predicates",
+    [
+        [],
+        [db.ReadOnlyPredicate(False, None)],
+        [object()],
+    ],
+)
+def test_create_session(monkeypatch, predicates):
     session_obj = pretend.stub()
     session_cls = pretend.call_recorder(lambda bind: session_obj)
     monkeypatch.setattr(db, "Session", session_cls)
@@ -96,6 +105,7 @@ def test_create_session(monkeypatch):
     request = pretend.stub(
         registry={"sqlalchemy.engine": engine},
         tm=pretend.stub(),
+        matched_route=pretend.stub(predicates=predicates),
     )
 
     register = pretend.call_recorder(lambda session, transaction_manager: None)
@@ -108,6 +118,37 @@ def test_create_session(monkeypatch):
     ]
 
 
+def test_creates_readonly_session(monkeypatch):
+    session_obj = pretend.stub(execute=pretend.call_recorder(lambda sql: None))
+    session_cls = pretend.call_recorder(lambda bind: session_obj)
+    monkeypatch.setattr(db, "Session", session_cls)
+
+    engine = pretend.stub()
+    request = pretend.stub(
+        registry={"sqlalchemy.engine": engine},
+        tm=pretend.stub(),
+        matched_route=pretend.stub(
+            predicates=[db.ReadOnlyPredicate(True, None)],
+        ),
+    )
+
+    register = pretend.call_recorder(lambda session, transaction_manager: None)
+    monkeypatch.setattr(zope.sqlalchemy, "register", register)
+
+    assert _create_session(request) is session_obj
+    assert session_cls.calls == [pretend.call(bind=engine)]
+    assert register.calls == [
+        pretend.call(session_obj, transaction_manager=request.tm),
+    ]
+    assert session_obj.execute.calls == [
+        pretend.call(
+            """ SET TRANSACTION
+                    ISOLATION LEVEL SERIALIZABLE READ ONLY DEFERRABLE
+                """
+        ),
+    ]
+
+
 def test_includeme(monkeypatch):
     class FakeRegistry(dict):
         settings = {"database.url": pretend.stub()}
@@ -118,6 +159,7 @@ class FakeRegistry(dict):
         add_directive=pretend.call_recorder(lambda *a: None),
         registry=FakeRegistry(),
         add_request_method=pretend.call_recorder(lambda f, name, reify: None),
+        add_route_predicate=pretend.call_recorder(lambda *a, **kw: None),
     )
 
     monkeypatch.setattr(sqlalchemy, "create_engine", create_engine)
@@ -137,3 +179,6 @@ class FakeRegistry(dict):
     assert config.add_request_method.calls == [
         pretend.call(_create_session, name="db", reify=True),
     ]
+    assert config.add_route_predicate.calls == [
+        pretend.call("read_only", db.ReadOnlyPredicate),
+    ]
diff --git a/tests/unit/test_routes.py b/tests/unit/test_routes.py
--- a/tests/unit/test_routes.py
+++ b/tests/unit/test_routes.py
@@ -40,19 +40,20 @@ def add_pypi_action_route(name, action, **kwargs):
 
         @staticmethod
         @pretend.call_recorder
-        def add_xmlrpc_endpoint(endpoint, pattern, header):
+        def add_xmlrpc_endpoint(endpoint, pattern, header, read_only=False):
             pass
 
     config = FakeConfig()
     includeme(config)
 
     assert config.add_route.calls == [
-        pretend.call('index', '/'),
+        pretend.call('index', '/', read_only=True),
         pretend.call(
             "accounts.profile",
             "/user/{username}/",
             factory="warehouse.accounts.models:UserFactory",
             traverse="/{username}",
+            read_only=True,
         ),
         pretend.call("accounts.login", "/account/login/"),
         pretend.call("accounts.logout", "/account/logout/"),
@@ -61,32 +62,37 @@ def add_xmlrpc_endpoint(endpoint, pattern, header):
             "/project/{name}/",
             factory="warehouse.packaging.models:ProjectFactory",
             traverse="/{name}",
+            read_only=True,
         ),
         pretend.call(
             "packaging.release",
             "/project/{name}/{version}/",
             factory="warehouse.packaging.models:ProjectFactory",
             traverse="/{name}/{version}",
+            read_only=True,
         ),
-        pretend.call("packaging.file", "/packages/{path:.*}"),
-        pretend.call("legacy.api.simple.index", "/simple/"),
+        pretend.call("packaging.file", "/packages/{path:.*}", read_only=True),
+        pretend.call("legacy.api.simple.index", "/simple/", read_only=True),
         pretend.call(
             "legacy.api.simple.detail",
             "/simple/{name}/",
             factory="warehouse.packaging.models:ProjectFactory",
             traverse="/{name}/",
+            read_only=True,
         ),
         pretend.call(
             "legacy.api.json.project",
             "/pypi/{name}/json",
             factory="warehouse.packaging.models:ProjectFactory",
             traverse="/{name}",
+            read_only=True,
         ),
         pretend.call(
             "legacy.api.json.release",
             "/pypi/{name}/{version}/json",
             factory="warehouse.packaging.models:ProjectFactory",
             traverse="/{name}/{version}",
+            read_only=True,
         ),
         pretend.call("legacy.docs", docs_route_url),
     ]
@@ -105,5 +111,10 @@ def add_xmlrpc_endpoint(endpoint, pattern, header):
     ]
 
     assert config.add_xmlrpc_endpoint.calls == [
-        pretend.call("pypi", pattern="/pypi", header="Content-Type:text/xml"),
+        pretend.call(
+            "pypi",
+            pattern="/pypi",
+            header="Content-Type:text/xml",
+            read_only=True,
+        ),
     ]