Repository: wagtail/wagtail

Issue:
Make StreamField lazily fetch related objects
I have a site that includes a list of pages in the footer (privacy policy, terms and conditions, etc). This is implemented using a `wagtailsettings` model with a join table/InlinePanel to `wagtailcore.Page`. Every page load thus fetches these pages to print their URL and title in the footer.

It appears that when a page is fetched, any related models referenced in that pages StreamField contents are also fetched. This means that loading the links for the footer of one page means fetching all of the images contained in the content of all of the footer pages. These extra database calls are clearly wasteful.

StreamField instances should lazily fetch related objects when the StreamField is first accessed, instead of as soon as it is loaded.



Discussion:

Resolving patch:
diff --git a/wagtail/wagtailcore/blocks/stream_block.py b/wagtail/wagtailcore/blocks/stream_block.py
--- a/wagtail/wagtailcore/blocks/stream_block.py
+++ b/wagtail/wagtailcore/blocks/stream_block.py
@@ -173,12 +173,12 @@ def clean(self, value):
 
     def to_python(self, value):
         # the incoming JSONish representation is a list of dicts, each with a 'type' and 'value' field.
-        # Convert this to a StreamValue backed by a list of (type, value) tuples
+        # This is passed to StreamValue to be expanded lazily - but first we reject any unrecognised
+        # block types from the list
         return StreamValue(self, [
-            (child_data['type'], self.child_blocks[child_data['type']].to_python(child_data['value']))
-            for child_data in value
+            child_data for child_data in value
             if child_data['type'] in self.child_blocks
-        ])
+        ], is_lazy=True)
 
     def get_prep_value(self, value):
         if value is None:
@@ -246,15 +246,36 @@ def block_type(self):
             """
             return self.block.name
 
-    def __init__(self, stream_block, stream_data):
+    def __init__(self, stream_block, stream_data, is_lazy=False):
+        """
+        Construct a StreamValue linked to the given StreamBlock,
+        with child values given in stream_data.
+
+        Passing is_lazy=True means that stream_data is raw JSONish data as stored
+        in the database, and needs to be converted to native values
+        (using block.to_python()) when accessed. In this mode, stream_data is a
+        list of dicts, each containing 'type' and 'value' keys.
+
+        Passing is_lazy=False means that stream_data consists of immediately usable
+        native values. In this mode, stream_data is a list of (type_name, value)
+        tuples.
+        """
+        self.is_lazy = is_lazy
         self.stream_block = stream_block  # the StreamBlock object that handles this value
         self.stream_data = stream_data  # a list of (type_name, value) tuples
         self._bound_blocks = {}  # populated lazily from stream_data as we access items through __getitem__
 
     def __getitem__(self, i):
         if i not in self._bound_blocks:
-            type_name, value = self.stream_data[i]
-            child_block = self.stream_block.child_blocks[type_name]
+            if self.is_lazy:
+                raw_value = self.stream_data[i]
+                type_name = raw_value['type']
+                child_block = self.stream_block.child_blocks[type_name]
+                value = child_block.to_python(raw_value['value'])
+            else:
+                type_name, value = self.stream_data[i]
+                child_block = self.stream_block.child_blocks[type_name]
+
             self._bound_blocks[i] = StreamValue.StreamChild(child_block, value)
 
         return self._bound_blocks[i]


Test patch:
diff --git a/wagtail/tests/testapp/migrations/0003_streammodel.py b/wagtail/tests/testapp/migrations/0003_streammodel.py
new file mode 100644
--- /dev/null
+++ b/wagtail/tests/testapp/migrations/0003_streammodel.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+import wagtail.wagtailcore.fields
+import wagtail.wagtailcore.blocks
+import wagtail.wagtailimages.blocks
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('tests', '0002_add_verbose_names'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='StreamModel',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('body', wagtail.wagtailcore.fields.StreamField([('text', wagtail.wagtailcore.blocks.CharBlock()), ('image', wagtail.wagtailimages.blocks.ImageChooserBlock())])),
+            ],
+        ),
+    ]
diff --git a/wagtail/tests/testapp/models.py b/wagtail/tests/testapp/models.py
--- a/wagtail/tests/testapp/models.py
+++ b/wagtail/tests/testapp/models.py
@@ -10,7 +10,8 @@
 from modelcluster.tags import ClusterTaggableManager
 
 from wagtail.wagtailcore.models import Page, Orderable
-from wagtail.wagtailcore.fields import RichTextField
+from wagtail.wagtailcore.fields import RichTextField, StreamField
+from wagtail.wagtailcore.blocks import CharBlock
 from wagtail.wagtailadmin.edit_handlers import FieldPanel, MultiFieldPanel, InlinePanel, PageChooserPanel, TabbedInterface, ObjectList
 from wagtail.wagtailimages.edit_handlers import ImageChooserPanel
 from wagtail.wagtaildocs.edit_handlers import DocumentChooserPanel
@@ -19,6 +20,7 @@
 from wagtail.wagtailsnippets.edit_handlers import SnippetChooserPanel
 from wagtail.wagtailsearch import index
 from wagtail.wagtailimages.models import AbstractImage, Image
+from wagtail.wagtailimages.blocks import ImageChooserBlock
 
 
 EVENT_AUDIENCE_CHOICES = (
@@ -400,3 +402,10 @@ class CustomImageWithAdminFormFields(AbstractImage):
     admin_form_fields = Image.admin_form_fields + (
         'caption',
     )
+
+
+class StreamModel(models.Model):
+    body = StreamField([
+        ('text', CharBlock()),
+        ('image', ImageChooserBlock()),
+    ])
diff --git a/wagtail/wagtailcore/tests/test_streamfield.py b/wagtail/wagtailcore/tests/test_streamfield.py
new file mode 100644
--- /dev/null
+++ b/wagtail/wagtailcore/tests/test_streamfield.py
@@ -0,0 +1,72 @@
+import json
+
+from django.test import TestCase
+
+from wagtail.tests.testapp.models import StreamModel
+from wagtail.wagtailimages.models import Image
+from wagtail.wagtailimages.tests.utils import get_test_image_file
+
+
+class TestLazyStreamField(TestCase):
+    def setUp(self):
+        self.image = Image.objects.create(
+            title='Test image',
+            file=get_test_image_file())
+        self.with_image = StreamModel.objects.create(body=json.dumps([
+            {'type': 'image', 'value': self.image.pk},
+            {'type': 'text', 'value': 'foo'}]))
+        self.no_image = StreamModel.objects.create(body=json.dumps([
+            {'type': 'text', 'value': 'foo'}]))
+
+    def test_lazy_load(self):
+        """
+        Getting a single item should lazily load the StreamField, only
+        accessing the database once the StreamField is accessed
+        """
+        with self.assertNumQueries(1):
+            # Get the instance. The StreamField should *not* load the image yet
+            instance = StreamModel.objects.get(pk=self.with_image.pk)
+
+        with self.assertNumQueries(0):
+            # Access the body. The StreamField should still not get the image.
+            body = instance.body
+
+        with self.assertNumQueries(1):
+            # Access the image item from the stream. The image is fetched now
+            body[0].value
+
+        with self.assertNumQueries(0):
+            # Everything has been fetched now, no further database queries.
+            self.assertEqual(body[0].value, self.image)
+            self.assertEqual(body[1].value, 'foo')
+
+    def test_lazy_load_no_images(self):
+        """
+        Getting a single item whose StreamField never accesses the database
+        should behave as expected.
+        """
+        with self.assertNumQueries(1):
+            # Get the instance, nothing else
+            instance = StreamModel.objects.get(pk=self.no_image.pk)
+
+        with self.assertNumQueries(0):
+            # Access the body. The StreamField has no images, so nothing should
+            # happen
+            body = instance.body
+            self.assertEqual(body[0].value, 'foo')
+
+    def test_lazy_load_queryset(self):
+        """
+        Ensure that lazy loading StreamField works when gotten as part of a
+        queryset list
+        """
+        with self.assertNumQueries(1):
+            instances = StreamModel.objects.filter(
+                pk__in=[self.with_image.pk, self.no_image.pk])
+            instances_lookup = {instance.pk: instance for instance in instances}
+
+        with self.assertNumQueries(1):
+            instances_lookup[self.with_image.pk].body[0]
+
+        with self.assertNumQueries(0):
+            instances_lookup[self.no_image.pk].body[0]