Repository: mozmeao/snippets-service

Issue:
Reduce ASRSnippetBundle.key time to execute
The `ASRSnippetBundle.key` method is used to decide whether the cached bundle exists and if it's still fresh and thus it gets called often, a lot more compared to the slower `ASRSnippetBundle.generate`.

Metrics taken (on local dev machine) with the Django Profiling app [Silk](https://github.com/jazzband/django-silk) indicate that the time needed to generate a new snippet bundle increased by 70%. Also the time to just return a cached pre-generated bundle doubled. The number of db queries almost doubled and tripled accordingly. 

The new code moves Templates in models so an increase of database queries was expected. Still the response time, especially of `ASRSnippetBundle.key` which handles the vast majority of the requests must be comparable to what we had before. 


Discussion:

Resolving patch:
diff --git a/snippets/base/bundles.py b/snippets/base/bundles.py
--- a/snippets/base/bundles.py
+++ b/snippets/base/bundles.py
@@ -3,6 +3,7 @@
 from datetime import datetime
 from urllib.parse import urljoin, urlparse
 
+from django.apps import apps
 from django.conf import settings
 from django.core.cache import cache
 from django.core.files.base import ContentFile
@@ -13,7 +14,7 @@
 import brotli
 
 from snippets.base import util
-from snippets.base.models import STATUS_CHOICES, ASRSnippet, Snippet
+from snippets.base.models import STATUS_CHOICES, ASRSnippet, Snippet, Template
 
 
 ONE_DAY = 60 * 60 * 24
@@ -46,6 +47,16 @@
         }
     ).encode('utf-8')).hexdigest()
 
+# On application load combine all the version strings of all available
+# templates into one. To be used in ASRSnippetBundle.key method to calculate
+# the bundle key. The point is that this string should change when the Template
+# schema changes.
+TEMPLATES_NG_VERSIONS = '-'.join([
+    model.VERSION
+    for model in apps.get_models()
+    if issubclass(model, Template) and not model.__name__ == 'Template'
+])
+
 
 class SnippetBundle(object):
     """
@@ -180,20 +191,6 @@ def key(self):
                 snippet.modified.isoformat(),
             ]
 
-            # To remove after migration completes. See
-            # https://github.com/mozmeao/snippets-service/issues/933
-            if hasattr(snippet, 'template_relation'):
-                attributes.append(snippet.template_ng.version)
-            else:
-                attributes.append(snippet.template.modified.isoformat())
-
-            attributes.extend(
-                [target.modified.isoformat() for target in snippet.targets.all()]
-            )
-
-            if snippet.campaign:
-                attributes.append(snippet.campaign.modified.isoformat())
-
             key_properties.append('-'.join([str(x) for x in attributes]))
 
         # Additional values used to calculate the key are the templates and the
@@ -202,6 +199,7 @@ def key(self):
             str(self.client.startpage_version),
             self.client.locale,
             str(settings.BUNDLE_BROTLI_COMPRESS),
+            TEMPLATES_NG_VERSIONS,
         ])
 
         key_string = '_'.join(key_properties)
@@ -215,7 +213,7 @@ def filename(self):
     def snippets(self):
         return (ASRSnippet.objects
                 .filter(status=STATUS_CHOICES['Published'])
-                .select_related('template', 'campaign', 'template_relation')
+                .select_related('campaign', 'template_relation')
                 .match_client(self.client)
                 .filter_by_available())
 
@@ -227,7 +225,7 @@ def generate(self):
             'messages': data,
             'metadata': {
                 'generated_at': datetime.utcnow().isoformat(),
-                'number_of_snippets': len(self.snippets),
+                'number_of_snippets': len(data),
             }
         })
 
diff --git a/snippets/base/models.py b/snippets/base/models.py
--- a/snippets/base/models.py
+++ b/snippets/base/models.py
@@ -1586,7 +1586,7 @@ def analytics_export(self):
 # another thing to do when we add more templates and that can be potentially
 # forgotten. Instead we're collecting all signals and we do instance type
 # checking.
-@receiver(post_save, dispatch_uid='update_icon')
+@receiver(post_save, dispatch_uid='update_asrsnippet_modified')
 def update_asrsnippet_modified_date(sender, instance, **kwargs):
     now = timezone.now()
     snippets = None
@@ -1599,6 +1599,11 @@ def update_asrsnippet_modified_date(sender, instance, **kwargs):
         # update() query to work.
         snippets = [id for id in instance.snippets.values_list('pk', flat=True)]
 
+    elif isinstance(instance, Campaign) or isinstance(instance, Target):
+        # Convert the value_list Queryset to a list, required for the upcoming
+        # update() query to work.
+        snippets = [id for id in instance.asrsnippet_set.values_list('pk', flat=True)]
+
     if snippets:
         ASRSnippet.objects.filter(pk__in=snippets).update(modified=now)
 


Test patch:
diff --git a/snippets/base/tests/test_bundles.py b/snippets/base/tests/test_bundles.py
--- a/snippets/base/tests/test_bundles.py
+++ b/snippets/base/tests/test_bundles.py
@@ -327,19 +327,6 @@ def test_key_snippet_modified(self):
         key_2 = bundle.key
         self.assertNotEqual(key_1, key_2)
 
-    def test_key_template_modified(self):
-        client1 = self._client(locale='en-US', startpage_version=4)
-        bundle = ASRSnippetBundle(client1)
-        bundle.snippets = [self.snippet1]
-        key_1 = bundle.key
-
-        # save template, touch modified
-        self.snippet1.template_ng.VERSION = 'new-version'
-        bundle = ASRSnippetBundle(client1)
-        bundle.snippets = [self.snippet1]
-        key_2 = bundle.key
-        self.assertNotEqual(key_1, key_2)
-
     @override_settings(BUNDLE_BROTLI_COMPRESS=False)
     def test_generate(self):
         """
diff --git a/snippets/base/tests/test_models.py b/snippets/base/tests/test_models.py
--- a/snippets/base/tests/test_models.py
+++ b/snippets/base/tests/test_models.py
@@ -546,3 +546,29 @@ def test_modified_date_updates_when_icon_updates(self):
         new_modified = snippet.modified
 
         self.assertNotEqual(old_modified, new_modified)
+
+    def test_modified_date_updates_when_campaign_updates(self):
+        snippet = ASRSnippetFactory()
+        old_modified = snippet.modified
+
+        campaign = snippet.campaign
+        campaign.name = 'new name'
+        campaign.save()
+
+        snippet.refresh_from_db()
+        new_modified = snippet.modified
+
+        self.assertNotEqual(old_modified, new_modified)
+
+    def test_modified_date_updates_when_target_updates(self):
+        new_target = TargetFactory()
+        snippet = ASRSnippetFactory()
+        snippet.targets.add(new_target)
+        old_modified = snippet.modified
+
+        new_target.name = 'new name'
+        new_target.save()
+        snippet.refresh_from_db()
+        new_modified = snippet.modified
+
+        self.assertNotEqual(old_modified, new_modified)