Repository: mozmeao/snippets-service

Issue:
Create view to redirect clients to the correct pregenerated bundle
## Why

Because custom Firefox builds alter the %CHANNEL% variable we cannot directly point to a single file. Instead we will deploy Lambda@Edge functions which will convert the request URL to an expected format, which is practically change %CHANNEL%-* to %CHANNEL%

#### List of highly used settings for %CHANNEL%
- release
- release-cck-ubuntu
- beta
- release-cck-mozillaonline
- nightly
- nrelease-cck- âmintâ
- aurora
- 641165 release-cck-yahoo
- release-cck-archlinux
- release-cck-mint
- release-cck-fedora
- release-cck-euballot

This must return the actual channel (`release`, `beta`, etc)

## Tasks
- [x] Remove deprecated Lambda functions from snippets stage cdn
- [x] What happens with redirects which 404? Maybe that generates a lot of traffic? We either implement #1113 or we check on the app level if file exists in S3 and if not we return 204 instead of 302. Or we do nothing is client is fine with 404? We handle that on the CDN level. See #1113 for comments.
- [x]  Confirm that we can change Snippets request URL for all Quantum versions via Normandy
- [x] Create `empty.json` on snippets-prod s3 bucket with Content-type and Cache-contorl headers


Discussion:
Hey @metadave 

If I'm not mistaken I remember you offering to setup me up with some TF to create a Lambda @ Edge function to use with the Snippets CloudFront Distribution.

I need a python  function at the Origin Request level (https://docs.aws.amazon.com/lambda/latest/dg/lambda-edge.html) for distribution d31g0e3nurrasj.cloudfront.net (stage) and another for d228z91au11ukj.cloudfront.net (prod)

Is this something you can do?

Thanks!

> Is this something you can do?

yes, but it looks like there isn't any TF automation for the stage CDN. I'll have to see if I can import it and go from there.
WIP here: https://github.com/mozmeao/infra/pull/1159

Resolving patch:
diff --git a/snippets/base/views.py b/snippets/base/views.py
--- a/snippets/base/views.py
+++ b/snippets/base/views.py
@@ -1,9 +1,11 @@
 import json
+from urllib.parse import urljoin
 
 from distutils.util import strtobool
 from django.conf import settings
 from django.contrib.auth.decorators import permission_required
 from django.core.exceptions import ValidationError
+from django.core.files.storage import default_storage
 from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseRedirect
 from django.shortcuts import get_object_or_404, render
 from django.utils.functional import lazy
@@ -20,7 +22,7 @@
 from snippets.base.bundles import ASRSnippetBundle, SnippetBundle
 from snippets.base.decorators import access_control
 from snippets.base.filters import JobFilter
-from snippets.base.models import ASRSnippet, Client, Snippet, SnippetTemplate
+from snippets.base.models import CHANNELS, ASRSnippet, Client, Snippet, SnippetTemplate
 from snippets.base.util import get_object_or_none
 
 
@@ -44,9 +46,33 @@ def template_name(self):
         return 'base/jobs_list_table.jinja'
 
 
+def fetch_snippets(request, **kwargs):
+    if settings.USE_PREGEN_BUNDLES:
+        return fetch_snippet_pregen_bundle(request, **kwargs)
+    return fetch_snippet_bundle(request, **kwargs)
+
+
+@cache_control(public=True, max_age=settings.SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT)
+def fetch_snippet_pregen_bundle(request, **kwargs):
+    client = Client(**kwargs)
+    product = 'Firefox'
+    channel = client.channel.lower()
+    channel = next((item for item in CHANNELS if channel.startswith(item)), None) or 'release'
+    locale = client.locale.lower()
+    distribution = client.distribution.lower()
+
+    filename = default_storage.url(
+        f'{settings.MEDIA_BUNDLES_PREGEN_ROOT}/{product}/{channel}/'
+        f'{locale}/{distribution}.json'
+    )
+    full_url = urljoin(settings.CDN_URL or settings.SITE_URL, filename)
+
+    return HttpResponseRedirect(full_url)
+
+
 @cache_control(public=True, max_age=SNIPPET_BUNDLE_TIMEOUT)
 @access_control(max_age=SNIPPET_BUNDLE_TIMEOUT)
-def fetch_snippets(request, **kwargs):
+def fetch_snippet_bundle(request, **kwargs):
     """
     Return one of the following responses:
     - 200 with empty body when the bundle is empty
diff --git a/snippets/settings.py b/snippets/settings.py
--- a/snippets/settings.py
+++ b/snippets/settings.py
@@ -209,6 +209,8 @@
 CSRF_COOKIE_SAMESITE = config('CSRF_COOKIE_SAMESITE', default='Lax')
 
 SNIPPET_BUNDLE_TIMEOUT = config('SNIPPET_BUNDLE_TIMEOUT', default=15 * 60, cast=int)  # 15 minutes
+SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT = config(
+    'SNIPPET_BUNDLE_PREGEN_REDIRECT_TIMEOUT', default=60 * 60 * 24, cast=int)  # One day
 
 BUNDLE_BROTLI_COMPRESS = config('BUNDLE_BROTLI_COMPRESS', default=False, cast=bool)
 
@@ -370,9 +372,10 @@
 # In minutes
 SNIPPETS_PUBLICATION_OFFSET = config('SNIPPETS_PUBLICATION_OFFSET', default=5, cast=int)
 
-
 REDASH_ENDPOINT = config('REDASH_ENDPOINT', default='https://sql.telemetry.mozilla.org')
 REDASH_API_KEY = config('REDASH_API_KEY', default=None)
 REDASH_MAX_WAIT = config('REDASH_MAX_WAIT', default=300)
 REDASH_QUERY_ID = config('REDASH_QUERY_ID', default=63146)
 REDASH_UPDATE_INTERVAL = config('REDASH_UPDATE_INTERVAL', default=600)
+
+USE_PREGEN_BUNDLES = config('USE_PREGEN_BUNDLES', default=False, cast=bool)
diff --git a/snippets/urls.py b/snippets/urls.py
--- a/snippets/urls.py
+++ b/snippets/urls.py
@@ -44,8 +44,9 @@ def robots_txt(request):
     def serve_media(*args, **kwargs):
         response = static_serve(*args, **kwargs)
         response['Access-Control-Allow-Origin'] = '*'
-        if ((settings.BUNDLE_BROTLI_COMPRESS and
-             kwargs['path'].startswith(settings.MEDIA_BUNDLES_ROOT))):
+        if (settings.BUNDLE_BROTLI_COMPRESS and (
+                kwargs['path'].startswith(settings.MEDIA_BUNDLES_ROOT) or
+                kwargs['path'].startswith(settings.MEDIA_BUNDLES_PREGEN_ROOT))):
             response['Content-Encoding'] = 'br'
         return response
 


Test patch:
diff --git a/snippets/base/tests/test_views.py b/snippets/base/tests/test_views.py
--- a/snippets/base/tests/test_views.py
+++ b/snippets/base/tests/test_views.py
@@ -1,11 +1,11 @@
 from collections import OrderedDict
+from unittest.mock import DEFAULT, patch
 
 from django.contrib.auth.models import User
 from django.test.client import RequestFactory
 from django.test.utils import override_settings
 from django.urls import reverse
 
-from unittest.mock import patch
 
 import snippets.base.models
 from snippets.base import views
@@ -107,6 +107,92 @@ def test_uuid_snippet(self):
 
 
 class FetchSnippetsTests(TestCase):
+    def test_base(self):
+        asrclient_kwargs = OrderedDict([
+            ('startpage_version', 6),
+            ('name', 'Firefox'),
+            ('version', '64.0'),
+            ('appbuildid', '20190110041606'),
+            ('build_target', 'Darwin_Universal-gcc3'),
+            ('locale', 'en-US'),
+            ('channel', 'release'),
+            ('os_version', 'Darwin 10.8.0'),
+            ('distribution', 'default'),
+            ('distribution_version', 'default_version'),
+        ])
+        request = RequestFactory().get('/')
+
+        with patch.multiple('snippets.base.views',
+                            fetch_snippet_pregen_bundle=DEFAULT,
+                            fetch_snippet_bundle=DEFAULT) as patches:
+            with override_settings(USE_PREGEN_BUNDLES=False):
+                views.fetch_snippets(request, **asrclient_kwargs)
+                self.assertFalse(patches['fetch_snippet_pregen_bundle'].called)
+                self.assertTrue(patches['fetch_snippet_bundle'].called)
+
+        with patch.multiple('snippets.base.views',
+                            fetch_snippet_pregen_bundle=DEFAULT,
+                            fetch_snippet_bundle=DEFAULT) as patches:
+            with override_settings(USE_PREGEN_BUNDLES=True):
+                views.fetch_snippets(request, **asrclient_kwargs)
+                self.assertTrue(patches['fetch_snippet_pregen_bundle'].called)
+                self.assertFalse(patches['fetch_snippet_bundle'].called)
+
+
+@override_settings(SITE_URL='http://example.org',
+                   MEDIA_BUNDLES_PREGEN_ROOT='/bundles/pregen/')
+class FetchSnippetPregenBundleTests(TestCase):
+    def setUp(self):
+        self.factory = RequestFactory()
+        self.request = self.factory.get('/')
+        self.asrclient_kwargs = OrderedDict([
+            ('startpage_version', 6),
+            ('name', 'Firefox'),
+            ('version', '70.0'),
+            ('appbuildid', '20190110041606'),
+            ('build_target', 'Darwin_Universal-gcc3'),
+            ('locale', 'el-GR'),
+            ('channel', 'default'),
+            ('os_version', 'Darwin 10.8.0'),
+            ('distribution', 'other-than-default'),
+            ('distribution_version', 'default_version'),
+        ])
+
+    def test_base(self):
+        response = views.fetch_snippet_pregen_bundle(self.request, **self.asrclient_kwargs)
+        expected_url = (
+            'http://example.org/media/bundles/pregen/Firefox/release/el-gr/other-than-default.json'
+        )
+        self.assertEqual(response.url, expected_url)
+
+    @override_settings(CDN_URL='https://cdn.com')
+    def test_cdn(self):
+        response = views.fetch_snippet_pregen_bundle(self.request, **self.asrclient_kwargs)
+        expected_url = (
+            'https://cdn.com/media/bundles/pregen/Firefox/release/el-gr/other-than-default.json'
+        )
+        self.assertEqual(response.url, expected_url)
+
+    def test_complicated_channel(self):
+        asrclient_kwargs = self.asrclient_kwargs.copy()
+        asrclient_kwargs['channel'] = 'nightly-cck-δφια'
+        response = views.fetch_snippet_pregen_bundle(self.request, **asrclient_kwargs)
+        expected_url = (
+            'http://example.org/media/bundles/pregen/Firefox/nightly/el-gr/other-than-default.json'
+        )
+        self.assertEqual(response.url, expected_url)
+
+    def test_other_product(self):
+        asrclient_kwargs = self.asrclient_kwargs.copy()
+        asrclient_kwargs['name'] = 'Edge'
+        response = views.fetch_snippet_pregen_bundle(self.request, **asrclient_kwargs)
+        expected_url = (
+            'http://example.org/media/bundles/pregen/Firefox/release/el-gr/other-than-default.json'
+        )
+        self.assertEqual(response.url, expected_url)
+
+
+class FetchSnippetBundleTests(TestCase):
     def setUp(self):
         self.factory = RequestFactory()
         self.request = self.factory.get('/')
@@ -141,7 +227,7 @@ def test_normal(self):
             bundle.url = '/foo/bar'
             bundle.empty = False
             bundle.cached = True
-            response = views.fetch_snippets(self.request, **self.client_kwargs)
+            response = views.fetch_snippet_bundle(self.request, **self.client_kwargs)
 
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response['Location'], '/foo/bar')
@@ -158,7 +244,7 @@ def test_normal_asr(self):
             bundle.url = '/foo/bar'
             bundle.empty = False
             bundle.cached = True
-            response = views.fetch_snippets(self.request, **self.asrclient_kwargs)
+            response = views.fetch_snippet_bundle(self.request, **self.asrclient_kwargs)
 
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response['Location'], '/foo/bar')
@@ -176,7 +262,7 @@ def test_regenerate(self):
             bundle.url = '/foo/bar'
             bundle.empty = False
             bundle.cached = False
-            response = views.fetch_snippets(self.request, **self.client_kwargs)
+            response = views.fetch_snippet_bundle(self.request, **self.client_kwargs)
 
         self.assertEqual(response.status_code, 302)
         self.assertEqual(response['Location'], '/foo/bar')
@@ -189,7 +275,7 @@ def test_empty(self):
         with patch.object(views, 'SnippetBundle') as SnippetBundle:
             bundle = SnippetBundle.return_value
             bundle.empty = True
-            response = views.fetch_snippets(self.request, **self.client_kwargs)
+            response = views.fetch_snippet_bundle(self.request, **self.client_kwargs)
 
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b'')
@@ -199,7 +285,7 @@ def test_empty_asr(self):
         with patch.object(views, 'ASRSnippetBundle') as ASRSnippetBundle:
             bundle = ASRSnippetBundle.return_value
             bundle.empty = True
-            response = views.fetch_snippets(self.request, **self.asrclient_kwargs)
+            response = views.fetch_snippet_bundle(self.request, **self.asrclient_kwargs)
 
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b'{}')