Repository: mozmeao/basket

Issue:
Errors when a subscription comes in with an unsupported language
See https://bugzilla.mozilla.org/show_bug.cgi?id=1358143


Discussion:

Resolving patch:
diff --git a/basket/news/utils.py b/basket/news/utils.py
--- a/basket/news/utils.py
+++ b/basket/news/utils.py
@@ -422,6 +422,38 @@ def get_best_request_lang(request):
     return None
 
 
+def _fix_supported_lang(code):
+    """
+    If there's a dash in the locale the second half should be upper case.
+    """
+    if '-' in code:
+        codebits = code.split('-')
+        code = '{}-{}'.format(codebits[0], codebits[1].upper())
+
+    return code
+
+
+def get_best_supported_lang(code):
+    """
+    Take whatever language code we get from the user and return the best one
+    we support for newsletters.
+    """
+    code = str(code).lower()
+    all_langs = [l.lower() for l in newsletter_languages()]
+    if code in all_langs:
+        return _fix_supported_lang(code)
+
+    all_langs_2char = {c[:2]: c for c in all_langs}
+    if len(code) > 2:
+        code2 = code[:2]
+        if code2 in all_langs:
+            return _fix_supported_lang(code2)
+        if code2 in all_langs_2char:
+            return _fix_supported_lang(all_langs_2char[code2])
+
+    return 'en'
+
+
 def process_email(email):
     """Validates that the email is valid.
 
diff --git a/basket/news/views.py b/basket/news/views.py
--- a/basket/news/views.py
+++ b/basket/news/views.py
@@ -41,6 +41,7 @@
     email_is_blocked,
     get_accept_languages,
     get_best_language,
+    get_best_supported_lang,
     get_user_data,
     get_user,
     has_valid_api_key,
@@ -364,6 +365,10 @@ def subscribe_main(request):
             else:
                 del data['lang']
 
+        # now ensure that if we do have a lang that it's a supported one
+        if 'lang' in data:
+            data['lang'] = get_best_supported_lang(data['lang'])
+
         # if source_url not provided we should store the referrer header
         # NOTE this is not a typo; Referrer is misspelled in the HTTP spec
         # https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.36
@@ -807,6 +812,10 @@ def update_user_task(request, api_call_type, data=None, optin=False, sync=False)
         if lang:
             data['lang'] = lang
 
+    # now ensure that if we do have a lang that it's a supported one
+    if 'lang' in data:
+        data['lang'] = get_best_supported_lang(data['lang'])
+
     email = data.get('email')
     token = data.get('token')
     if not (email or token):


Test patch:
diff --git a/basket/news/tests/test__utils.py b/basket/news/tests/test__utils.py
--- a/basket/news/tests/test__utils.py
+++ b/basket/news/tests/test__utils.py
@@ -18,6 +18,7 @@
     has_valid_fxa_oauth,
     is_authorized,
     language_code_is_valid,
+    get_best_supported_lang,
     parse_newsletters_csv,
     parse_phone_number,
     process_email,
@@ -310,6 +311,42 @@ def test_no_langs(self):
         self._test([], None)
 
 
+class TestGetBestSupportedLang(TestCase):
+    def setUp(self):
+        patcher = patch('basket.news.utils.newsletter_languages', return_value=[
+            'de', 'en', 'es', 'fr', 'id', 'pt', 'ru', 'pl', 'hu', 'zh-TW'])
+        self.addCleanup(patcher.stop)
+        patcher.start()
+
+    def test_empty_string(self):
+        """Empty string is accepted as a language code"""
+        self.assertEqual(get_best_supported_lang(''), 'en')
+
+    def test_exact_codes(self):
+        """2 or 5 character code that's in the list is valid"""
+        self.assertEqual(get_best_supported_lang('es'), 'es')
+        self.assertEqual(get_best_supported_lang('zh-TW'), 'zh-TW')
+
+    def test_case_insensitive(self):
+        """Matching is not case sensitive"""
+        self.assertEqual(get_best_supported_lang('ES'), 'es')
+        self.assertEqual(get_best_supported_lang('es-MX'), 'es')
+        self.assertEqual(get_best_supported_lang('PT-br'), 'pt')
+
+    def test_invalid_codes(self):
+        """A code that's not in the list gets the default."""
+        self.assertEqual(get_best_supported_lang('hi'), 'en')
+        self.assertEqual(get_best_supported_lang('hi-IN'), 'en')
+        self.assertEqual(get_best_supported_lang('dude'), 'en')
+
+    def test_weak_match(self):
+        """Matching is best based on first 2 characters."""
+        self.assertEqual(get_best_supported_lang('es-MX'), 'es')
+        self.assertEqual(get_best_supported_lang('pt-BR'), 'pt')
+        self.assertEqual(get_best_supported_lang('en-ZA'), 'en')
+        self.assertEqual(get_best_supported_lang('zh-CN'), 'zh-TW')
+
+
 class TestLanguageCodeIsValid(TestCase):
     def test_empty_string(self):
         """Empty string is accepted as a language code"""
diff --git a/basket/news/tests/test_update_user_task.py b/basket/news/tests/test_update_user_task.py
--- a/basket/news/tests/test_update_user_task.py
+++ b/basket/news/tests/test_update_user_task.py
@@ -49,9 +49,11 @@ def test_invalid_newsletter(self):
             self.assert_response_error(response, 400, errors.BASKET_INVALID_NEWSLETTER)
 
     @patch('basket.news.views.get_best_language')
-    def test_accept_lang(self, get_best_language_mock):
+    @patch('basket.news.utils.newsletter_languages')
+    def test_accept_lang(self, nl_mock, get_best_language_mock):
         """If accept_lang param is provided, should set the lang in data."""
         get_best_language_mock.return_value = 'pt'
+        nl_mock.return_value = ['pt', 'en', 'de']
         request = self.factory.post('/')
         data = {'email': 'dude@example.com', 'accept_lang': 'pt-pt,fr;q=0.8'}
         after_data = {'email': 'dude@example.com', 'lang': 'pt'}
@@ -61,9 +63,11 @@ def test_accept_lang(self, get_best_language_mock):
         self.upsert_user.delay.assert_called_with(SUBSCRIBE, after_data, start_time=ANY)
 
     @patch('basket.news.utils.get_best_language')
-    def test_accept_lang_header(self, get_best_language_mock):
+    @patch('basket.news.utils.newsletter_languages')
+    def test_accept_lang_header(self, nl_mock, get_best_language_mock):
         """If accept-language header is provided, should set the lang in data."""
         get_best_language_mock.return_value = 'pt'
+        nl_mock.return_value = ['pt', 'en', 'de']
         request = self.factory.post('/', HTTP_ACCEPT_LANGUAGE='pt-pt,fr;q=0.8')
         data = {'email': 'dude@example.com'}
         after_data = {'email': 'dude@example.com', 'lang': 'pt'}
@@ -73,12 +77,14 @@ def test_accept_lang_header(self, get_best_language_mock):
         self.upsert_user.delay.assert_called_with(SUBSCRIBE, after_data, start_time=ANY)
 
     @patch('basket.news.utils.get_best_language')
-    def test_lang_overrides_accept_lang(self, get_best_language_mock):
+    @patch('basket.news.utils.newsletter_languages')
+    def test_lang_overrides_accept_lang(self, nl_mock, get_best_language_mock):
         """
         If lang is provided it was from the user, and accept_lang isn't as reliable, so we should
         prefer lang.
         """
         get_best_language_mock.return_value = 'pt-BR'
+        nl_mock.return_value = ['pt', 'en', 'de']
         request = self.factory.post('/')
         data = {'email': 'a@example.com',
                 'lang': 'de',
@@ -89,6 +95,24 @@ def test_lang_overrides_accept_lang(self, get_best_language_mock):
         # basically asserts that the data['lang'] value wasn't changed.
         self.upsert_user.delay.assert_called_with(SUBSCRIBE, data, start_time=ANY)
 
+    @patch('basket.news.utils.get_best_language')
+    @patch('basket.news.utils.newsletter_languages')
+    def test_lang_default_if_not_in_list(self, nl_mock, get_best_language_mock):
+        """
+        If lang is provided it was from the user, and accept_lang isn't as reliable, so we should
+        prefer lang.
+        """
+        get_best_language_mock.return_value = 'pt-BR'
+        nl_mock.return_value = ['pt', 'en', 'de']
+        request = self.factory.post('/')
+        data = {'email': 'a@example.com', 'lang': 'hi'}
+        after_data = {'email': 'a@example.com', 'lang': 'en'}
+
+        response = views.update_user_task(request, SUBSCRIBE, data, sync=False)
+        self.assert_response_ok(response)
+        # basically asserts that the data['lang'] value wasn't changed.
+        self.upsert_user.delay.assert_called_with(SUBSCRIBE, after_data, start_time=ANY)
+
     def test_missing_email(self):
         """
         If the email is missing, return a 400 error.
diff --git a/basket/news/tests/test_views.py b/basket/news/tests/test_views.py
--- a/basket/news/tests/test_views.py
+++ b/basket/news/tests/test_views.py
@@ -515,7 +515,7 @@ def test_lang_instead_of_accept_language(self):
             'newsletters': 'slug',
             'email': 'dude@example.com',
             'privacy': 'true',
-            'lang': 'de',
+            'lang': 'fr',
         }, HTTP_ACCEPT_LANGUAGE='de,fr,en-US')
         assert response.status_code == 200
         self.upsert_user.delay.assert_called_with(SUBSCRIBE, {
@@ -523,7 +523,30 @@ def test_lang_instead_of_accept_language(self):
             'email': 'dude@example.com',
             'privacy': True,
             'format': 'H',
-            'lang': 'de',
+            'lang': 'fr',
+            'first_name': '',
+            'last_name': '',
+            'country': '',
+            'source_url': '',
+        }, start_time=ANY)
+
+    def test_lang_default_if_unsupported(self):
+        # specifying a lang still overrides
+        self.process_email.return_value = 'dude@example.com'
+        self.email_is_blocked.return_value = False
+        response = self._request({
+            'newsletters': 'slug',
+            'email': 'dude@example.com',
+            'privacy': 'true',
+            'lang': 'es',
+        })
+        assert response.status_code == 200
+        self.upsert_user.delay.assert_called_with(SUBSCRIBE, {
+            'newsletters': ['slug'],
+            'email': 'dude@example.com',
+            'privacy': True,
+            'format': 'H',
+            'lang': 'en',
             'first_name': '',
             'last_name': '',
             'country': '',