Repository: mozmeao/basket

Issue:
Ensure double-opt-in emails are sent if the user has not clicked to opt-in
We have reports that if a user has attempted to subscribe to a newsletter, but for whatever reason has not clicked the link in the opt-in email to opt-in, then attempts to subscribe again or to another newsletter that they won't receive another email with the opt-in link. I'm guessing this is for cases when a user attempts to subscribe to the same newsletter again since I believe basket will see that as a no-op since they're already subscribed technically. But if they don't have the opt-in then they are in limbo. So we should be checking that and sending the opt-in email to them again.


Discussion:
Definitely something to this. I just tried it a couple of times with the form on the mozorg home page. I got the first confirm email, then verified that data for my test email was in CTMS with a call to `/news/lookup-user/`, then subscribed again, and got no second confirm email. I checked the rate limits and it should have worked. 
Another strange thing is that I also tried subscribing to a newsletter that did not have "Requires double optin" set, which should not send a confirm email, and just set the user as `optin=true` but I did get a confirm email. Something seems to be subtly wrong with our logic because it looks like it should all be working like we expect it from a quick trace through the code, but it's clearly not. 

Resolving patch:
diff --git a/basket/news/newsletters.py b/basket/news/newsletters.py
--- a/basket/news/newsletters.py
+++ b/basket/news/newsletters.py
@@ -91,6 +91,14 @@ def newsletter_inv_map():
     return {v: k for k, v in newsletter_map().items()}
 
 
+def newsletter_obj(slug):
+    """Lookup the newsletter object for the given slug"""
+    try:
+        return _newsletters()["by_name"][slug]
+    except KeyError:
+        return None
+
+
 def newsletter_field(name):
     """Lookup the backend-specific field (vendor ID) for the newsletter"""
     try:
diff --git a/basket/news/tasks.py b/basket/news/tasks.py
--- a/basket/news/tasks.py
+++ b/basket/news/tasks.py
@@ -22,7 +22,7 @@
     Interest,
     Newsletter,
 )
-from basket.news.newsletters import get_transactional_message_ids, newsletter_languages
+from basket.news.newsletters import get_transactional_message_ids, newsletter_languages, newsletter_obj
 from basket.news.utils import (
     SUBSCRIBE,
     UNSUBSCRIBE,
@@ -325,6 +325,7 @@ def upsert_contact(api_call_type, data, user_data):
         to_subscribe_slugs = [nl for nl, sub in update_data["newsletters"].items() if sub]
         check_optin = not (forced_optin or (user_data and user_data.get("optin")))
         check_mofo = not (user_data and user_data.get("mofo_relevant"))
+
         if to_subscribe_slugs and (check_optin or check_mofo):
             to_subscribe = Newsletter.objects.filter(slug__in=to_subscribe_slugs)
 
@@ -381,6 +382,16 @@ def upsert_contact(api_call_type, data, user_data):
     else:
         ctms.update(user_data, update_data)
 
+    # In the rare case the user hasn't confirmed their email and is subscribing to the same newsletter, send the confirmation again.
+    # We catch this by checking if the user `optin` is `False` and if the `update_data["newsletters"]` is empty.
+    if user_data and user_data.get("optin", False) is False and not update_data["newsletters"]:
+        newsletter_objs = filter(None, [newsletter_obj(n) for n in newsletters])
+        if newsletter_objs:
+            needs_confirmation = any(n.requires_double_optin for n in newsletter_objs)
+            if needs_confirmation:
+                send_fx_confirm = all(n.firefox_confirm for n in newsletter_objs)
+                send_confirm = "fx" if send_fx_confirm else "moz"
+
     if send_confirm and settings.SEND_CONFIRM_MESSAGES:
         send_confirm_message.delay(
             data["email"],


Test patch:
diff --git a/basket/news/tests/test_newsletters.py b/basket/news/tests/test_newsletters.py
--- a/basket/news/tests/test_newsletters.py
+++ b/basket/news/tests/test_newsletters.py
@@ -52,6 +52,11 @@ def setUp(self):
         ]
         self.groupies[0].newsletters.add(self.newsies[1], self.newsies[2])
 
+    def test_newsletter_obj(self):
+        self.assertEqual(newsletters.newsletter_obj("bowling"), self.newsies[0])
+        self.assertEqual(newsletters.newsletter_obj("surfing"), self.newsies[1])
+        self.assertIsNone(newsletters.newsletter_obj("failing"))
+
     def test_newsletter_private_slugs(self):
         self.assertEqual(newsletters.newsletter_private_slugs(), ["papers"])
 
diff --git a/basket/news/tests/test_upsert_user.py b/basket/news/tests/test_upsert_user.py
--- a/basket/news/tests/test_upsert_user.py
+++ b/basket/news/tests/test_upsert_user.py
@@ -29,12 +29,12 @@ def setUp(self):
 
     def test_update_first_last_names(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
         """sending name fields should result in names being passed to SF/CTMS"""
-        get_user_data.return_value = None  # Does not exist yet
+        get_user_mock.return_value = None  # Does not exist yet
         email_id = str(uuid4())
         ctms_mock.add.return_value = {"email": {"email_id": email_id}}
         models.Newsletter.objects.create(
@@ -63,7 +63,7 @@ def test_update_first_last_names(
 
     def test_update_user_set_works_if_no_newsletters(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
@@ -90,24 +90,24 @@ def test_update_user_set_works_if_no_newsletters(
         update_data = data.copy()
         update_data["newsletters"] = {"slug": False}
 
-        get_user_data.return_value = self.get_user_data
+        get_user_mock.return_value = self.get_user_data
 
         upsert_user(SET, data)
         # We should have looked up the user's data
-        get_user_data.assert_called()
+        get_user_mock.assert_called()
         # We'll specifically unsubscribe each newsletter the user is
         # subscribed to.
         ctms_mock.update.assert_called_with(self.get_user_data, update_data)
 
     def test_resubscribe_doesnt_update_newsletter(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
         """
         When subscribing to things the user is already subscribed to, we
-        do not pass that newsletter to SF/CTMS because we don't want that newsletter
+        do not pass that newsletter to CTMS because we don't want that newsletter
         to be updated for no reason as that could cause another welcome to be sent.
         """
         models.Newsletter.objects.create(
@@ -128,23 +128,23 @@ def test_resubscribe_doesnt_update_newsletter(
         update_data = data.copy()
         update_data["newsletters"] = {}
 
-        get_user_data.return_value = self.get_user_data
+        get_user_mock.return_value = self.get_user_data
 
         upsert_user(SUBSCRIBE, data)
         # We should have looked up the user's data
-        get_user_data.assert_called()
+        get_user_mock.assert_called()
         # We should not have mentioned this newsletter in our call to ET
         ctms_mock.update.assert_called_with(self.get_user_data, update_data)
 
     def test_set_doesnt_update_newsletter(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
         """
         When setting the newsletters to ones the user is already subscribed
-        to, we do not pass that newsletter to SF/CTMS because we
+        to, we do not pass that newsletter to CTMS because we
         don't want that newsletter to send a new welcome.
         """
         models.Newsletter.objects.create(
@@ -167,15 +167,15 @@ def test_set_doesnt_update_newsletter(
         update_data["newsletters"] = {}
 
         # Mock user data - we want our user subbed to our newsletter to start
-        get_user_data.return_value = self.get_user_data
+        get_user_mock.return_value = self.get_user_data
 
         upsert_user(SET, data)
         # We should have looked up the user's data
-        self.assertTrue(get_user_data.called)
-        # We should not have mentioned this newsletter in our call to SF/CTMS
+        self.assertTrue(get_user_mock.called)
+        # We should not have mentioned this newsletter in our call to CTMS
         ctms_mock.update.assert_called_with(self.get_user_data, update_data)
 
-    def test_unsub_is_careful(self, get_user_data, ctms_mock, confirm_mock):
+    def test_unsub_is_careful(self, get_user_mock, ctms_mock, confirm_mock):
         """
         When unsubscribing, we only unsubscribe things the user is
         currently subscribed to.
@@ -205,11 +205,11 @@ def test_unsub_is_careful(self, get_user_data, ctms_mock, confirm_mock):
         update_data = data.copy()
         # We should only mention slug, not slug2
         update_data["newsletters"] = {"slug": False}
-        get_user_data.return_value = self.get_user_data
+        get_user_mock.return_value = self.get_user_data
 
         upsert_user(UNSUBSCRIBE, data)
         # We should have looked up the user's data
-        self.assertTrue(get_user_data.called)
+        self.assertTrue(get_user_mock.called)
         ctms_mock.update.assert_called_with(self.get_user_data, update_data)
 
     def test_update_user_without_format_doesnt_send_format(
@@ -219,11 +219,11 @@ def test_update_user_without_format_doesnt_send_format(
         confirm_mock,
     ):
         """
-        SF/CTMS format not changed if update_user call doesn't specify.
+        CTMS format not changed if update_user call doesn't specify.
 
         If update_user call doesn't specify a format (e.g. if bedrock
         doesn't get a changed value on a form submission), then Basket
-        doesn't send any format to SF/CTMS.
+        doesn't send any format to CTMS.
 
         It does use the user's choice of format to send them their
         welcome message.
@@ -293,9 +293,9 @@ def test_update_user_with_email_id(
         upsert_user(SUBSCRIBE, data)
         ctms_mock.update.assert_called_with(get_user_mock.return_value, update_data)
 
-    def test_send_confirm(self, get_user_data, ctms_mock, confirm_mock):
+    def test_send_confirm(self, get_user_mock, ctms_mock, confirm_mock):
         """Subscribing to a newsletter should send a confirm email"""
-        get_user_data.return_value = None  # Does not exist yet
+        get_user_mock.return_value = None  # Does not exist yet
         email_id = str(uuid4())
         ctms_mock.add.return_value = {"email": {"email_id": email_id}}
         models.Newsletter.objects.create(
@@ -321,9 +321,9 @@ def test_send_confirm(self, get_user_data, ctms_mock, confirm_mock):
         update_data["email_id"] = email_id
         confirm_mock.delay.assert_called_with(self.email, ANY, "en", "moz")
 
-    def test_send_fx_confirm(self, get_user_data, ctms_mock, confirm_mock):
+    def test_send_fx_confirm(self, get_user_mock, ctms_mock, confirm_mock):
         """Subscribing to a Fx newsletter should send a Fx confirm email"""
-        get_user_data.return_value = None  # Does not exist yet
+        get_user_mock.return_value = None  # Does not exist yet
         email_id = str(uuid4())
         ctms_mock.add.return_value = {"email": {"email_id": email_id}}
         models.Newsletter.objects.create(
@@ -350,9 +350,9 @@ def test_send_fx_confirm(self, get_user_data, ctms_mock, confirm_mock):
         update_data["email_id"] = email_id
         confirm_mock.delay.assert_called_with(self.email, ANY, "en", "fx")
 
-    def test_send_moz_confirm(self, get_user_data, ctms_mock, confirm_mock):
+    def test_send_moz_confirm(self, get_user_mock, ctms_mock, confirm_mock):
         """Subscribing to a Fx and moz newsletters should send a moz confirm email"""
-        get_user_data.return_value = None  # Does not exist yet
+        get_user_mock.return_value = None  # Does not exist yet
         email_id = str(uuid4())
         ctms_mock.add.return_value = {"email": {"email_id": email_id}}
         models.Newsletter.objects.create(
@@ -390,7 +390,7 @@ def test_send_moz_confirm(self, get_user_data, ctms_mock, confirm_mock):
 
     def test_no_send_confirm_newsletter(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
@@ -398,7 +398,7 @@ def test_no_send_confirm_newsletter(
         Subscribing to a newsletter should not send a confirm email
         if the newsletter does not require it
         """
-        get_user_data.return_value = None  # Does not exist yet
+        get_user_mock.return_value = None  # Does not exist yet
         email_id = str(uuid4())
         ctms_mock.add.return_value = {"email": {"email_id": email_id}}
         models.Newsletter.objects.create(
@@ -427,7 +427,7 @@ def test_no_send_confirm_newsletter(
 
     def test_no_send_confirm_user(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
@@ -438,7 +438,7 @@ def test_no_send_confirm_user(
         user_data = self.get_user_data.copy()
         user_data["optin"] = True
         user_data["newsletters"] = ["not-slug"]
-        get_user_data.return_value = user_data
+        get_user_mock.return_value = user_data
         models.Newsletter.objects.create(
             slug="slug",
             title="title",
@@ -460,15 +460,83 @@ def test_no_send_confirm_user(
         ctms_mock.update.assert_called_with(user_data, update_data)
         confirm_mock.delay.assert_not_called()
 
+    def test_send_confirm_optin_false_double_opt_in(
+        self,
+        get_user_mock,
+        ctms_mock,
+        confirm_mock,
+    ):
+        """
+        Re-subscribing to a newsletter should send a confirm email if the user is not already confirmed
+        """
+        user_data = self.get_user_data.copy()
+        user_data["optin"] = False
+        user_data["newsletters"] = ["slug"]
+        get_user_mock.return_value = user_data
+        models.Newsletter.objects.create(
+            slug="slug",
+            title="title",
+            active=True,
+            languages="en,fr",
+            vendor_id="VENDOR1",
+            requires_double_optin=True,
+        )
+        data = {
+            "country": "US",
+            "lang": "en",
+            "format": "H",
+            "newsletters": "slug",
+            "email": self.email,
+        }
+        upsert_user(SUBSCRIBE, data)
+        update_data = data.copy()
+        update_data["newsletters"] = {}  # Gets stripped since it's already subscribed to.
+        ctms_mock.update.assert_called_with(user_data, update_data)
+        confirm_mock.delay.assert_called()
+
+    def test_send_confirm_optin_false_not_double_opt_in(
+        self,
+        get_user_mock,
+        ctms_mock,
+        confirm_mock,
+    ):
+        """
+        Re-subscribing to a newsletter should not send a confirm email if the user is not already
+        confirmed but the newsletter does not require double opt-in.
+        """
+        user_data = self.get_user_data.copy()
+        user_data["optin"] = False
+        user_data["newsletters"] = ["slug"]
+        get_user_mock.return_value = user_data
+        models.Newsletter.objects.create(
+            slug="slug",
+            title="title",
+            active=True,
+            languages="en,fr",
+            vendor_id="VENDOR1",
+            requires_double_optin=False,
+        )
+        data = {
+            "country": "US",
+            "lang": "en",
+            "format": "H",
+            "newsletters": "slug",
+            "email": self.email,
+        }
+        upsert_user(SUBSCRIBE, data)
+        update_data = data.copy()
+        update_data["newsletters"] = {}  # Gets stripped since it's already subscribed to.
+        ctms_mock.update.assert_called_with(user_data, update_data)
+        confirm_mock.delay.assert_not_called()
+
     def test_new_subscription_with_ctms_conflict(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
-        """When CTMS returns an error for a new contact, the email_id is not
-        sent to SF"""
-        get_user_data.return_value = None  # Does not exist yet
+        """Test when CTMS returns an error for a new contact"""
+        get_user_mock.return_value = None  # Does not exist yet
         ctms_mock.add.return_value = None  # Conflict on create
         models.Newsletter.objects.create(
             slug="slug",
@@ -494,13 +562,13 @@ def test_new_subscription_with_ctms_conflict(
 
     def test_new_user_subscribes_to_mofo_newsletter(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
         """Subscribing to a MoFo-relevant newsletter makes the new user
         mofo-relevant."""
-        get_user_data.return_value = None  # Does not exist yet
+        get_user_mock.return_value = None  # Does not exist yet
         email_id = str(uuid4())
         ctms_mock.add.return_value = {"email": {"email_id": email_id}}
         models.Newsletter.objects.create(
@@ -531,13 +599,13 @@ def test_new_user_subscribes_to_mofo_newsletter(
 
     def test_existing_user_subscribes_to_mofo_newsletter(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
         """Subscribing to a MoFo-relevant newsletter makes the user mofo-relevant."""
         user_data = self.get_user_data.copy()
-        get_user_data.return_value = user_data
+        get_user_mock.return_value = user_data
         models.Newsletter.objects.create(
             slug="mozilla-foundation",
             title="The Mozilla Foundation News",
@@ -564,14 +632,14 @@ def test_existing_user_subscribes_to_mofo_newsletter(
 
     def test_existing_mofo_user_subscribes_to_mofo_newsletter(
         self,
-        get_user_data,
+        get_user_mock,
         ctms_mock,
         confirm_mock,
     ):
         """If a user is already MoFo-relevant, a subscription does not set it again."""
         user_data = self.get_user_data.copy()
         user_data["mofo_relevant"] = True
-        get_user_data.return_value = user_data
+        get_user_mock.return_value = user_data
         models.Newsletter.objects.create(
             slug="mozilla-foundation",
             title="The Mozilla Foundation News",