Repository: mozmeao/basket

Issue:
Allow clearing existing CTMS values
If a CTMS field is None or an empty string, the field is not sent to CTMS. When the change is an update to CTMS, and the existing field is populated, an empty update should be allowed, to allow clearing data for existing contacts.


Discussion:

Resolving patch:
diff --git a/basket/news/backends/ctms.py b/basket/news/backends/ctms.py
--- a/basket/news/backends/ctms.py
+++ b/basket/news/backends/ctms.py
@@ -154,7 +154,7 @@ def process_country(raw_country):
             country = new_country
 
     if country not in SFDC_COUNTRIES_LIST:
-        return None
+        raise ValueError(f"{country} not in SFDC_COUNTRIES_LIST")
     return country
 
 
@@ -173,12 +173,12 @@ def process_lang(raw_lang):
 def truncate_string(max_length, raw_string):
     """Truncate the a string to a maximum length, and return None for empty."""
     if raw_string is None:
-        return None
+        raise ValueError("expected string, got None")
     string = raw_string.strip()
     if len(string) > max_length:
         statsd.incr("news.backends.ctms.data_truncated")
         return string[:max_length]
-    return string or None
+    return string
 
 
 TO_VENDOR_PROCESSORS = {
@@ -205,7 +205,7 @@ def __str__(self):
         return f"Unknown basket key {self.unknown_key!r}"
 
 
-def to_vendor(data):
+def to_vendor(data, existing_data=None):
     """
     Transform basket key-value data and convert to CTMS nested data
 
@@ -213,21 +213,31 @@ def to_vendor(data):
     * No equivalent to Subscriber__c, UAT_Test_Data__c
     * Doesn't convert SFDC values Browser_Locale__c, FSA_*, CV_*, MailingCity
     * CTMS API handles boolean conversion
+    * Allows setting a value to an empty string or None, if existing is set.
 
-    @params data: basket data
+    @params data: data to update, basket format
+    @params existing_data: existing user data, basket format
     @return: dict in CTMS format
     """
     ctms_data = {}
     amo_deleted = False
-    unsub_all = False
+    newsletters = None
+    newsletter_subscription_default = {}
+    existing_data = existing_data or {}
+    if "lang" in existing_data:
+        default_lang = process_lang(existing_data["lang"])
+        newsletter_subscription_default["lang"] = default_lang
+    if "format" in existing_data:
+        newsletter_subscription_default["format"] = existing_data["format"]
 
     for name, raw_value in data.items():
         # Pre-process raw_value, which may remove it.
         processor = TO_VENDOR_PROCESSORS.get(name)
         if processor:
-            value = processor(raw_value)
-            if value is None:
-                continue
+            try:
+                value = processor(raw_value)
+            except ValueError:
+                continue  # Skip invalid values
         else:
             value = raw_value
 
@@ -236,56 +246,65 @@ def to_vendor(data):
             value = value.strip()
         except AttributeError:
             pass
-        if value is None or value == "":
-            # TODO: Should we allow clearing previously set values?
+        # Skip empty values if new record or also unset in existing data
+        if (value is None or value == "") and not existing_data.get(name):
             continue
 
         # Place in CTMS contact structure
         if name in BASKET_TO_CTMS_NAMES:
             group_name, key = BASKET_TO_CTMS_NAMES[name]
             ctms_data.setdefault(group_name, {})[key] = value
+            if name in {"lang", "format"}:
+                newsletter_subscription_default[name] = value
         elif name == "newsletters":
-            valid_slugs = newsletter_slugs()
-            output = []
-            if isinstance(raw_value, dict):
-                # Detect unsubscribe all
-                optout = data.get("optout", False) or False
-                if (
-                    optout
-                    and (not any(raw_value.values()))
-                    and (set(valid_slugs) == set(raw_value.keys()))
-                ):
-                    unsub_all = True
-                else:
-                    # Dictionary of slugs to sub/unsub flags
-                    for slug, subscribed in raw_value.items():
-                        if slug in valid_slugs:
-                            output.append(
-                                {"name": slug, "subscribed": bool(subscribed)}
-                            )
-            else:
-                # List of slugs for subscriptions
-                source_url = (data.get("source_url", "") or "").strip()
-                for slug in raw_value:
-                    if slug in valid_slugs:
-                        sub = {"name": slug, "subscribed": True}
-                        if source_url:
-                            sub["source"] = source_url
-                        output.append(sub)
-            if output:
-                ctms_data["newsletters"] = output
+            # Process newsletters after gathering all newsletter keys
+            newsletters = value
         elif name == "amo_deleted":
             amo_deleted = bool(value)
         elif name not in DISCARD_BASKET_NAMES:
             # TODO: SFDC ignores unknown fields, maybe this should as well
             raise CTMSUnknownKeyError(name)
 
+    # Process the newsletters, which may include extra data from the email group
+    if newsletters:
+        valid_slugs = newsletter_slugs()
+        output = []
+        if isinstance(newsletters, dict):
+            # Detect unsubscribe all
+            optout = data.get("optout", False) or False
+            if (
+                optout
+                and (not any(newsletters.values()))
+                and (set(valid_slugs) == set(newsletters.keys()))
+            ):
+                # When unsubscribe all is requested, let CTMS unsubscribe from all
+                output = "UNSUBSCRIBE"
+            else:
+                # Dictionary of slugs to sub/unsub flags
+                for slug, subscribed in newsletters.items():
+                    if slug in valid_slugs:
+                        if subscribed:
+                            nl_sub = newsletter_subscription_default.copy()
+                            nl_sub.update({"name": slug, "subscribed": True})
+                        else:
+                            nl_sub = {"name": slug, "subscribed": False}
+                        output.append(nl_sub)
+        else:
+            # List of slugs for subscriptions, which may include a source
+            source_url = (data.get("source_url", "") or "").strip()
+            if source_url:
+                newsletter_subscription_default["source"] = source_url
+            for slug in newsletters:
+                if slug in valid_slugs:
+                    nl_sub = newsletter_subscription_default.copy()
+                    nl_sub.update({"name": slug, "subscribed": True})
+                    output.append(nl_sub)
+        if output:
+            ctms_data["newsletters"] = output
+
     # When an AMO account is deleted, reset data to defaults
     if amo_deleted:
         ctms_data["amo"] = "DELETE"
-    # When unsubscribe all is requested, let CTMS unsubscribe from all
-    if unsub_all:
-        ctms_data["newsletters"] = "UNSUBSCRIBE"
 
     return ctms_data
 
@@ -706,7 +725,8 @@ def update(self, existing_data, update_data):
             # TODO: When CTMS is primary, this should be an error
             return None
         try:
-            return self.interface.patch_by_email_id(email_id, to_vendor(update_data))
+            ctms_data = to_vendor(update_data, existing_data)
+            return self.interface.patch_by_email_id(email_id, ctms_data)
         except Exception:
             sentry_sdk.capture_exception()
             return None


Test patch:
diff --git a/basket/news/tests/test_ctms.py b/basket/news/tests/test_ctms.py
--- a/basket/news/tests/test_ctms.py
+++ b/basket/news/tests/test_ctms.py
@@ -181,7 +181,14 @@ def test_sample_format(self, mock_nl_slugs):
                 "primary_email": "my-fxa-acct@example.com",
             },
             "mofo": {"mofo_relevant": False},
-            "newsletters": [{"name": "mozilla-welcome", "subscribed": True}],
+            "newsletters": [
+                {
+                    "name": "mozilla-welcome",
+                    "subscribed": True,
+                    "format": "H",
+                    "lang": "en",
+                }
+            ],
             "vpn_waitlist": {"geo": "fr", "platform": "ios,mac"},
         }
 
@@ -261,6 +268,29 @@ def test_truncate_empty_to_none(self):
         prepared = to_vendor(data)
         assert prepared == {}
 
+    def test_allow_rewrite_to_empty(self):
+        """Allow setting to an empty string or None if there is an existing value"""
+        existing_data = {
+            "first_name": "Walter",
+            "last_name": "Sobchak",
+            "reason": "",
+            "fxa_id": "12345",
+            "amo_id": 54321,
+        }
+        data = {
+            "first_name": " ",
+            "last_name": "\t",
+            "reason": "",
+            "fxa_id": None,
+            "amo_id": None,
+        }
+        prepared = to_vendor(data, existing_data)
+        assert prepared == {
+            "amo": {"user_id": None},
+            "email": {"first_name": "", "last_name": ""},
+            "fxa": {"fxa_id": None},
+        }
+
     @patch(
         "basket.news.backends.ctms.newsletter_slugs",
         return_value=["slug1", "slug2", "slug3", "slug4"],
@@ -277,23 +307,72 @@ def test_newsletter_list(self, mock_nl_slugs):
             ]
         }
 
+    @patch("basket.news.newsletters.newsletter_languages", return_value=["en", "es"])
     @patch(
         "basket.news.backends.ctms.newsletter_slugs",
         return_value=["slug1", "slug2", "slug3", "slug4"],
     )
-    def test_newsletter_list_with_source_url(self, mock_nl_slugs):
-        """A newsletter list can have a subscription source URL."""
+    def test_newsletter_list_with_extra_data(self, mock_nl_slugs, mock_langs):
+        """A newsletter list can have additional data."""
         data = {
             "newsletters": ["slug1", "slug2", "slug3", "other"],
             "source_url": "  https://example.com",
+            "lang": "es",
+            "format": "T",
         }
         prepared = to_vendor(data)
         assert prepared == {
+            "email": {"email_format": "T", "email_lang": "es"},
             "newsletters": [
-                {"name": "slug1", "subscribed": True, "source": "https://example.com"},
-                {"name": "slug2", "subscribed": True, "source": "https://example.com"},
-                {"name": "slug3", "subscribed": True, "source": "https://example.com"},
-            ]
+                {
+                    "name": "slug1",
+                    "subscribed": True,
+                    "format": "T",
+                    "lang": "es",
+                    "source": "https://example.com",
+                },
+                {
+                    "name": "slug2",
+                    "subscribed": True,
+                    "format": "T",
+                    "lang": "es",
+                    "source": "https://example.com",
+                },
+                {
+                    "name": "slug3",
+                    "subscribed": True,
+                    "format": "T",
+                    "lang": "es",
+                    "source": "https://example.com",
+                },
+            ],
+        }
+
+    @patch("basket.news.newsletters.newsletter_languages", return_value=["en", "fr"])
+    @patch("basket.news.backends.ctms.newsletter_slugs", return_value=["slug1"])
+    def test_newsletter_list_with_defaults(self, mock_nl_slugs, mock_langs):
+        """A newsletter list uses the default language and format"""
+        data = {"newsletters": ["slug1"]}
+        existing_data = {"lang": "fr", "format": "H"}
+        prepared = to_vendor(data, existing_data)
+        assert prepared == {
+            "newsletters": [
+                {"name": "slug1", "subscribed": True, "format": "H", "lang": "fr"},
+            ],
+        }
+
+    @patch("basket.news.newsletters.newsletter_languages", return_value=["en", "fr"])
+    @patch("basket.news.backends.ctms.newsletter_slugs", return_value=["slug1"])
+    def test_newsletter_list_with_defaults_override(self, mock_nl_slugs, mock_langs):
+        """A newsletter list uses the updated data rather than the defaults"""
+        data = {"lang": "en", "format": "T", "newsletters": ["slug1"]}
+        existing_data = {"lang": "fr", "format": "H"}
+        prepared = to_vendor(data, existing_data)
+        assert prepared == {
+            "email": {"email_format": "T", "email_lang": "en"},
+            "newsletters": [
+                {"name": "slug1", "subscribed": True, "format": "T", "lang": "en"},
+            ],
         }
 
     @patch(
@@ -335,6 +414,34 @@ def test_newsletter_map(self, mock_nl_slugs):
             ]
         }
 
+    @patch("basket.news.newsletters.newsletter_languages", return_value=["en", "es"])
+    @patch(
+        "basket.news.backends.ctms.newsletter_slugs",
+        return_value=["slug1", "slug2", "slug3", "slug4"],
+    )
+    def test_newsletter_map_with_extra_data(self, mock_nl_slugs, mock_langs):
+        """A newsletter map adds extra data to subscriptions"""
+        data = {
+            "newsletters": {
+                "slug1": True,
+                "slug2": False,
+                "slug3": True,
+                "other": True,
+            },
+            "source_url": "  https://example.com",
+            "lang": "es",
+            "format": "T",
+        }
+        prepared = to_vendor(data)
+        assert prepared == {
+            "email": {"email_format": "T", "email_lang": "es"},
+            "newsletters": [
+                {"name": "slug1", "subscribed": True, "format": "T", "lang": "es"},
+                {"name": "slug2", "subscribed": False},
+                {"name": "slug3", "subscribed": True, "format": "T", "lang": "es"},
+            ],
+        }
+
     @patch(
         "basket.news.backends.ctms.newsletter_slugs",
         return_value=["slug1", "slug2", "slug3", "slug4"],
@@ -972,6 +1079,31 @@ def test_update_email_id_not_in_existing_data(self):
         update_data = {"first_name": "Elizabeth", "email_id": "a-new-email-id"}
         assert ctms.update(user_data, update_data) is None
 
+    @patch("basket.news.newsletters.newsletter_languages", return_value=["en", "fr"])
+    @patch("basket.news.backends.ctms.newsletter_slugs", return_value=["slug1"])
+    def test_update_use_existing_lang_and_format(self, mock_slugs, mock_langs):
+        """CTMS.update uses the existing language and format"""
+        updated = {"updated": "fake_response"}
+        interface = mock_interface("PATCH", 200, updated)
+        ctms = CTMS(interface)
+
+        user_data = {
+            "token": "an-existing-user",
+            "email_id": "an-existing-id",
+            "lang": "fr",
+            "format": "T",
+        }
+        update_data = {"newsletters": ["slug1"]}
+        assert ctms.update(user_data, update_data) == updated
+        interface.session.patch.assert_called_once_with(
+            "/ctms/an-existing-id",
+            json={
+                "newsletters": [
+                    {"name": "slug1", "subscribed": True, "lang": "fr", "format": "T"}
+                ]
+            },
+        )
+
     def test_update_by_token(self):
         """CTMS can update a record by basket token."""
         record = {"email": {"email_id": "the-email-id", "token": "the-token"}}