Repository: mozmeao/basket

Issue:
ctms.to_vendor() could not convert unknown data
After the `is_waitlist` changes, we're seeing some sentry errors for unknown data for "fpn_country". 

https://mozilla.sentry.io/issues/4087704376/?project=6260602&query=is%3Aunresolved&referrer=issue-stream&stream_index=0


Discussion:
The following data is posted:
```
email: [Filtered]
format: H
fpn_country: br
lang: en
newsletters: guardian-vpn-waitlist
source_url: https://www.mozilla.org/en-US/products/vpn/invite/
```

When the slug `guardian-vpn` is sent, we should consume `fpn_*` fields. 

Same as `relay-vpn-bundle` and `relay-phone-masking` slugs, which should consume `relay_*` fields.

The piece of code responsible of "converting" data is this one:
https://github.com/mozmeao/basket/blob/cdab6d102bf5d3d87c8b2efbf186b21ada60de77/basket/news/backends/ctms.py#L236-L248

This is a bug, and I don't find appropriate tests in `test_ctms.py`. We can fix this. Please assign the issue to me.

Resolving patch:
diff --git a/basket/news/backends/ctms.py b/basket/news/backends/ctms.py
--- a/basket/news/backends/ctms.py
+++ b/basket/news/backends/ctms.py
@@ -123,7 +123,7 @@ def from_vendor(contact):
                         data["fpn_platform"] = platform
                 else:
                     wl_name += "-waitlist"
-                if wl_name == "relay-waitlist":
+                if wl_name.startswith("relay") and wl_name.endswith("-waitlist"):
                     data["relay_country"] = waitlist["fields"].get("geo")
                 data.setdefault("newsletters", []).append(wl_name)
         else:
@@ -230,16 +230,18 @@ def waitlist_fields_for_slug(data, slug):
           "currency": "eur",
         }
     """
-    prefix = re.sub("[^0-9a-zA-Z]+", "_", slug) + "_"
-    consumed_keys = []
-
     # Specific cases for legacy waitlists:
+    if slug in ("relay-vpn-bundle", "relay-phone-masking"):
+        slug = "relay"
     new_fields = {
         "relay_country": "relay_geo",
         "fpn_country": "vpn_geo",
         "fpn_platform": "vpn_platform",
     }
 
+    prefix = re.sub("[^0-9a-zA-Z]+", "_", slug) + "_"
+    consumed_keys = []
+
     # Turn flat fields into a dict.
     fields = {}
     for name, raw_value in data.items():
@@ -362,7 +364,11 @@ def to_vendor(data, existing_data=None):
                         )
                         # Remove all consumed waitlist fields from unknown data
                         for field_name in consumed_fields:
-                            del unknown_data[field_name]
+                            try:
+                                del unknown_data[field_name]
+                            except KeyError:
+                                # The same field was consumed by several waitlists.
+                                pass
                         # Submit the waitlist details, with potential source URL.
                         wl_sub = {
                             "name": slug,


Test patch:
diff --git a/basket/news/tests/test_ctms.py b/basket/news/tests/test_ctms.py
--- a/basket/news/tests/test_ctms.py
+++ b/basket/news/tests/test_ctms.py
@@ -94,6 +94,12 @@
         "geo": "fr"
       }
     },
+    {
+      "name": "relay-vpn-bundle",
+      "fields": {
+        "geo": "fr"
+      }
+    },
     {
       "name": "super-product",
       "source": "some website",
@@ -138,6 +144,7 @@
         "mozilla-welcome",
         "guardian-vpn-waitlist",
         "relay-waitlist",
+        "relay-vpn-bundle-waitlist",
         "super-product-waitlist",
     ],
     "optin": True,
@@ -664,6 +671,85 @@ def test_arbitrary_data_is_turned_into_waitlist(self, mock_wl_slugs, mock_nl_slu
             ],
         }
 
+    @patch(
+        "basket.news.backends.ctms.newsletter_slugs",
+        return_value=[
+            "guardian-vpn-waitlist",
+        ],
+    )
+    @patch(
+        "basket.news.backends.ctms.newsletter_waitlist_slugs",
+        return_value=[
+            "guardian-vpn-waitlist",
+        ],
+    )
+    def test_guardian_vpn_picks_appropriate_prefixed_fields(
+        self, mock_wl_slugs, mock_nl_slugs
+    ):
+        """Subscribe with arbitrary fields and unsubscribe from another one."""
+        data = {
+            "newsletters": {
+                "guardian-vpn-waitlist": True,
+            },
+            "fpn_country": "br",
+            "source_url": "https://www.mozilla.org/en-US/products/vpn/invite/",
+        }
+        prepared = to_vendor(data)
+        assert prepared == {
+            "waitlists": [
+                {
+                    "name": "vpn",
+                    "subscribed": True,
+                    "source": "https://www.mozilla.org/en-US/products/vpn/invite/",
+                    "fields": {"geo": "br"},
+                },
+            ],
+        }
+
+    @patch(
+        "basket.news.backends.ctms.newsletter_slugs",
+        return_value=[
+            "relay-phone-masking-waitlist",
+            "relay-vpn-bundle-waitlist",
+        ],
+    )
+    @patch(
+        "basket.news.backends.ctms.newsletter_waitlist_slugs",
+        return_value=[
+            "relay-phone-masking-waitlist",
+            "relay-vpn-bundle-waitlist",
+        ],
+    )
+    def test_relay_waitlists_pick_appropriate_prefixed_fields(
+        self, mock_wl_slugs, mock_nl_slugs
+    ):
+        """Subscribe with arbitrary fields and unsubscribe from another one."""
+        data = {
+            "newsletters": {
+                "relay-phone-masking-waitlist": True,
+                "relay-vpn-bundle-waitlist": True,
+            },
+            "relay_country": "it",
+            "source_url": "https://relay.firefox.com/phone/waitlist/",
+        }
+        prepared = to_vendor(data)
+        assert prepared == {
+            "waitlists": [
+                {
+                    "name": "relay-phone-masking",
+                    "subscribed": True,
+                    "source": "https://relay.firefox.com/phone/waitlist/",
+                    "fields": {"geo": "it"},
+                },
+                {
+                    "name": "relay-vpn-bundle",
+                    "subscribed": True,
+                    "source": "https://relay.firefox.com/phone/waitlist/",
+                    "fields": {"geo": "it"},
+                },
+            ],
+        }
+
     @patch(
         "basket.news.backends.ctms.newsletter_slugs",
         return_value=["guardian-vpn-waitlist"],