Repository: mdn/kuma

Issue:
[regression] Multi-locale search is not searching in all locales
**Summary**
http://localhost.org:8000/api/v1/search?q=video&locale=fr&locale=de
actually searches on `locale == ['de']` and 
http://localhost.org:8000/api/v1/search?q=video&locale=de&locale=fr actually searches on `locale== ['fr']`.

Originally from here:
https://github.com/mdn/yari/pull/1473#pullrequestreview-584750752




Discussion:

Resolving patch:
diff --git a/kuma/api/v1/search/forms.py b/kuma/api/v1/search/forms.py
--- a/kuma/api/v1/search/forms.py
+++ b/kuma/api/v1/search/forms.py
@@ -3,15 +3,26 @@
 from django.utils.datastructures import MultiValueDict
 
 
+class MultipleChoiceFieldICase(forms.MultipleChoiceField):
+    """Just like forms.MultipleChoiceField but everything's case insentive.
+
+    For simplicity, this field assumes that each choice is a tuple where
+    the first element is always a string.
+    """
+
+    def valid_value(self, value):
+        return str(value).lower() in [x[0].lower() for x in self.choices]
+
+
 class SearchForm(forms.Form):
     q = forms.CharField(max_length=settings.ES_Q_MAXLENGTH)
-    locale = forms.MultipleChoiceField(
+    locale = MultipleChoiceFieldICase(
         required=False,
         # The `settings.LANGUAGES` looks like this:
         #   [('en-US', 'English (US)'), ...]
         # But all locales are stored in lowercase in Elasticsearch, so
         # force everything to lowercase.
-        choices=[(code.lower(), name) for code, name in settings.LANGUAGES],
+        choices=[(code, name) for code, name in settings.LANGUAGES],
     )
 
     SORT_CHOICES = ("best", "relevance", "popularity")
@@ -34,18 +45,6 @@ def __init__(self, data, **kwargs):
         # See https://www.peterbe.com/plog/initial-values-bound-django-form-rendered
         data = MultiValueDict({**{k: [v] for k, v in initial.items()}, **data})
 
-        # Because the `?locale=en-US&locale=Fr` might come in from the `request.GET`
-        # we can't edit it there. So instead, we mutate it here in the `data`
-        if "locale" in data:
-            # Always force it to lowercase, because that's what the ChoiceField
-            # is configured to. And the searches should always be in lower case.
-            # Remember, Django forms will allow this to be a single string
-            # (e.g. `?locale=Fr`) or a multi-value (`?locale=fr&locale=En-US`).
-            if isinstance(data["locale"], str):
-                data["locale"] = data["locale"].lower()
-            else:
-                data["locale"] = [x.lower() for x in data["locale"]]
-
         # If, for keys we have an initial value for, it was passed an empty string,
         # then swap it for the initial value.
         # For example `?q=searching&page=` you probably meant to omit it


Test patch:
diff --git a/kuma/api/v1/tests/test_forms.py b/kuma/api/v1/tests/test_forms.py
new file mode 100644
--- /dev/null
+++ b/kuma/api/v1/tests/test_forms.py
@@ -0,0 +1,58 @@
+from django.test import RequestFactory
+
+from kuma.api.v1.search.forms import SearchForm
+
+
+def test_search_form_locale_happy_path():
+    """The way the form handles 'locale' is a bit overly complicated.
+    These unit tests focuses exclusively on that and when the form is valid."""
+
+    initial = {"page": 1, "size": 10}
+    request = RequestFactory().get("/api/v1/search?q=foo")
+    form = SearchForm(request.GET, initial=initial)
+    assert form.is_valid()
+    assert form.cleaned_data["locale"] == []
+
+    request = RequestFactory().get("/api/v1/search?q=foo")
+    initial["locale"] = "sv-Se"
+    form = SearchForm(request.GET, initial=initial)
+    assert form.is_valid()
+    assert form.cleaned_data["locale"] == ["sv-Se"]
+
+    request = RequestFactory().get("/api/v1/search?q=foo&locale=Fr")
+    form = SearchForm(request.GET, initial=initial)
+    assert form.is_valid()
+    assert form.cleaned_data["locale"] == ["Fr"]
+
+    request = RequestFactory().get("/api/v1/search?q=foo&locale=Fr&locale=de")
+    form = SearchForm(request.GET, initial=initial)
+    assert form.is_valid()
+    print(form.cleaned_data)
+    assert form.cleaned_data["locale"] == ["Fr", "de"]
+
+    # Note, same as the initial default
+    request = RequestFactory().get("/api/v1/search?q=foo&locale=SV-se")
+    form = SearchForm(request.GET, initial=initial)
+    assert form.is_valid()
+    assert form.cleaned_data["locale"] == ["SV-se"]
+
+    request = RequestFactory().get("/api/v1/search?q=foo&locale=SV-se&locale=fr")
+    form = SearchForm(request.GET, initial=initial)
+    assert form.is_valid()
+    assert form.cleaned_data["locale"] == ["SV-se", "fr"]
+
+
+def test_search_form_locale_validation_error():
+    """The way the form handles 'locale' is a bit overly complicated.
+    These unit tests focuses exclusively on that and when the form is NOT valid."""
+
+    initial = {"page": 1, "size": 10}
+    request = RequestFactory().get("/api/v1/search?q=foo&locale=xxx")
+    form = SearchForm(request.GET, initial=initial)
+    assert not form.is_valid()
+    assert form.errors["locale"]
+
+    request = RequestFactory().get("/api/v1/search?q=foo&locale=")
+    form = SearchForm(request.GET, initial=initial)
+    assert not form.is_valid()
+    assert form.errors["locale"]
diff --git a/kuma/api/v1/tests/test_search.py b/kuma/api/v1/tests/test_search.py
--- a/kuma/api/v1/tests/test_search.py
+++ b/kuma/api/v1/tests/test_search.py
@@ -104,24 +104,3 @@ def test_search_basic_match(user_client, settings, mock_elasticsearch):
             "summary": "Foo summary",
         }
     ]
-
-
-def test_search_case_insensitive_locale(user_client, settings, mock_elasticsearch):
-    mock_elasticsearch.index(
-        settings.SEARCH_INDEX_NAME,
-        {
-            "id": "/en-us/docs/Foo",
-            "title": "Foo Title",
-            "summary": "Foo summary",
-            "locale": "en-us",
-            "archived": False,
-            "slug": "Foo",
-            "popularity": 0,
-        },
-        id="/en-us/docs/Foo",
-    )
-    url = reverse("api.v1.search")
-    response = user_client.get(url, {"q": "x", "locale": "EN-US"})
-    assert response.status_code == 200
-    response = user_client.get(url, {"q": "x", "locale": ["EN-US", "Fr"]})
-    assert response.status_code == 200