Repository: translate/pootle

Issue:
Try simpler language code as fallback before settings.LANGUAGE_CODE
In https://github.com/translate/pootle/blob/10913224/pootle/i18n/override.py#L87-L101 if the language code `it-IT` (for example) is tried and eventually falls back to `settings.LANGUAGE_CODE`, but it makes sense to first try `it` (simpler version of `it-IT`) before falling back to `settings.LANGUAGE_CODE`.


Discussion:
my only concern is whether this corresponds to the language codes for which there are select2 translations, as its this which borks the site if it cant get a correct match
Select2 safely defaults to English if you provide some unexistent language code in Select2 l10n.
not sure @unho - this was what was borking when the server lang was not recognised
That shouldn't be the case because Pootle does not serve the Select2 l10n file if the language is not supported by Select2 l10n: https://github.com/translate/pootle/commit/5c2d72773122820d67c9436ffa2cc6fede1ecc08#diff-db6ee1857cb4184ef57b57f238915bfbR49
it might be just that other js was borking causing the select2 fail.

im thinking that if all else fails - ie we dont recognise the server lang or the server lang root we should fall back to `en`
I agree. `settings.LANGUAGE_CODE` must be checked too to see if it is supported https://github.com/translate/pootle/blob/10913224/pootle/i18n/override.py#L35-L38 and then fall back to `en`.
A small correction, the fallback should be `en-us` to align with default for `settings.LANGUAGE_CODE` https://docs.djangoproject.com/ko/1.11/ref/settings/#language-code

Resolving patch:
diff --git a/pootle/i18n/override.py b/pootle/i18n/override.py
--- a/pootle/i18n/override.py
+++ b/pootle/i18n/override.py
@@ -38,24 +38,35 @@ def supported_langs():
     return settings.LANGUAGES
 
 
-def get_lang_from_session(request, supported):
-    if hasattr(request, 'session'):
-        lang_code = request.session.get(LANGUAGE_SESSION_KEY, None)
-        if lang_code and lang_code in supported:
-            return lang_code
+def get_language_supported(lang_code, supported):
+    normalized = data.normalize_code(data.simplify_to_common(lang_code))
+    if normalized in supported:
+        return normalized
+
+    # FIXME: horribly slow way of dealing with languages with @ in them
+    for lang in supported.keys():
+        if normalized == data.normalize_code(lang):
+            return lang
 
     return None
 
 
+def get_lang_from_session(request, supported):
+    if not hasattr(request, 'session'):
+        return None
+    lang_code = request.session.get(LANGUAGE_SESSION_KEY, None)
+    if not lang_code:
+        return None
+    return get_language_supported(lang_code, supported)
+
+
 def get_lang_from_cookie(request, supported):
     """See if the user's browser sent a cookie with a preferred language."""
     from django.conf import settings
     lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME)
-
-    if lang_code and lang_code in supported:
-        return lang_code
-
-    return None
+    if not lang_code:
+        return None
+    return get_language_supported(lang_code, supported)
 
 
 def get_lang_from_http_header(request, supported):
@@ -70,17 +81,9 @@ def get_lang_from_http_header(request, supported):
     for accept_lang, __ in trans_real.parse_accept_lang_header(accept):
         if accept_lang == '*':
             return None
-
-        normalized = data.normalize_code(data.simplify_to_common(accept_lang))
-        if normalized in ['en-us', 'en']:
-            return None
-        if normalized in supported:
-            return normalized
-
-        # FIXME: horribly slow way of dealing with languages with @ in them
-        for lang in supported.keys():
-            if normalized == data.normalize_code(lang):
-                return lang
+        supported_lang = get_language_supported(accept_lang, supported)
+        if supported_lang:
+            return supported_lang
     return None
 
 
@@ -98,7 +101,9 @@ def get_language_from_request(request, check_path=False):
         if lang is not None:
             return lang
     from django.conf import settings
-    return settings.LANGUAGE_CODE
+    if settings.LANGUAGE_CODE in supported:
+        return settings.LANGUAGE_CODE
+    return 'en-us'
 
 
 def get_language_bidi():


Test patch:
diff --git a/tests/i18n/override.py b/tests/i18n/override.py
new file mode 100644
--- /dev/null
+++ b/tests/i18n/override.py
@@ -0,0 +1,196 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) Pootle contributors.
+#
+# This file is a part of the Pootle project. It is distributed under the GPL3
+# or later license. See the LICENSE file for a copy of the license and the
+# AUTHORS file for copyright and authorship information.
+
+import pytest
+
+from django.conf import settings
+from django.utils.translation import LANGUAGE_SESSION_KEY
+
+from pootle.i18n.override import (get_lang_from_cookie,
+                                  get_lang_from_http_header,
+                                  get_language_from_request,
+                                  get_lang_from_session)
+
+
+SUPPORTED_LANGUAGES = {
+    'es-ar': 'es-ar',
+    'fr': 'fr',
+    'gl': 'gl',
+}
+
+
+@pytest.mark.django_db
+def test_get_lang_from_session(rf, client):
+    # Test no session.
+    request = rf.get("")
+    assert not hasattr(request, 'session')  # Check no session before test.
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) is None
+
+    # Test session with no language.
+    response = client.get("")
+    request = response.wsgi_request
+    assert LANGUAGE_SESSION_KEY not in request.session
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) is None
+
+    # Test session with supported language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'gl'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) == 'gl'
+
+    # Test session with supported language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'es-AR'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) == 'es-ar'
+
+    # Test cookie with longer underscore language code for a supported
+    # language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'gl_ES'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) == 'gl'
+
+    # Test cookie with longer hyphen language code for a supported language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'fr-FR'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) == 'fr'
+
+    # Test header with shorter language code for a supported language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'es'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with unsupported language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'FAIL'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with unsupported longer underscore language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'the_FAIL'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with unsupported longer hyphen language.
+    response = client.get("")
+    request = response.wsgi_request
+    request.session[LANGUAGE_SESSION_KEY] = 'the-FAIL'
+    assert get_lang_from_session(request, SUPPORTED_LANGUAGES) is None
+
+
+def test_get_lang_from_cookie(rf):
+    request = rf.get("")
+
+    # Test no cookie.
+    assert settings.LANGUAGE_COOKIE_NAME not in request.COOKIES
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) is None
+
+    # Test cookie with supported language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'gl'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) == 'gl'
+
+    # Test cookie with longer supported language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'es-AR'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) == 'es-ar'
+
+    # Test cookie with longer underscore language code for a supported
+    # language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'gl_ES'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) == 'gl'
+
+    # Test cookie with longer hyphen language code for a supported language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'fr-FR'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) == 'fr'
+
+    # Test cookie with shorter language code for a supported language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'es'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) is None
+
+    # Test cookie with unsupported language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'FAIL'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) is None
+
+    # Test cookie with unsupported longer underscore language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'the_FAIL'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) is None
+
+    # Test cookie with unsupported longer hyphen language.
+    request.COOKIES[settings.LANGUAGE_COOKIE_NAME] = 'the-FAIL'
+    assert get_lang_from_cookie(request, SUPPORTED_LANGUAGES) is None
+
+
+def test_get_lang_from_http_header(rf):
+    # Test no header.
+    request = rf.get("")
+    assert 'HTTP_ACCEPT_LANGUAGE' not in request.META
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+    # Test empty header.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with wildcard.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='*')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with supported language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='gl')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) == 'gl'
+
+    # Test cookie with longer supported language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='es-AR')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) == 'es-ar'
+
+    # Test header with longer underscore language code for a supported
+    # language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='gl_ES')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with longer hyphen language code for a supported language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='fr-FR')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) == 'fr'
+
+    # Test header with shorter language code for a supported language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='es')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with unsupported language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='FAIL')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with unsupported longer underscore language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='the_FAIL')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+    # Test header with unsupported longer hyphen language.
+    request = rf.get("", HTTP_ACCEPT_LANGUAGE='the-FAIL')
+    assert get_lang_from_http_header(request, SUPPORTED_LANGUAGES) is None
+
+
+def test_get_language_from_request(rf):
+    request = rf.get("")
+
+    # Ensure the response doesn't come from any of the `lang_getter` functions.
+    assert not hasattr(request, 'session')
+    assert settings.LANGUAGE_COOKIE_NAME not in request.COOKIES
+    assert 'HTTP_ACCEPT_LANGUAGE' not in request.META
+
+    # Test default server language fallback.
+    SUPPORTED_LANGUAGES[settings.LANGUAGE_CODE] = settings.LANGUAGE_CODE
+    assert settings.LANGUAGE_CODE in SUPPORTED_LANGUAGES
+    assert (get_language_from_request(request, SUPPORTED_LANGUAGES) ==
+            settings.LANGUAGE_CODE)
+
+    # Test ultimate fallback.
+    SUPPORTED_LANGUAGES.pop(settings.LANGUAGE_CODE)
+    assert settings.LANGUAGE_CODE not in SUPPORTED_LANGUAGES
+    assert get_language_from_request(request, SUPPORTED_LANGUAGES) == 'en-us'