Repository: freedomofpress/securedrop

Issue:
Fail gracefully if language no longer supported
In the course of development, a language may fall below an acceptable level of translation coverage, in which case we may want to remove it from the list of supported languages.

Currently, the app will fail (Source Interface will display an Internal Server Error) if an unsupported language is specified in `SUPPORTED_LOCALES`. This means that removing support for a language could break running instances.

Safer behavior could be to ignore such locales (and fall back to English when specified as the default locale). An error could be displayed in `securedrop-admin` to the user next time they step through `sdconfig`, and an OSSEC email generated if an unsupported locale is detected in the configuration.


Discussion:

Resolving patch:
diff --git a/securedrop/i18n.py b/securedrop/i18n.py
--- a/securedrop/i18n.py
+++ b/securedrop/i18n.py
@@ -17,7 +17,7 @@
 #
 import collections
 
-from typing import Dict, List
+from typing import List, Set
 
 from babel.core import (
     Locale,
@@ -29,7 +29,7 @@
 from flask import Flask, g, request, session
 from flask_babel import Babel
 
-from sdconfig import SDConfig
+from sdconfig import SDConfig, FALLBACK_LOCALE
 
 
 class RequestLocaleInfo:
@@ -126,32 +126,46 @@ def configure_babel(config: SDConfig, app: Flask) -> Babel:
     return babel
 
 
+def parse_locale_set(codes: List[str]) -> Set[Locale]:
+    return {Locale.parse(code) for code in codes}
+
+
 def validate_locale_configuration(config: SDConfig, babel: Babel) -> None:
     """
-    Ensure that the configured locales are valid and translated.
+    Check that configured locales are available in the filesystem and therefore usable by
+    Babel.  Warn about configured locales that are not usable, unless we're left with
+    no usable default or fallback locale, in which case raise an exception.
     """
-    if config.DEFAULT_LOCALE not in config.SUPPORTED_LOCALES:
-        raise ValueError(
-            'The default locale "{}" is not included in the set of supported locales "{}"'.format(
-                config.DEFAULT_LOCALE, config.SUPPORTED_LOCALES
-            )
+    # These locales are available and loadable from the filesystem.
+    available = set(babel.list_translations())
+    available.add(Locale.parse(FALLBACK_LOCALE))
+
+    # These locales were configured via "securedrop-admin sdconfig", meaning
+    # they were present on the Admin Workstation at "securedrop-admin" runtime.
+    configured = parse_locale_set(config.SUPPORTED_LOCALES)
+
+    # The intersection of these sets is the set of locales usable by Babel.
+    usable = available & configured
+
+    missing = configured - usable
+    if missing:
+        babel.app.logger.error(
+            f'Configured locales {missing} are not in the set of usable locales {usable}'
         )
 
-    translations = babel.list_translations()
-    for locale in config.SUPPORTED_LOCALES:
-        if locale == "en_US":
-            continue
+    defaults = parse_locale_set([config.DEFAULT_LOCALE, FALLBACK_LOCALE])
+    if not defaults & usable:
+        raise ValueError(
+            f'None of the default locales {defaults} are in the set of usable locales {usable}'
+        )
 
-        parsed = Locale.parse(locale)
-        if parsed not in translations:
-            raise ValueError(
-                'Configured locale "{}" is not in the set of translated locales "{}"'.format(
-                    parsed, translations
-                )
-            )
+    global USABLE_LOCALES
+    USABLE_LOCALES = usable
 
 
+# TODO(#6420): avoid relying on and manipulating on this global state
 LOCALES = collections.OrderedDict()  # type: collections.OrderedDict[str, RequestLocaleInfo]
+USABLE_LOCALES = set()  # type: Set[Locale]
 
 
 def map_locale_display_names(config: SDConfig) -> None:
@@ -163,16 +177,19 @@ def map_locale_display_names(config: SDConfig) -> None:
     to distinguish them. For languages with more than one translation,
     like Chinese, we do need the additional detail.
     """
-    language_locale_counts = collections.defaultdict(int)  # type: Dict[str, int]
-    for l in sorted(config.SUPPORTED_LOCALES):
-        locale = RequestLocaleInfo(l)
-        language_locale_counts[locale.language] += 1
-
+    seen: Set[str] = set()
     locale_map = collections.OrderedDict()
     for l in sorted(config.SUPPORTED_LOCALES):
+        if Locale.parse(l) not in USABLE_LOCALES:
+            continue
+
         locale = RequestLocaleInfo(l)
-        if language_locale_counts[locale.language] > 1:
+        if locale.language in seen:
+            # Disambiguate translations for this language.
             locale.use_display_name = True
+        else:
+            seen.add(locale.language)
+
         locale_map[str(locale)] = locale
 
     global LOCALES
@@ -193,23 +210,24 @@ def get_locale(config: SDConfig) -> str:
     - l request argument or session['locale']
     - browser suggested locale, from the Accept-Languages header
     - config.DEFAULT_LOCALE
+    - config.FALLBACK_LOCALE
     """
-    # Default to any locale set in the session.
-    locale = session.get("locale")
-
-    # A valid locale specified in request.args takes precedence.
+    preferences = []
+    if session.get("locale"):
+        preferences.append(session.get("locale"))
     if request.args.get("l"):
-        negotiated = negotiate_locale([request.args["l"]], LOCALES.keys())
-        if negotiated:
-            locale = negotiated
+        preferences.insert(0, request.args.get("l"))
+    if not preferences:
+        preferences.extend(get_accepted_languages())
+    preferences.append(config.DEFAULT_LOCALE)
+    preferences.append(FALLBACK_LOCALE)
+
+    negotiated = negotiate_locale(preferences, LOCALES.keys())
 
-    # If the locale is not in the session or request.args, negotiate
-    # the best supported option from the browser's accepted languages.
-    if not locale:
-        locale = negotiate_locale(get_accepted_languages(), LOCALES.keys())
+    if not negotiated:
+        raise ValueError("No usable locale")
 
-    # Finally, fall back to the default locale if necessary.
-    return locale or config.DEFAULT_LOCALE
+    return negotiated
 
 
 def get_accepted_languages() -> List[str]:
diff --git a/securedrop/sdconfig.py b/securedrop/sdconfig.py
--- a/securedrop/sdconfig.py
+++ b/securedrop/sdconfig.py
@@ -8,6 +8,9 @@
 from typing import Set
 
 
+FALLBACK_LOCALE = "en_US"
+
+
 class SDConfig:
     def __init__(self) -> None:
         try:
@@ -120,7 +123,7 @@ def __init__(self) -> None:
         # Config entries used by i18n.py
         # Use en_US as the default locale if the key is not defined in _config
         self.DEFAULT_LOCALE = getattr(
-            _config, "DEFAULT_LOCALE", "en_US"
+            _config, "DEFAULT_LOCALE", FALLBACK_LOCALE,
         )  # type: str
         supported_locales = set(getattr(
             _config, "SUPPORTED_LOCALES", [self.DEFAULT_LOCALE]


Test patch:
diff --git a/securedrop/tests/test_i18n.py b/securedrop/tests/test_i18n.py
--- a/securedrop/tests/test_i18n.py
+++ b/securedrop/tests/test_i18n.py
@@ -32,13 +32,17 @@
 from flask import request
 from flask import session
 from flask_babel import gettext
-from sdconfig import SDConfig
+from i18n import parse_locale_set
+from sdconfig import FALLBACK_LOCALE, SDConfig
 from sh import pybabel
 from sh import sed
 from .utils.env import TESTS_DIR
 from werkzeug.datastructures import Headers
 
 
+NEVER_LOCALE = 'eo'  # Esperanto
+
+
 def verify_i18n(app):
     not_translated = 'code hello i18n'
     translated_fr = 'code bonjour'
@@ -225,32 +229,77 @@ def test_i18n(journalist_app, config):
         verify_i18n(app)
 
 
-def test_supported_locales(config):
-    fake_config = SDConfig()
+def test_parse_locale_set():
+    assert parse_locale_set([FALLBACK_LOCALE]) == set([Locale.parse(FALLBACK_LOCALE)])
+
 
-    # Check that an invalid locale raises an error during app
-    # configuration.
-    fake_config.SUPPORTED_LOCALES = ['en_US', 'yy_ZZ']
+def test_no_usable_fallback_locale(journalist_app, config):
+    """
+    The apps fail if neither the default nor the fallback locale is usable.
+    """
+    fake_config = SDConfig()
+    fake_config.DEFAULT_LOCALE = NEVER_LOCALE
+    fake_config.SUPPORTED_LOCALES = [NEVER_LOCALE]
     fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
 
-    with pytest.raises(UnknownLocaleError):
+    i18n.USABLE_LOCALES = set()
+
+    with pytest.raises(ValueError, match='in the set of usable locales'):
         journalist_app_module.create_app(fake_config)
 
-    with pytest.raises(UnknownLocaleError):
+    with pytest.raises(ValueError, match='in the set of usable locales'):
         source_app.create_app(fake_config)
 
-    # Check that a valid but unsupported locale raises an error during
-    # app configuration.
-    fake_config.SUPPORTED_LOCALES = ['en_US', 'wae_CH']
+
+def test_unusable_default_but_usable_fallback_locale(config, caplog):
+    """
+    The apps start even if the default locale is unusable, as along as the fallback locale is
+    usable, but log an error for OSSEC to pick up.
+    """
+    fake_config = SDConfig()
+    fake_config.DEFAULT_LOCALE = NEVER_LOCALE
+    fake_config.SUPPORTED_LOCALES = [NEVER_LOCALE, FALLBACK_LOCALE]
     fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
 
-    with pytest.raises(ValueError, match="not in the set of translated locales"):
+    for app in (journalist_app_module.create_app(fake_config),
+                source_app.create_app(fake_config)):
+        with app.app_context():
+            assert NEVER_LOCALE in caplog.text
+            assert 'not in the set of usable locales' in caplog.text
+
+
+def test_invalid_locales(config):
+    """
+    An invalid locale raises an error during app configuration.
+    """
+    fake_config = SDConfig()
+    fake_config.SUPPORTED_LOCALES = [FALLBACK_LOCALE, 'yy_ZZ']
+    fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
+
+    with pytest.raises(UnknownLocaleError):
         journalist_app_module.create_app(fake_config)
 
-    with pytest.raises(ValueError, match="not in the set of translated locales"):
+    with pytest.raises(UnknownLocaleError):
         source_app.create_app(fake_config)
 
 
+def test_valid_but_unusable_locales(config, caplog):
+    """
+    The apps start with one or more unusable, but still valid, locales, but log an error for
+    OSSEC to pick up.
+    """
+    fake_config = SDConfig()
+
+    fake_config.SUPPORTED_LOCALES = [FALLBACK_LOCALE, 'wae_CH']
+    fake_config.TRANSLATION_DIRS = Path(config.TEMP_DIR)
+
+    for app in (journalist_app_module.create_app(fake_config),
+                source_app.create_app(fake_config)):
+        with app.app_context():
+            assert 'wae' in caplog.text
+            assert 'not in the set of usable locales' in caplog.text
+
+
 def test_language_tags():
     assert i18n.RequestLocaleInfo(Locale.parse('en')).language_tag == 'en'
     assert i18n.RequestLocaleInfo(Locale.parse('en-US', sep="-")).language_tag == 'en-US'