Repository: python-babel/babel
Pull request #1085 (2024-05-13 15:44:09)

Make pgettext search plurals when translation is not found
Given this po file:

```po
msgctxt "ctx"
msgid "foo"
msgid_plural "foos"
msgstr[0] "foo translated"
```

`pgettext` fails to find the singular translation i.e. `pgettext("ctx", "foo") == "foo"` when one would expect to get `foo translated`.

The same issue has already been fixed upstream in `gettext`: 
https://github.com/python/cpython/issues/62519
https://github.com/python/cpython/pull/107118

This PR applies the same patch to babel.

codecov[bot]: ## [Codecov](https://app.codecov.io/gh/python-babel/babel/pull/1085?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel) Report
Attention: Patch coverage is `83.33333%` with `1 lines` in your changes are missing coverage. Please review.
> Project coverage is 90.05%. Comparing base [(`e0d1018`)](https://app.codecov.io/gh/python-babel/babel/commit/e0d10183635b9ae1d37c31811e23c8974a1bc31e?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel) to head [(`9e31d84`)](https://app.codecov.io/gh/python-babel/babel/pull/1085?dropdown=coverage&src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel).
> Report is 7 commits behind head on master.

| [Files](https://app.codecov.io/gh/python-babel/babel/pull/1085?dropdown=coverage&src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel) | Patch % | Lines |
|---|---|---|
| [babel/support.py](https://app.codecov.io/gh/python-babel/babel/pull/1085?src=pr&el=tree&filepath=babel%2Fsupport.py&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel#diff-YmFiZWwvc3VwcG9ydC5weQ==) | 83.33% | [1 Missing :warning: ](https://app.codecov.io/gh/python-babel/babel/pull/1085?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel) |

<details><summary>Additional details and impacted files</summary>


```diff
@@            Coverage Diff             @@
##           master    #1085      +/-   ##
==========================================
- Coverage   90.99%   90.05%   -0.95%     
==========================================
  Files          26       26              
  Lines        4444     4453       +9     
==========================================
- Hits         4044     4010      -34     
- Misses        400      443      +43     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/python-babel/babel/pull/1085?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=python-babel).


Patch:
diff --git a/babel/localedata.py b/babel/localedata.py
index a9f7d4b..2aabfd1 100644
--- a/babel/localedata.py
+++ b/babel/localedata.py
@@ -95,6 +95,27 @@ def locale_identifiers() -> list[str]:
     ]
 
 
+def _is_non_likely_script(name: str) -> bool:
+    """Return whether the locale is of the form ``lang_Script``,
+    and the script is not the likely script for the language.
+
+    This implements the behavior of the ``nonlikelyScript`` value of the
+    ``localRules`` attribute for parent locales added in CLDR 45.
+    """
+    from babel.core import get_global, parse_locale
+
+    try:
+        lang, territory, script, variant, *rest = parse_locale(name)
+    except ValueError:
+        return False
+
+    if lang and script and not territory and not variant and not rest:
+        likely_subtag = get_global('likely_subtags').get(lang)
+        _, _, likely_script, *_ = parse_locale(likely_subtag)
+        return script != likely_script
+    return False
+
+
 def load(name: os.PathLike[str] | str, merge_inherited: bool = True) -> dict[str, Any]:
     """Load the locale data for the given locale.
 
@@ -132,8 +153,11 @@ def load(name: os.PathLike[str] | str, merge_inherited: bool = True) -> dict[str
                 from babel.core import get_global
                 parent = get_global('parent_exceptions').get(name)
                 if not parent:
-                    parts = name.split('_')
-                    parent = "root" if len(parts) == 1 else "_".join(parts[:-1])
+                    if _is_non_likely_script(name):
+                        parent = 'root'
+                    else:
+                        parts = name.split('_')
+                        parent = "root" if len(parts) == 1 else "_".join(parts[:-1])
                 data = load(parent).copy()
             filename = resolve_locale_filename(name)
             with open(filename, 'rb') as fileobj:
diff --git a/babel/support.py b/babel/support.py
index e6ee378..7dcda5c 100644
--- a/babel/support.py
+++ b/babel/support.py
@@ -466,10 +466,12 @@ class NullTranslations(gettext.NullTranslations):
         missing = object()
         tmsg = self._catalog.get(ctxt_msg_id, missing)
         if tmsg is missing:
-            if self._fallback:
-                return self._fallback.pgettext(context, message)
-            return message
-        return tmsg
+            tmsg = self._catalog.get((ctxt_msg_id, self.plural(1)), missing)
+        if tmsg is not missing:
+            return tmsg
+        if self._fallback:
+            return self._fallback.pgettext(context, message)
+        return message
 
     def lpgettext(self, context: str, message: str) -> str | bytes | object:
         """Equivalent to ``pgettext()``, but the translation is returned in the
diff --git a/scripts/download_import_cldr.py b/scripts/download_import_cldr.py
index 10a2dea..4a9805d 100755
--- a/scripts/download_import_cldr.py
+++ b/scripts/download_import_cldr.py
@@ -9,10 +9,10 @@ import sys
 import zipfile
 from urllib.request import urlretrieve
 
-URL = 'https://unicode.org/Public/cldr/44/cldr-common-44.0.zip'
-FILENAME = 'cldr-common-44.0.zip'
-# Via https://unicode.org/Public/cldr/44/hashes/SHASUM512
-FILESUM = 'f2cd8733948caf308d6e39eae21724da7f29f528f8969d456514e1e84ecd5f1e6936d0460414a968888bb1b597bc1ee723950ea47df5cba21a02bb14f96d18b6'
+URL = 'https://unicode.org/Public/cldr/45/cldr-common-45.0.zip'
+FILENAME = 'cldr-common-45.0.zip'
+# Via https://unicode.org/Public/cldr/45/hashes/SHASUM512.txt
+FILESUM = '638123882bd29911fc9492ec152926572fec48eb6c1f5dd706aee3e59cad8be4963a334bb7a09a645dbedc3356f60ef7ac2ef7ab4ccf2c8926b547782175603c'
 BLKSIZE = 131072
 
 
diff --git a/scripts/import_cldr.py b/scripts/import_cldr.py
index 633ca9a..761372a 100755
--- a/scripts/import_cldr.py
+++ b/scripts/import_cldr.py
@@ -315,6 +315,11 @@ def parse_global(srcdir, sup):
 
         for paternity in parentBlock.findall('./parentLocale'):
             parent = paternity.attrib['parent']
+            if parent == 'root':
+                # Since CLDR-45, the 'root' parent locale uses 'localeRules="nonlikelyScript"' instead of
+                # 'locales'. This special case is handled in babel when loading locale data
+                # (https://cldr.unicode.org/index/downloads/cldr-45#h.5rbkhkncdqi9)
+                continue
             for child in paternity.attrib['locales'].split():
                 parent_exceptions[child] = parent
 


Test patch:
diff --git a/tests/test_localedata.py b/tests/test_localedata.py
index 8a4fbef..721b91f 100644
--- a/tests/test_localedata.py
+++ b/tests/test_localedata.py
@@ -63,6 +63,21 @@ def test_load():
     assert localedata.load('en_US') is localedata.load('en_US')
 
 
+def test_load_inheritance(monkeypatch):
+    from babel.localedata import _cache
+
+    _cache.clear()
+    localedata.load('hi_Latn')
+    # Must not be ['root', 'hi_Latn'] even though 'hi_Latn' matches the 'lang_Script'
+    # form used by 'nonLikelyScripts'. This is because 'hi_Latn' has an explicit parent locale 'en_IN'.
+    assert list(_cache.keys()) == ['root', 'en', 'en_001', 'en_IN', 'hi_Latn']
+
+    _cache.clear()
+    localedata.load('az_Arab')
+    # Must not include 'az' as 'Arab' is not a likely script for 'az'.
+    assert list(_cache.keys()) == ['root', 'az_Arab']
+
+
 def test_merge():
     d = {1: 'foo', 3: 'baz'}
     localedata.merge(d, {1: 'Foo', 2: 'Bar'})
diff --git a/tests/test_numbers.py b/tests/test_numbers.py
index ed0531c..a96bdbe 100644
--- a/tests/test_numbers.py
+++ b/tests/test_numbers.py
@@ -250,7 +250,7 @@ def test_list_currencies():
 
     assert list_currencies(locale='pa_Arab') == {'PKR', 'INR', 'EUR'}
 
-    assert len(list_currencies()) == 305
+    assert len(list_currencies()) == 306
 
 
 def test_validate_currency():
diff --git a/tests/test_support.py b/tests/test_support.py
index 1262720..366c294 100644
--- a/tests/test_support.py
+++ b/tests/test_support.py
@@ -71,6 +71,14 @@ class TranslationsTestCase(unittest.TestCase):
         self.assertEqualTypeToo('Voh', self.translations.gettext('foo'))
         self.assertEqualTypeToo('VohCTX', self.translations.pgettext('foo',
                                                                      'foo'))
+        self.assertEqualTypeToo('VohCTX1', self.translations.pgettext('foo',
+                                                                      'foo1'))
+
+    def test_pgettext_fallback(self):
+        fallback = self.translations._fallback
+        self.translations._fallback = support.NullTranslations()
+        assert self.translations.pgettext('foo', 'bar') == 'bar'
+        self.translations._fallback = fallback
 
     def test_upgettext(self):
         self.assertEqualTypeToo('Voh', self.translations.ugettext('foo'))