Repository: python-babel/babel (python)

Issue:
localedata.locale_identifiers is slow
# localedata.locale_identifiers is slow

`localedata.locale_identifiers` reads the available locale identifiers from disk every time it is called. Since the locale data files are distributed with the lib, there shouldn't be a need to scan the disk every time this is called, as it could be cached.

I noticed a hot spot in this method when profiling a service that uses this library. Specifically I noticed that babel.Locale.parse performs very poorly when supplied bogus locales. I ran a small benchmark to demonstrate this:

# Steps to Reproduce

```
import timeit

import babel


def wrapped():
    try:
        babel.Locale.parse("en_ZH")
    except babel.UnknownLocaleError:
        pass


print(timeit.timeit(wrapped, number=1000))
```

# Actual Results
Above snippet takes several seconds to execute. 

Output on my machine:
```
3.91633701324
```

# Expected Results
Should be much faster.

# Reproducibility
100%

# Additional Information:
Could this information be built up in `scripts/import_cldr.py` and written to the `global.dat` file? Then the data could be quickly loaded as needed in `localedata.py`.




Pull request description:
Cache locale_identifiers()
Fixes #620

This speeds up the test code from #620 19-fold on my machine:

```
(babel) ~/b/babel (master) $ python exp_issue_620.py
7.1793154459999995
(babel) ~/b/babel (cache-locale-identifiers) $ python exp_issue_620.py
0.359975983
```

Patch:
diff --git a/babel/localedata.py b/babel/localedata.py
index 2a8c442..03381e2 100644
--- a/babel/localedata.py
+++ b/babel/localedata.py
@@ -57,13 +57,24 @@ def locale_identifiers():
     """Return a list of all locale identifiers for which locale data is
     available.
 
+    This data is cached after the first invocation in `locale_identifiers.cache`.
+
+    Removing the `locale_identifiers.cache` attribute or setting it to `None`
+    will cause this function to re-read the list from disk.
+
     .. versionadded:: 0.8.1
 
     :return: a list of locale identifiers (strings)
     """
-    return [stem for stem, extension in [
-        os.path.splitext(filename) for filename in os.listdir(_dirname)
-    ] if extension == '.dat' and stem != 'root']
+    data = getattr(locale_identifiers, 'cache', None)
+    if data is None:
+        locale_identifiers.cache = data = [
+            stem
+            for stem, extension in
+            (os.path.splitext(filename) for filename in os.listdir(_dirname))
+            if extension == '.dat' and stem != 'root'
+        ]
+    return data
 
 
 def load(name, merge_inherited=True):
diff --git a/scripts/download_import_cldr.py b/scripts/download_import_cldr.py
index f0d9b10..f118c69 100755
--- a/scripts/download_import_cldr.py
+++ b/scripts/download_import_cldr.py
@@ -13,9 +13,9 @@ except ImportError:
     from urllib import urlretrieve
 
 
-URL = 'https://unicode.org/Public/cldr/34/core.zip'
-FILENAME = 'cldr-core-34.0.zip'
-FILESUM = '1d79f6ce294ff350a3ee59d7dd69451fd24776f7105fd6a55c255ffb813c03ba'
+URL = 'https://unicode.org/Public/cldr/35.1/core.zip'
+FILENAME = 'cldr-core-35.1.zip'
+FILESUM = 'e2ede8cb8f9c29157e281ee9e696ce540a72c598841bed595a406b710eea87b0'
 BLKSIZE = 131072
 
 


Test patch:
diff --git a/tests/test_localedata.py b/tests/test_localedata.py
index dceb984..bffe763 100644
--- a/tests/test_localedata.py
+++ b/tests/test_localedata.py
@@ -17,6 +17,7 @@ from operator import methodcaller
 
 from babel import localedata
 
+
 class MergeResolveTestCase(unittest.TestCase):
 
     def test_merge_items(self):
@@ -78,6 +79,7 @@ def test_locale_identification():
     for l in localedata.locale_identifiers():
         assert localedata.exists(l)
 
+
 def test_unique_ids():
     # Check all locale IDs are uniques.
     all_ids = localedata.locale_identifiers()
@@ -93,6 +95,7 @@ def test_mixedcased_locale():
             methodcaller(random.choice(['lower', 'upper']))(c) for c in l])
         assert localedata.exists(locale_id)
 
+
 def test_locale_argument_acceptance():
     # Testing None input.
     normalized_locale = localedata.normalize_locale(None)
@@ -105,3 +108,26 @@ def test_locale_argument_acceptance():
     assert normalized_locale is None
     locale_exist = localedata.exists(['en_us', None])
     assert locale_exist == False
+
+
+def test_locale_identifiers_cache(monkeypatch):
+    original_listdir = localedata.os.listdir
+    listdir_calls = []
+    def listdir_spy(*args):
+        rv = original_listdir(*args)
+        listdir_calls.append((args, rv))
+        return rv
+    monkeypatch.setattr(localedata.os, 'listdir', listdir_spy)
+
+    # In case we've already run some tests...
+    if hasattr(localedata.locale_identifiers, 'cache'):
+        del localedata.locale_identifiers.cache
+
+    assert not listdir_calls
+    assert localedata.locale_identifiers()
+    assert len(listdir_calls) == 1
+    assert localedata.locale_identifiers() is localedata.locale_identifiers.cache
+    assert len(listdir_calls) == 1
+    localedata.locale_identifiers.cache = None
+    assert localedata.locale_identifiers()
+    assert len(listdir_calls) == 2