Repository: elixir-gettext/gettext (elixir)

Issue:
Compilation time with many locales
In my application, gettext compile takes around 10 minutes in developer's machines (docker inside mac, so i don't expect it to fly. On CI and production it is several minutes, but not as much as 10)
I have around 20 locales with around 6000 strings each, and I will keep adding more, so I need a way to decrease developer's time spent on recompilations.
I'm not as concerned about paying this price on production or CI, but for tests and development, i'm trying to find a way to use less locales.

My first thought was to have a `priv/gettext_dev` folder for development, and the normal `priv/gettext` for production. But that would force me to an extra step each time we run gettext_extract, as it's ran in the development environment, copying the .pot files from one folder to the other.
Second thought is to have a way to restrict the number of locales processed in development environment. Would you accept a PR with a whitelist of locales to compile, so it can be configured per environment?

Do you have any other idea or out of the box solution that i'm missing?

Pull request:
Support configuration of locales by environment
Added a configuration option `allowed_locales` to restrict the locales that gettext bundles in the backend.

This patch introduces slightly more changes than I was expecting. 
In the previous version, the filesystem was read twice. One to discover the locales, and another to load the po files.
If I had kept this approach, I should have done the filtering of locales twice, with different inputs.
In the current approach, I reach for the filesystem once, and filter the PO files there based on the configuration, returning the path, locale and domain in the same data structure.
This structure is then cascaded to the different compilers

closes #238 

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7d5b133..9c29b72 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## v0.18.0
+
+* Add `:allowed_locales` to restrict the locales bundled in the backend
+
 ## v0.17.4
 
 * Do not change the return types of `*_noop` macros (regression in v0.17.2 and v0.17.3)
diff --git a/lib/gettext.ex b/lib/gettext.ex
index 9259ceb..5a41256 100644
--- a/lib/gettext.ex
+++ b/lib/gettext.ex
@@ -472,6 +472,9 @@ defmodule Gettext do
       This reduces compilation times and beam file sizes for large projects.
       This option requires Elixir v1.6.
 
+    * `:allowed_locales` - a list of locales to bundle in the backend.
+      Defaults to all the locales discovered in the `:priv` directory
+
   ### Mix tasks configuration
 
   You can configure Gettext Mix tasks under the `:gettext` key in the
diff --git a/lib/gettext/compiler.ex b/lib/gettext/compiler.ex
index 57b0c28..ef79ce4 100644
--- a/lib/gettext/compiler.ex
+++ b/lib/gettext/compiler.ex
@@ -33,7 +33,8 @@ defmodule Gettext.Compiler do
     priv = Keyword.get(opts, :priv, @default_priv)
     translations_dir = Application.app_dir(otp_app, priv)
     external_file = String.replace(Path.join(".compile", priv), "/", "_")
-    known_locales = known_locales(translations_dir)
+    known_po_files = known_po_files(translations_dir, opts)
+    known_locales = Enum.map(known_po_files, & &1[:locale]) |> Enum.uniq()
 
     default_locale =
       opts[:default_locale] || quote(do: Application.fetch_env!(:gettext, :default_locale))
@@ -62,7 +63,7 @@ defmodule Gettext.Compiler do
       def lgettext(locale, domain, msgctxt \\ nil, msgid, bindings)
       def lngettext(locale, domain, msgctxt \\ nil, msgid, msgid_plural, n, bindings)
 
-      unquote(compile_po_files(env, translations_dir, opts))
+      unquote(compile_po_files(env, known_po_files, opts))
 
       # Catch-all clauses.
       def lgettext(locale, domain, msgctxt, msgid, bindings),
@@ -353,13 +354,12 @@ defmodule Gettext.Compiler do
 
   # Compiles all the `.po` files in the given directory (`dir`) into `lgettext/4`
   # and `lngettext/6` function clauses.
-  defp compile_po_files(env, dir, opts) do
+  defp compile_po_files(env, known_po_files, opts) do
     plural_mod = Keyword.get(opts, :plural_forms, Gettext.Plural)
-    po_files = po_files_in_dir(dir)
 
     if Keyword.get(opts, :one_module_per_locale, false) do
       {quoted, locales} =
-        Enum.map_reduce(po_files, %{}, &compile_parallel_po_file(env, &1, &2, plural_mod))
+        Enum.map_reduce(known_po_files, %{}, &compile_parallel_po_file(env, &1, &2, plural_mod))
 
       locales
       |> Enum.map(&Kernel.ParallelCompiler.async(fn -> create_locale_module(env, &1) end))
@@ -367,7 +367,7 @@ defmodule Gettext.Compiler do
 
       quoted
     else
-      Enum.map(po_files, &compile_serial_po_file(env, &1, plural_mod))
+      Enum.map(known_po_files, &compile_serial_po_file(env, &1, plural_mod))
     end
   end
 
@@ -377,9 +377,9 @@ defmodule Gettext.Compiler do
     :ok
   end
 
-  defp compile_serial_po_file(env, path, plural_mod) do
+  defp compile_serial_po_file(env, po_file, plural_mod) do
     {locale, domain, singular_fun, plural_fun, quoted} =
-      compile_po_file(:defp, path, env, plural_mod)
+      compile_po_file(:defp, po_file, env, plural_mod)
 
     quote do
       unquote(quoted)
@@ -394,9 +394,9 @@ defmodule Gettext.Compiler do
     end
   end
 
-  defp compile_parallel_po_file(env, path, locales, plural_mod) do
+  defp compile_parallel_po_file(env, po_file, locales, plural_mod) do
     {locale, domain, singular_fun, plural_fun, locale_module_quoted} =
-      compile_po_file(:def, path, env, plural_mod)
+      compile_po_file(:def, po_file, env, plural_mod)
 
     module = :"#{env.module}.T_#{locale}"
 
@@ -417,8 +417,8 @@ defmodule Gettext.Compiler do
 
   # Compiles a .po file into a list of lgettext/5 (for translations) and
   # lngettext/7 (for plural translations) clauses.
-  defp compile_po_file(kind, path, env, plural_mod) do
-    {locale, domain} = locale_and_domain_from_path(path)
+  defp compile_po_file(kind, po_file, env, plural_mod) do
+    %{locale: locale, domain: domain, path: path} = po_file
     %PO{translations: translations, file: file} = PO.parse_file!(path)
 
     singular_fun = :"#{locale}_#{domain}_lgettext"
@@ -629,12 +629,19 @@ defmodule Gettext.Compiler do
     |> Path.wildcard()
   end
 
-  # Returns all the locales in `translations_dir` (which are the locales known
-  # by the compiled backend).
-  defp known_locales(translations_dir) do
+  # Returns the known the PO files in `translations_dir` with their locale and domain
+  # If allowed_locales is configured, it removes all the PO files that do not belong
+  # to those locales
+  defp known_po_files(translations_dir, opts) do
     case File.ls(translations_dir) do
-      {:ok, files} ->
-        Enum.filter(files, &File.dir?(Path.join(translations_dir, &1)))
+      {:ok, _} ->
+        translations_dir
+        |> po_files_in_dir()
+        |> Enum.map(fn path ->
+          {locale, domain} = locale_and_domain_from_path(path)
+          %{locale: locale, path: path, domain: domain}
+        end)
+        |> maybe_restrict_locales(opts[:allowed_locales])
 
       {:error, :enoent} ->
         []
@@ -643,4 +650,13 @@ defmodule Gettext.Compiler do
         raise File.Error, reason: reason, action: "list directory", path: translations_dir
     end
   end
+
+  defp maybe_restrict_locales(po_files, nil) do
+    po_files
+  end
+
+  defp maybe_restrict_locales(po_files, allowed_locales) when is_list(allowed_locales) do
+    allowed_locales = MapSet.new(allowed_locales)
+    Enum.filter(po_files, &MapSet.member?(allowed_locales, &1[:locale]))
+  end
 end
diff --git a/lib/mix/tasks/gettext.merge.ex b/lib/mix/tasks/gettext.merge.ex
index d1a9a1e..6d70457 100644
--- a/lib/mix/tasks/gettext.merge.ex
+++ b/lib/mix/tasks/gettext.merge.ex
@@ -214,6 +214,7 @@ defmodule Mix.Tasks.Gettext.Merge do
   end
 
   defp write_file(path, contents, stats) do
+    File.mkdir_p!(Path.dirname(path))
     File.write!(path, contents)
     Mix.shell().info("Wrote #{path} (#{format_stats(stats)})")
   end


Test patch:
diff --git a/test/fixtures/test_application/translations/es/LC_MESSAGES/default.po b/test/fixtures/test_application/translations/es/LC_MESSAGES/default.po
new file mode 100644
index 0000000..3485621
--- /dev/null
+++ b/test/fixtures/test_application/translations/es/LC_MESSAGES/default.po
@@ -0,0 +1,2 @@
+msgid "Hello world"
+msgstr "Hola mundo"
diff --git a/test/gettext_test.exs b/test/gettext_test.exs
index c65b9a2..24f6fd1 100644
--- a/test/gettext_test.exs
+++ b/test/gettext_test.exs
@@ -6,6 +6,10 @@ defmodule GettextTest.TranslatorWithCustomPriv do
   use Gettext, otp_app: :test_application, priv: "translations"
 end
 
+defmodule GettextTest.TranslatorWithAllowedLocales do
+  use Gettext, otp_app: :test_application, priv: "translations", allowed_locales: ["es"]
+end
+
 defmodule GettextTest.TranslatorWithCustomPluralForms do
   defmodule Plural do
     @behaviour Gettext.Plural
@@ -40,11 +44,13 @@ defmodule GettextTest do
 
   alias GettextTest.Translator
   alias GettextTest.TranslatorWithCustomPriv
+  alias GettextTest.TranslatorWithAllowedLocales
   alias GettextTest.TranslatorWithCustomPluralForms
   alias GettextTest.HandleMissingTranslation
 
   require Translator
   require TranslatorWithCustomPriv
+  require TranslatorWithAllowedLocales
 
   test "the default locale is \"en\"" do
     assert Gettext.get_locale() == "en"
@@ -135,6 +141,14 @@ defmodule GettextTest do
              {:ok, "Indirizzo email non valido"}
   end
 
+  test "allowed_locales ignores other locales" do
+    assert TranslatorWithAllowedLocales.lgettext("it", "default", nil, "Hello world", %{}) ==
+             {:default, "Hello world"}
+
+    assert TranslatorWithAllowedLocales.lgettext("es", "default", nil, "Hello world", %{}) ==
+             {:ok, "Hola mundo"}
+  end
+
   test "using a custom Gettext.Plural module" do
     alias TranslatorWithCustomPluralForms, as: T
 
@@ -700,7 +714,8 @@ defmodule GettextTest do
 
   test "known_locales/1: returns all the locales for which a backend has PO files" do
     assert Gettext.known_locales(Translator) == ["it"]
-    assert Gettext.known_locales(TranslatorWithCustomPriv) == ["it"]
+    assert Gettext.known_locales(TranslatorWithCustomPriv) == ["es", "it"]
+    assert Gettext.known_locales(TranslatorWithAllowedLocales) == ["es"]
   end
 
   test "a warning is issued in l(n)gettext when the domain contains slashes" do