Repository: lingui/js-lingui
Pull request #1080 (2021-06-08 15:54:21)

feat: load remote catalogs with remoteLoader()
This new feature opens a whole world for loading the messages:

Now you could perfectly store your `.json`, `.po`, or even `.csv` files of Lingui in remote environments like an AWS S3, and recover them dynamically.

Basically now you could so something similar to this:

```ts
import { i18n } from "@lingui/core"
import { remoteLoader } from "@lingui/loader"

export async function remoteActive(locale: string) {
  const remoteMessages = await fetch(`https://some-api/${locale}/messages`)
  const compiledMessages = remoteLoader(locale, remoteMessages)
  i18n.load(locale, compiledMessages)
  i18n.activate(locale)
}
```

### How does this handle fallbacks?

You can perfectly pass a fallback messages object, it can be a .json file or even a .po file.

```ts
import { i18n } from "@lingui/core"
import { remoteLoader } from "@lingui/loader"

import { messages as fallbackMessages } from "./locales/en/messages"

export async function remoteActive(locale: string) {
  const remoteMessages = await fetch(`https://some-api/${locale}/messages`)
  const compiledMessages = remoteLoader(locale, remoteMessages, fallbackMessages)
  i18n.load(locale, compiledMessages)
  i18n.activate(locale)
}
```

There's pending to write documentation of this, I'll push the documentation along the week since I'm not sure if this will 100% work great

vercel[bot]: This pull request is being automatically deployed with Vercel ([learn more](https://vercel.link/github-learn-more)).  
To see the status of your deployment, click below or on the icon next to each commit.

🔍 Inspect: [https://vercel.com/lingui-js/js-lingui/Ax31mf3u7CKe3VsnKSQPMmtKnQup](https://vercel.com/lingui-js/js-lingui/Ax31mf3u7CKe3VsnKSQPMmtKnQup)  
✅ Preview: [https://js-lingui-git-feat-remote-loader-lingui-js.vercel.app](https://js-lingui-git-feat-remote-loader-lingui-js.vercel.app)


github-actions[bot]: ## size-limit report 📦 
| Path                                                               | Size         |
| ------------------------------------------------------------------ | ------------ |
| ./packages/core/build/cjs/core.production.min.js                   | 2.78 KB (0%) |
| ./packages/detect-locale/build/cjs/detect-locale.production.min.js | 798 B (0%)   |
| ./packages/react/build/cjs/react.production.min.js                 | 4.85 KB (0%) |
| ./packages/core/build/esm/index.js                                 | 2.75 KB (0%) |
| ./packages/core/build/esm/dev.js                                   | 7.01 KB (0%) |
| ./packages/detect-locale/build/esm/index.js                        | 852 B (0%)   |
| ./packages/react/build/esm/index.js                                | 4.83 KB (0%) |

Patch:
diff --git a/packages/cli/src/api/formats/csv.ts b/packages/cli/src/api/formats/csv.ts
index 73f8901e..c2c11419 100644
--- a/packages/cli/src/api/formats/csv.ts
+++ b/packages/cli/src/api/formats/csv.ts
@@ -48,6 +48,10 @@ const csv: CatalogFormatter = {
       throw new Error(`Cannot read ${filename}: ${e.message}`)
     }
   },
+
+  parse(content) {
+    return deserialize(content)
+  }
 }
 
 export default csv
diff --git a/packages/cli/src/api/formats/index.ts b/packages/cli/src/api/formats/index.ts
index 02001550..e75d4f68 100644
--- a/packages/cli/src/api/formats/index.ts
+++ b/packages/cli/src/api/formats/index.ts
@@ -28,6 +28,7 @@ export type CatalogFormatter = {
     options?: CatalogFormatOptionsInternal
   ): void
   read(filename: string): CatalogType | null
+  parse(content): any
 }
 
 export default function getFormat(name: CatalogFormat): CatalogFormatter {
diff --git a/packages/cli/src/api/formats/lingui.ts b/packages/cli/src/api/formats/lingui.ts
index b010d8a4..6b97fb6f 100644
--- a/packages/cli/src/api/formats/lingui.ts
+++ b/packages/cli/src/api/formats/lingui.ts
@@ -49,6 +49,10 @@ const lingui: CatalogFormatter = {
       throw new Error(`Cannot read ${filename}: ${e.message}`)
     }
   },
+
+  parse(content) {
+    return content
+  }
 }
 
 export default lingui
diff --git a/packages/cli/src/api/formats/minimal.ts b/packages/cli/src/api/formats/minimal.ts
index e59cc0cf..635da993 100644
--- a/packages/cli/src/api/formats/minimal.ts
+++ b/packages/cli/src/api/formats/minimal.ts
@@ -36,6 +36,10 @@ const minimal: CatalogFormatter = {
       throw new Error(`Cannot read ${filename}: ${e.message}`)
     }
   },
+
+  parse(content: Record<string, any>) {
+    return deserialize(content)
+  }
 }
 
 export default minimal
diff --git a/packages/cli/src/api/formats/po-gettext.ts b/packages/cli/src/api/formats/po-gettext.ts
index 2fc75906..9f99017a 100644
--- a/packages/cli/src/api/formats/po-gettext.ts
+++ b/packages/cli/src/api/formats/po-gettext.ts
@@ -332,7 +332,7 @@ const poGettext: CatalogFormatter & PoFormatter = {
     return this.parse(raw)
   },
 
-  parse(raw) {
+  parse(raw: string) {
     const po = PO.parse(raw)
     convertPluralsToICU(po.items, po.headers.Language)
     return deserialize(indexItems(po.items))
diff --git a/packages/cli/src/api/formats/po.ts b/packages/cli/src/api/formats/po.ts
index 914cb1ee..7ed55681 100644
--- a/packages/cli/src/api/formats/po.ts
+++ b/packages/cli/src/api/formats/po.ts
@@ -106,7 +106,7 @@ const po: CatalogFormatter & PoFormatter = {
     return this.parse(raw)
   },
 
-  parse(raw) {
+  parse(raw: string) {
     const po = PO.parse(raw)
     validateItems(po.items)
     return deserialize(indexItems(po.items))
diff --git a/packages/loader/index.d.ts b/packages/loader/index.d.ts
new file mode 100644
index 00000000..d19498dd
--- /dev/null
+++ b/packages/loader/index.d.ts
@@ -0,0 +1,3 @@
+declare type RemoteLoaderMessages<T> = string | Record<string, any> | T;
+export declare function remoteLoader<T>(locale: string, messages: RemoteLoaderMessages<T>, fallbackMessages?: RemoteLoaderMessages<T>): any;
+export {};
diff --git a/packages/loader/index.js b/packages/loader/index.js
index c14e96c0..39ac9ca5 100644
--- a/packages/loader/index.js
+++ b/packages/loader/index.js
@@ -1,1 +1,2 @@
 export { default } from "./src"
+export { remoteLoader } from "./src/remoteLoader"
\ No newline at end of file
diff --git a/packages/loader/package.json b/packages/loader/package.json
index 72b307b9..5f926a65 100644
--- a/packages/loader/package.json
+++ b/packages/loader/package.json
@@ -2,6 +2,7 @@
   "name": "@lingui/loader",
   "version": "3.9.0",
   "description": "webpack loader for lingui message catalogs",
+  "types": "index.d.ts",
   "main": "index.js",
   "author": {
     "name": "Tomáš Ehrlich",
diff --git a/packages/loader/src/index.ts b/packages/loader/src/index.ts
new file mode 100644
index 00000000..a4df3dc3
--- /dev/null
+++ b/packages/loader/src/index.ts
@@ -0,0 +1,2 @@
+export { default } from "./webpackLoader"
+export { remoteLoader } from "./remoteLoader"
\ No newline at end of file
diff --git a/packages/loader/src/remoteLoader.ts b/packages/loader/src/remoteLoader.ts
new file mode 100644
index 00000000..e89bd042
--- /dev/null
+++ b/packages/loader/src/remoteLoader.ts
@@ -0,0 +1,57 @@
+import R from "ramda"
+import { getConfig } from "@lingui/conf"
+import { createCompiledCatalog, getFormat } from "@lingui/cli/api"
+
+type RemoteLoaderMessages<T> = string | Record<string, any> | T
+
+export function remoteLoader<T>(locale: string, messages: RemoteLoaderMessages<T>, fallbackMessages?: RemoteLoaderMessages<T>) {
+  const config = getConfig()
+
+  // When format is minimal, everything works fine because are .json files,
+  // but when is csv, .po, .po-gettext needs to be parsed to something interpretable
+  let parsedMessages;
+  let parsedFallbackMessages;
+  if (config.format) {
+    const formatter = getFormat(config.format)
+    if (fallbackMessages) {
+      // we do this because, people could just import the fallback and import the ./en/messages.js
+      // generated by lingui and the use case of format .po but fallback as .json module could be perfectly valid
+      parsedFallbackMessages = typeof fallbackMessages === "object" ? getFormat("minimal").parse(fallbackMessages) : formatter.parse(fallbackMessages)
+    }
+
+    parsedMessages = formatter.parse(messages)
+  } else {
+    throw new Error(`
+        *format* value in the Lingui configuration is required to make this loader 100% functional
+        Read more about this here: https://lingui.js.org/ref/conf.html#format
+      `)
+  }
+
+
+  const mapTranslationsToInterporlatedString = R.mapObjIndexed(
+    (_, key) => {
+      // if there's fallback and translation is empty, return the fallback
+      if (parsedMessages[key].translation === "" && parsedFallbackMessages?.[key]?.translation) {
+        return parsedFallbackMessages[key].translation
+      }
+
+      return parsedMessages[key].translation
+    },
+    parsedMessages
+  )
+
+  // In production we don't want untranslated strings. It's better to use message
+  // keys as a last resort.
+  // In development, however, we want to catch missing strings with `missing` parameter
+  // of I18nProvider (React) or setupI18n (core) and therefore we need to get
+  // empty translations if missing.
+  const strict = process.env.NODE_ENV !== "production"
+  return createCompiledCatalog(locale, mapTranslationsToInterporlatedString, {
+    strict,
+    ...config,
+    namespace: config.compileNamespace,
+    pseudoLocale: config.pseudoLocale,
+  })
+
+}
+
diff --git a/packages/loader/src/index.js b/packages/loader/src/webpackLoader.ts
similarity index 99%
rename from packages/loader/src/index.js
rename to packages/loader/src/webpackLoader.ts
index 5a244590..b6cf178c 100644
--- a/packages/loader/src/index.js
+++ b/packages/loader/src/webpackLoader.ts
@@ -18,7 +18,7 @@ try {
   JavascriptGenerator = require("webpack/lib/JavascriptGenerator")
 } catch (error) {
   if (error.code !== "MODULE_NOT_FOUND") {
-    throw e
+    throw error
   }
 }
 


Test patch:
diff --git a/packages/loader/src/remoteLoader.test.ts b/packages/loader/src/remoteLoader.test.ts
new file mode 100644
index 00000000..e2c08748
--- /dev/null
+++ b/packages/loader/src/remoteLoader.test.ts
@@ -0,0 +1,95 @@
+import { remoteLoader } from "./remoteLoader"
+import fs from "fs"
+import path from "path"
+
+describe("remote-loader", () => {
+  it("should compile correctly JSON messages coming from the fly", async () => {
+    const unlink = createConfig("minimal")
+    const messages = await simulatedJsonResponse()
+    expect(remoteLoader("en", messages)).toMatchInlineSnapshot(
+      `/*eslint-disable*/module.exports={messages:{"property.key":"value","{0} Deposited":[["0"]," Deposited"],"{0} Strategy":[["0"]," Strategy"]}};`
+    )
+    unlink()
+  })
+
+  it("should compile correctly .po messages coming from the fly", async () => {
+    const unlink = createConfig("po")
+    const messages = await simulatedPoResponse()
+    expect(remoteLoader("en", messages)).toMatchInlineSnapshot(
+      `/*eslint-disable*/module.exports={messages:{"Hello World":"Hello World","My name is {name}":["My name is ",["name"]]}};`
+    )
+    unlink()
+  })
+
+  describe("fallbacks", () => {
+    it("should fallback correctly to the fallback collection", async () => {
+      const unlink = createConfig("minimal")
+      const messages = await simulatedJsonResponse(true)
+      const fallbackMessages = await simulatedJsonResponse()
+
+      expect(
+        remoteLoader("en", messages, fallbackMessages)
+      ).toMatchInlineSnapshot(
+        `/*eslint-disable*/module.exports={messages:{"property.key":"value","{0} Deposited":[["0"]," Deposited"],"{0} Strategy":[["0"]," Strategy"]}};`
+      )
+      unlink()
+    })
+
+    it("should fallback to compiled fallback", async () => {
+      const unlink = createConfig("po")
+      const messages = await simulatedPoResponse("es")
+      const fallbackMessages = await simulatedPoCompiledFile()
+
+      expect(
+        remoteLoader("en", messages, fallbackMessages)
+      ).toMatchInlineSnapshot(
+        `/*eslint-disable*/module.exports={messages:{"Hello World":"Hello World","My name is {name}":["My name is ",["name"]]}};`
+      )
+      unlink()
+    })
+  })
+})
+
+function simulatedJsonResponse(nully?: boolean) {
+  return new Promise((resolve) => {
+    resolve({
+      "property.key": nully ? "" : "value",
+      "{0} Deposited": "{0} Deposited",
+      "{0} Strategy": "{0} Strategy",
+    })
+  })
+}
+
+function simulatedPoResponse(locale = "en") {
+  return new Promise((resolve) => {
+    const file = fs.readFileSync(
+      path.join(__dirname, "..", "test/locale/" + locale + "/messages.po"),
+      "utf-8"
+    )
+    resolve(file)
+  })
+}
+
+function simulatedPoCompiledFile() {
+  return new Promise((resolve) => {
+    resolve({
+      "Hello World": "Hello World",
+      "My name is {name}": ["My name is ", ["name"]],
+    })
+  })
+}
+
+function createConfig(format: string) {
+  const filename = `${process.cwd()}/.linguirc`
+  const config = `
+  {
+    'locales': ['en'],
+    'catalogs': [{
+      'path': 'locale/{locale}/messages'
+    }],
+    'format': '${format}'
+  }
+  `
+  fs.writeFileSync(filename, config)
+  return () => fs.unlinkSync(filename)
+}
diff --git a/packages/loader/test/locale/es/messages.po b/packages/loader/test/locale/es/messages.po
new file mode 100644
index 00000000..6647c108
--- /dev/null
+++ b/packages/loader/test/locale/es/messages.po
@@ -0,0 +1,5 @@
+msgid "Hello World"
+msgstr ""
+
+msgid "My name is {name}"
+msgstr "My name is {name}"