Repository: lingui/js-lingui
Pull request #1170 (2021-11-22 12:10:08)

feat(cli): make extract cmd asynchronous execution
Some [custom extractors](https://lingui.js.org/ref/conf.html#extractors) are required to have async execution. For instance, in order to extract messages from Svelte+Typescript you need to preprocess svelte files before compilation. Preprocessing is an async operation in Svelte (see [doc](https://svelte.dev/docs#svelte_preprocess)). But lingui-cli extract command doesn't support async extract method from custom extractor – see `catalog.make` method in `packages/cli/src/lingui-extract.ts` and all function calls down to `extract` method of a particular extractor.

This PR adds support for async extractors by awaiting for `extract` is finished before calling `traverse` function in `Catalog.collect` method

vercel[bot]: This pull request is being automatically deployed with Vercel ([learn more](https://vercel.link/github-learn-more)).  
To see the status of your deployment, click below or on the icon next to each commit.

🔍 Inspect: [https://vercel.com/lingui-js/js-lingui/2Vj1zugYn8G1u2i6cXKVDsJewAjJ](https://vercel.com/lingui-js/js-lingui/2Vj1zugYn8G1u2i6cXKVDsJewAjJ)  
✅ Preview: *In Progress*



Patch:
diff --git a/docs/misc/react-intl.rst b/docs/misc/react-intl.rst
index 153a9cfd..35f7fd84 100644
--- a/docs/misc/react-intl.rst
+++ b/docs/misc/react-intl.rst
@@ -126,7 +126,7 @@ Let's compare it with `react-intl`_ solution to see the difference:
          <Trans>Read the <a href="/docs">documentation</a>.</Trans>
       </p>
 
-   The message ID is ``Read the <0>documentation</0>.`` instead of ``msg.id``. Both
+   The message ID is ``Read the <0>documentation</0>.`` instead of ``msg.docs``. Both
    solutions have pros and cons and the library lets you choose the one which works best for you.
 
 Plurals
diff --git a/docs/misc/showroom.rst b/docs/misc/showroom.rst
index bd601d33..dde81842 100644
--- a/docs/misc/showroom.rst
+++ b/docs/misc/showroom.rst
@@ -17,3 +17,4 @@ Feel free to `send a PR <https://github.com/lingui/js-lingui/issues/new>`__ to l
 - `Notos <https://www.notos.co>`__
 - `Greenlight <https://greenlightready.com>`__
 - `OkCupid <https://www.okcupid.com>`__
+- `mapflow.ai <https://mapflow.ai>`__
diff --git a/packages/babel-plugin-extract-messages/src/index.ts b/packages/babel-plugin-extract-messages/src/index.ts
index 4ebfb2fb..91d9f6a1 100644
--- a/packages/babel-plugin-extract-messages/src/index.ts
+++ b/packages/babel-plugin-extract-messages/src/index.ts
@@ -96,6 +96,19 @@ function mapReplacer(key, value) {
   return value;
 }
 
+function extractStringContatentation(t, node, error): string {
+  if (t.isStringLiteral(node)) {
+    return node.value
+  } else if (t.isBinaryExpression(node)) {
+    return (
+      extractStringContatentation(t, node.left, error) +
+      extractStringContatentation(t, node.right, error)
+    )
+  } else {
+    throw error
+  }
+}
+
 export default function ({ types: t }) {
   let localTransComponentName
 
@@ -282,13 +295,31 @@ export default function ({ types: t }) {
         path.node.properties
           .filter(({ key }) => copyProps.indexOf(key.name) !== -1)
           .forEach(({ key, value }, i) => {
+            // By default, the value is just the string value of the object property.
+            let valueToExtract = value.value;
+
             if (key.name === "comment" && !t.isStringLiteral(value)) {
-              throw path
+              // Comments can be single or multi-line strings.
+              const errorIfNotAString = path
                 .get(`properties.${i}.value`)
                 .buildCodeFrameError("Only strings are supported as comments.")
+
+              if (t.isBinaryExpression(value)) {
+                valueToExtract = extractStringContatentation(
+                  t,
+                  value,
+                  errorIfNotAString
+                )
+              } else {
+                throw errorIfNotAString
+              }
+            } else if (key.name === "id") {
+                const isIdLiteral = !value.value && t.isTemplateLiteral(value)
+                if (isIdLiteral) {
+                    valueToExtract = value?.quasis[0]?.value?.cooked;
+                }
             }
-            const isIdLiteral = !value.value && key.name === "id" && t.isTemplateLiteral(value)
-            props[key.name] = isIdLiteral ? value?.quasis[0]?.value?.cooked : value.value
+            props[key.name] = valueToExtract;
           })
 
         collectMessage(path, file, props)
diff --git a/packages/cli/src/api/catalog.ts b/packages/cli/src/api/catalog.ts
index 85e90b27..40678099 100644
--- a/packages/cli/src/api/catalog.ts
+++ b/packages/cli/src/api/catalog.ts
@@ -98,8 +98,8 @@ export class Catalog {
     this.format = getFormat(config.format)
   }
 
-  make(options: MakeOptions) {
-    const nextCatalog = this.collect(options)
+  async make(options: MakeOptions) {
+    const nextCatalog = await this.collect(options)
     const prevCatalogs = this.readAll()
 
     const catalogs = this.merge(prevCatalogs, nextCatalog, {
@@ -126,8 +126,8 @@ export class Catalog {
     }
   }
 
-  makeTemplate(options: MakeTemplateOptions) {
-    const catalog = this.collect(options)
+  async makeTemplate(options: MakeTemplateOptions) {
+    const catalog = await this.collect(options)
     const sort = order(options.orderBy) as (catalog: CatalogType) => CatalogType
     this.writeTemplate(sort(catalog as CatalogType))
   }
@@ -135,7 +135,7 @@ export class Catalog {
   /**
    * Collect messages from source paths. Return a raw message catalog as JSON.
    */
-  collect(options: CollectOptions) {
+  async collect(options: CollectOptions) {
     const tmpDir = path.join(os.tmpdir(), `lingui-${process.pid}`)
 
     if (fs.existsSync(tmpDir)) {
@@ -152,15 +152,15 @@ export class Catalog {
         paths = paths.filter((path: string) => regex.test(path))
       }
 
-      paths.forEach((filename) =>
-        extract(filename, tmpDir, {
+      for (let filename of paths) {
+        await extract(filename, tmpDir, {
           verbose: options.verbose,
           configPath: options.configPath,
           babelOptions: this.config.extractBabelOptions,
           extractors: options.extractors,
           projectType: options.projectType,
         })
-      )
+      }
 
       return (function traverse(directory) {
         return fs
@@ -228,7 +228,7 @@ export class Catalog {
             options.overwrite)
 
         const translation = updateFromDefaults
-          ? nextCatalog[key].message
+          ? nextCatalog[key].message || key
           : prevCatalog[key].translation
 
         return {
diff --git a/packages/cli/src/api/compile.ts b/packages/cli/src/api/compile.ts
index 297a6434..0ebd8add 100644
--- a/packages/cli/src/api/compile.ts
+++ b/packages/cli/src/api/compile.ts
@@ -27,7 +27,7 @@ export type CreateCompileCatalogOptions = {
  */
 function compileSingleKey(key: string, translation: string, shouldPseudolocalize: boolean): t.ObjectProperty {
   if (shouldPseudolocalize) {
-    translation = pseudoLocalize(translation)
+    translation = pseudoLocalize(key)
   }
 
   return t.objectProperty(t.stringLiteral(key), compile(translation))
diff --git a/packages/cli/src/api/extract.ts b/packages/cli/src/api/extract.ts
index 5692ca81..d7e10f0c 100644
--- a/packages/cli/src/api/extract.ts
+++ b/packages/cli/src/api/extract.ts
@@ -38,7 +38,7 @@ function mergeMessage(msgId, prev, next) {
   }
 }
 
-export function extract(
+export async function extract(
   srcPaths: Array<string>,
   targetPath: string,
   options: ExtractOptions = {}
@@ -46,12 +46,12 @@ export function extract(
   const { ignore = [] } = options
   const ignorePattern = ignore.length ? new RegExp(ignore.join("|"), "i") : null
 
-  srcPaths.forEach((srcFilename) => {
+  for (let srcFilename of srcPaths) {
     if (
       !fs.existsSync(srcFilename) ||
       (ignorePattern && ignorePattern.test(srcFilename))
     )
-      return
+      continue
 
     if (fs.statSync(srcFilename).isDirectory()) {
       const subdirs = fs
@@ -59,12 +59,12 @@ export function extract(
         .sort()
         .map((filename) => path.join(srcFilename, filename))
 
-      extract(subdirs, targetPath, options)
-      return
+      await extract(subdirs, targetPath, options)
+      continue
     }
 
-    cliExtractor(srcFilename, targetPath, options)
-  })
+    await cliExtractor(srcFilename, targetPath, options)
+  }
 }
 
 export function collect(buildDir: string) {
diff --git a/packages/cli/src/api/extractors/index.ts b/packages/cli/src/api/extractors/index.ts
index 263eed94..530d5767 100644
--- a/packages/cli/src/api/extractors/index.ts
+++ b/packages/cli/src/api/extractors/index.ts
@@ -18,17 +18,17 @@ export type ExtractOptions = {
 
 export type ExtractorType = {
   match(filename: string): boolean
-  extract(filename: string, targetDir: string, options?: ExtractOptions): void
+  extract(filename: string, targetDir: string, options?: ExtractOptions): Promise<void> | void
 }
 
-export default function extract(
+export default async function extract(
   filename: string,
   targetPath: string,
   options: ExtractOptions
-): boolean {
+): Promise<boolean> {
   const extractorsToExtract = options.extractors ?? DEFAULT_EXTRACTORS
 
-  return extractorsToExtract.some((e) => {
+  for (let e of extractorsToExtract) {
     let ext: ExtractorType = e;
     if (typeof e === "string") {
       // in case of the user using require.resolve in their extractors, we require that module
@@ -38,13 +38,13 @@ export default function extract(
       ext = (ext as any).default
     }
 
-    if (!ext.match(filename)) return false
+    if (!ext.match(filename)) continue
 
     let spinner
     if (options.verbose) spinner = ora().start(filename)
 
     try {
-      ext.extract(filename, targetPath, options)
+      await ext.extract(filename, targetPath, options)
     } catch (e) {
       if (options.verbose && spinner) {
         spinner.fail(e.message)
@@ -56,5 +56,7 @@ export default function extract(
 
     if (options.verbose && spinner) spinner.succeed()
     return true
-  })
+  }
+
+  return false
 }
diff --git a/packages/cli/src/lingui-extract.ts b/packages/cli/src/lingui-extract.ts
index e3b83311..5a9dc90b 100644
--- a/packages/cli/src/lingui-extract.ts
+++ b/packages/cli/src/lingui-extract.ts
@@ -22,10 +22,10 @@ export type CliExtractOptions = {
   watch?: boolean
 }
 
-export default function command(
+export default async function command(
   config: LinguiConfig,
   options: Partial<CliExtractOptions>
-): boolean {
+): Promise<boolean> {
   // `react-app` babel plugin used by CRA requires either BABEL_ENV or NODE_ENV to be
   // set. We're setting it here, because lingui macros are going to use them as well.
   if (!process.env.BABEL_ENV && !process.env.NODE_ENV) {
@@ -41,8 +41,8 @@ export default function command(
 
   const catalogs = getCatalogs(config)
   const catalogStats: { [path: string]: AllCatalogsType } = {}
-  catalogs.forEach((catalog) => {
-    catalog.make({
+  for (let catalog of catalogs) {
+    await catalog.make({
       ...options,
       orderBy: config.orderBy,
       extractors: config.extractors,
@@ -50,7 +50,7 @@ export default function command(
     })
 
     catalogStats[catalog.path] = catalog.readAll()
-  })
+  }
 
   Object.entries(catalogStats).forEach(([key, value]) => {
     console.log(`Catalog statistics for ${key}: `)
@@ -173,11 +173,11 @@ if (require.main === module) {
 
     // CLear the previous timer if there is any, and schedule the next
     debounceTimer && clearTimeout(debounceTimer)
-    debounceTimer = setTimeout(() => {
+    debounceTimer = setTimeout(async () => {
       const filePath = [...changedPaths]
       changedPaths.clear()
 
-      extract(filePath)
+      await extract(filePath)
     }, program.debounce)
   }
 
@@ -210,10 +210,12 @@ if (require.main === module) {
   } else if (program.args) {
     // this behaviour occurs when we extract files by his name
     // for ex: lingui extract src/app, this will extract only files included in src/app
-    const result = extract(program.args)
-    if (!result) process.exit(1)
+    extract(program.args).then(result => {
+      if (!result) process.exit(1)
+    })
   } else {
-    const result = extract()
-    if (!result) process.exit(1)
+    extract().then(result => {
+      if (!result) process.exit(1)
+    })
   }
 }
diff --git a/packages/cli/src/services/translationIO.ts b/packages/cli/src/services/translationIO.ts
index 07c391c3..3fca19c2 100644
--- a/packages/cli/src/services/translationIO.ts
+++ b/packages/cli/src/services/translationIO.ts
@@ -43,7 +43,8 @@ export default function syncProcess(config, options) {
 // Cf. https://translation.io/docs/create-library#initialization
 function init(config, options, successCallback, failCallback) {
   const sourceLocale  = config.sourceLocale || 'en'
-  const targetLocales = config.locales.filter((value) => value != sourceLocale)
+  const pseudoLocale  = config.pseudoLocale || 'pseudo'
+  const targetLocales = config.locales.filter((value) => value != sourceLocale && value != pseudoLocale)
   const paths         = poPathsPerLocale(config)
 
   let segments = {}


Test patch:
diff --git a/packages/babel-plugin-extract-messages/test/__snapshots__/index.ts.snap b/packages/babel-plugin-extract-messages/test/__snapshots__/index.ts.snap
index b77949ae..bbb46d84 100644
--- a/packages/babel-plugin-extract-messages/test/__snapshots__/index.ts.snap
+++ b/packages/babel-plugin-extract-messages/test/__snapshots__/index.ts.snap
@@ -131,7 +131,7 @@ Object {
       origin: Array [
         Array [
           js-without-macros.js,
-          11,
+          13,
         ],
       ],
     },
@@ -153,7 +153,7 @@ Object {
     origin: Array [
       Array [
         js-without-macros.js,
-        7,
+        9,
       ],
     ],
   },
@@ -166,12 +166,23 @@ Object {
       ],
     ],
   },
+  Multiline: Object {
+    extractedComments: Array [
+      this is 2 lines long,
+    ],
+    origin: Array [
+      Array [
+        js-without-macros.js,
+        7,
+      ],
+    ],
+  },
   Values {param}: Object {
     extractedComments: Array [],
     origin: Array [
       Array [
         js-without-macros.js,
-        9,
+        11,
       ],
     ],
   },
diff --git a/packages/babel-plugin-extract-messages/test/fixtures/js-without-macros.js b/packages/babel-plugin-extract-messages/test/fixtures/js-without-macros.js
index 832361b9..c98941ba 100644
--- a/packages/babel-plugin-extract-messages/test/fixtures/js-without-macros.js
+++ b/packages/babel-plugin-extract-messages/test/fixtures/js-without-macros.js
@@ -4,6 +4,8 @@ const msg = /*i18n*/{id: 'Message'}
 
 const withDescription = /*i18n*/{id: 'Description', comment: "description"}
 
+const withMultilineComment = /*i18n*/{id: 'Multiline', comment: "this is " + "2 lines long"}
+
 const withId = /*i18n*/{id: 'ID', message: 'Message with id'}
 
 const withValues = /*i18n*/{id: 'Values {param}', values: { param: param }}
diff --git a/packages/cli/src/api/catalog.test.ts b/packages/cli/src/api/catalog.test.ts
index 88547feb..2f58cbe4 100644
--- a/packages/cli/src/api/catalog.test.ts
+++ b/packages/cli/src/api/catalog.test.ts
@@ -35,7 +35,7 @@ describe("Catalog", function () {
   })
 
   describe("make", function () {
-    it("should collect and write catalogs", function () {
+    it("should collect and write catalogs", async function () {
       const localeDir = copyFixture(fixture("locales", "initial"))
       const catalog = new Catalog(
         {
@@ -55,11 +55,11 @@ describe("Catalog", function () {
       // Everything should be empty
       expect(catalog.readAll()).toMatchSnapshot()
 
-      catalog.make(defaultMakeOptions)
+      await catalog.make(defaultMakeOptions)
       expect(catalog.readAll()).toMatchSnapshot()
     })
 
-    it("should only update the specified locale", function () {
+    it("should only update the specified locale", async function () {
       const localeDir = copyFixture(fixture("locales", "initial"))
       const catalog = new Catalog(
         {
@@ -79,11 +79,11 @@ describe("Catalog", function () {
       // Everything should be empty
       expect(catalog.readAll()).toMatchSnapshot()
 
-      catalog.make({ ...defaultMakeOptions, locale: "en" })
+      await catalog.make({ ...defaultMakeOptions, locale: "en" })
       expect(catalog.readAll()).toMatchSnapshot()
     })
 
-    it("should merge with existing catalogs", function () {
+    it("should merge with existing catalogs", async function () {
       const localeDir = copyFixture(fixture("locales", "existing"))
       const catalog = new Catalog(
         {
@@ -100,13 +100,13 @@ describe("Catalog", function () {
       // Everything should be empty
       expect(catalog.readAll()).toMatchSnapshot()
 
-      catalog.make(defaultMakeOptions)
+      await catalog.make(defaultMakeOptions)
       expect(catalog.readAll()).toMatchSnapshot()
     })
   })
 
   describe("makeTemplate", function () {
-    it("should collect and write a template", function () {
+    it("should collect and write a template", async function () {
       const localeDir = copyFixture(fixture("locales", "initial"))
       const catalog = new Catalog(
         {
@@ -126,13 +126,13 @@ describe("Catalog", function () {
       // Everything should be empty
       expect(catalog.readTemplate()).toMatchSnapshot()
 
-      catalog.makeTemplate(defaultMakeTemplateOptions)
+      await catalog.makeTemplate(defaultMakeTemplateOptions)
       expect(catalog.readTemplate()).toMatchSnapshot()
     })
   })
 
   describe("collect", function () {
-    it("should extract messages from source files", function () {
+    it("should extract messages from source files", async function () {
       const catalog = new Catalog(
         {
           name: "messages",
@@ -143,11 +143,11 @@ describe("Catalog", function () {
         mockConfig()
       )
 
-      const messages = catalog.collect(defaultMakeOptions)
+      const messages = await catalog.collect(defaultMakeOptions)
       expect(messages).toMatchSnapshot()
     })
 
-    it("should extract only files passed on options", function () {
+    it("should extract only files passed on options", async function () {
       const catalog = new Catalog(
         {
           name: "messages",
@@ -158,7 +158,7 @@ describe("Catalog", function () {
         mockConfig()
       )
 
-      const messages = catalog.collect({
+      const messages = await catalog.collect({
         ...defaultMakeOptions,
         files: [fixture("collect/componentA")]
       })
@@ -176,8 +176,8 @@ describe("Catalog", function () {
         mockConfig()
       )
 
-      mockConsole((console) => {
-        const messages = catalog.collect(defaultMakeOptions)
+      mockConsole(async (console) => {
+        const messages = await catalog.collect(defaultMakeOptions)
         expect(console.error).toBeCalledWith(
           expect.stringContaining(`Cannot process file`)
         )
@@ -330,12 +330,18 @@ describe("Catalog", function () {
             message: "",
             translation: "Message with custom ID",
           }),
+          "Message with <0>auto-generated</0> ID": makePrevMessage({
+            translation: "Source of message with <0>auto-generated</0> ID",
+          }),
         },
         cs: {
           "custom.id": makePrevMessage({
             message: "",
             translation: "Translation of message with custom ID",
           }),
+          "Message with <0>auto-generated</0> ID": makePrevMessage({
+            translation: "Translation of message with auto-generated ID",
+          }),
         },
       }
 
@@ -343,10 +349,11 @@ describe("Catalog", function () {
         "custom.id": makeNextMessage({
           message: "Message with custom ID, possibly changed",
         }),
+        "Message with <0>auto-generated</0> ID": makeNextMessage(),
       }
 
       // Without `overwrite`:
-      // The translation of `custom.id` message for `sourceLocale` is kept intact
+      // The translations of all IDs for `sourceLocale` are kept intact
       expect(
         makeCatalog({ sourceLocale: "en" }).merge(
           prevCatalogs,
@@ -359,17 +366,23 @@ describe("Catalog", function () {
             message: "Message with custom ID, possibly changed",
             translation: "Message with custom ID",
           }),
+          "Message with <0>auto-generated</0> ID": expect.objectContaining({
+            translation: "Source of message with <0>auto-generated</0> ID",
+          }),
         },
         cs: {
           "custom.id": expect.objectContaining({
             message: "Message with custom ID, possibly changed",
             translation: "Translation of message with custom ID",
           }),
+          "Message with <0>auto-generated</0> ID": expect.objectContaining({
+            translation: "Translation of message with auto-generated ID",
+          }),
         },
       })
 
       // With `overwrite`
-      // The translation of `custom.id` message for `sourceLocale` is changed
+      // The translations of all IDs for `sourceLocale` are changed
       expect(
         makeCatalog({ sourceLocale: "en" }).merge(prevCatalogs, nextCatalog, {
           overwrite: true,
@@ -380,12 +393,18 @@ describe("Catalog", function () {
             message: "Message with custom ID, possibly changed",
             translation: "Message with custom ID, possibly changed",
           }),
+          "Message with <0>auto-generated</0> ID": expect.objectContaining({
+            translation: "Message with <0>auto-generated</0> ID",
+          }),
         },
         cs: {
           "custom.id": expect.objectContaining({
             message: "Message with custom ID, possibly changed",
             translation: "Translation of message with custom ID",
           }),
+          "Message with <0>auto-generated</0> ID": expect.objectContaining({
+            translation: "Translation of message with auto-generated ID",
+          }),
         },
       })
     })
diff --git a/packages/cli/src/api/extract.test.ts b/packages/cli/src/api/extract.test.ts
index 3b2fd273..dfe776d2 100644
--- a/packages/cli/src/api/extract.test.ts
+++ b/packages/cli/src/api/extract.test.ts
@@ -60,8 +60,8 @@ describe("extract", function () {
     mockFs.restore()
   })
 
-  it("should traverse directory and call extractors", function () {
-    extract(["src"], "locale", {
+  it("should traverse directory and call extractors", async function () {
+    await extract(["src"], "locale", {
       ignore: ["forbidden"],
       extractors: [
         babel,
@@ -153,8 +153,8 @@ describe("extract", function () {
     )
   })
 
-  it("by default the traverse directory only uses babel", function () {
-    extract(["src"], "locale", {
+  it("by default the traverse directory only uses babel", async function () {
+    await extract(["src"], "locale", {
       ignore: ["forbidden"],
       babelOptions: {},
     })
diff --git a/packages/cli/src/api/formats/__snapshots__/po-gettext.test.ts.snap b/packages/cli/src/api/formats/__snapshots__/po-gettext.test.ts.snap
index b1d7f6ab..505f1d63 100644
--- a/packages/cli/src/api/formats/__snapshots__/po-gettext.test.ts.snap
+++ b/packages/cli/src/api/formats/__snapshots__/po-gettext.test.ts.snap
@@ -4,6 +4,7 @@ exports[` 1`] = `
 Object {
   message_with_id: Object {
     comments: Array [],
+    context: null,
     extractedComments: Array [],
     flags: Array [],
     obsolete: false,
@@ -12,6 +13,7 @@ Object {
   },
   message_with_id_but_without_translation: Object {
     comments: Array [],
+    context: null,
     extractedComments: Array [
       Comment made by the developers.,
     ],
@@ -22,6 +24,7 @@ Object {
   },
   {anotherCount, plural, one {Singular case} other {Case number {anotherCount}}}: Object {
     comments: Array [],
+    context: null,
     extractedComments: Array [],
     flags: Array [],
     obsolete: false,
@@ -30,6 +33,7 @@ Object {
   },
   {count, plural, one {Singular} other {Plural}}: Object {
     comments: Array [],
+    context: null,
     extractedComments: Array [],
     flags: Array [],
     obsolete: false,