Repository: lingui/js-lingui (ts)

Issue:
Origins loss for second locale when formatOptions.lineNumbers is set to false
I'm using lingui version 3.8.9

In my config I have defined two locales

`"locales": ["en", "de"]`

and set formatOptions.lineNumbers to false

`"formatOptions":{"origins": true, "lineNumbers": false}`

My catalogs path is `locales/{locale}/messages`

**Describe the bug**
After I run `lingui extract` I got the following result:

My `locales/en/messages` turned from
```
#: src/App.js:10
msgid "This is a Test"
msgstr "This is a Test"
```
into
```
#: src/App.js
msgid "This is a Test"
msgstr "This is a Test"
```
(Which is fine)

But for my `de` locale the catalog `locales/de/messages` turned from
```
#: src/App.js:10
msgid "This is a Test"
msgstr "Das ist ein Test"
```
into
```
#: 
msgid "This is a Test"
msgstr "Das ist ein Test"
```

As you can see, not just the line number is removed also the origin is not longer in the catalog.
If I switch the direction of the locales in the config from `"locales": ["en", "de"]` to `"locales": ["de", "en"]`
the bug effects my `locales/en/messages` and just `locales/de/messages` is fine


**Expected behavior**
With the config `"formatOptions":{"origins": true, "lineNumbers": false}`
all catalogs get rid of the lineNumbers but keep the origins.

**Additional context**
- jsLingui version 3.8.9


Pull request description:
fix: lineNumbers false was removing the entire origin
Resolves #1039 

Patch:
diff --git a/docs/misc/react-intl.rst b/docs/misc/react-intl.rst
index 178c2ecd..5f7f6ef8 100644
--- a/docs/misc/react-intl.rst
+++ b/docs/misc/react-intl.rst
@@ -42,7 +42,7 @@ Suppose we have the following text:
 
 .. code-block:: html
 
-   <p>Read the <a href="/docs>documentation</a>.</p>
+   <p>Read the <a href="/docs">documentation</a>.</p>
 
 In `react-intl`_, this would be translated as:
 
@@ -83,7 +83,7 @@ Let's go back to the previous example:
 .. code-block:: html
 
    <p>
-      Read the <a href="/docs>documentation</a>.
+      Read the <a href="/docs">documentation</a>.
    </p>
 
 All we need to do is to wrap the message in a :jsxmacro:`Trans` macro:
@@ -91,7 +91,7 @@ All we need to do is to wrap the message in a :jsxmacro:`Trans` macro:
 .. code-block:: html
 
    <p>
-      <Trans id="msg.docs">Read the <a href="/docs>documentation</a>.</Trans>
+      <Trans id="msg.docs">Read the <a href="/docs">documentation</a>.</Trans>
    </p>
 
 The macro then parses the :jsxmacro:`Trans` macro children and generates
@@ -123,7 +123,7 @@ Let's compare it with `react-intl`_ solution to see the difference:
    .. code-block:: html
 
       <p>
-         <Trans>Read the <a href="/docs>documentation</a>.</Trans>
+         <Trans>Read the <a href="/docs">documentation</a>.</Trans>
       </p>
 
    The message ID is ``Read the <0>documentation</0>.`` instead of ``msg.id``. Both
diff --git a/docs/ref/conf.rst b/docs/ref/conf.rst
index 9403d981..0120aebf 100644
--- a/docs/ref/conf.rst
+++ b/docs/ref/conf.rst
@@ -29,6 +29,7 @@ Default config:
     "fallbackLocales": {},
     "format": "po",
     "locales": [],
+    "extractors": ["babel"],
     "orderBy": "messageId",
     "pseudoLocale": "",
     "rootDir": ".",
@@ -506,3 +507,26 @@ providing custom translation.
 The difference between :conf:`fallbackLocales` and :conf:`sourceLocale` is that
 :conf:`fallbackLocales` is used in translation, while :conf:`sourceLocale` is
 used for the message ID.
+
+extractors
+------------
+
+Default: ``[babel]``
+
+Extractors it's the way to customize which extractor you want for your codebase, a long time ago Babel wasn't ready yet to work with Typescript,
+so we added two extractors as default ``[babel, typescript]``, but right now Babel already works good with Typescript so isn't a requirement anymore to compile two times the same code.
+
+Anyway, if you want to use the typescript extractor in conjuntion with babel you can do:
+
+.. code-block:: js
+
+   {
+      "extractors": [
+         require.resolve("@lingui/cli/api/extractors/babel"),
+         require.resolve("@lingui/cli/api/extractors/typescript"),
+      ]
+   }
+
+Of course you can build your own extractor, take a look to babel and typescript extractors to see how you should do it, but basically exports two methods:
+ - match: regex to a filename extension, should return true|false
+ - extract: is the responsible of transforming the code and using @lingui/babel-plugin-extract-messages
diff --git a/packages/cli/src/api/catalog.ts b/packages/cli/src/api/catalog.ts
index 7bd24973..f6b98f5d 100644
--- a/packages/cli/src/api/catalog.ts
+++ b/packages/cli/src/api/catalog.ts
@@ -155,6 +155,8 @@ export class Catalog {
         extract(filename, tmpDir, {
           verbose: options.verbose,
           babelOptions: this.config.extractBabelOptions,
+          // @ts-ignore
+          extractors: options.extractors,
           projectType: options.projectType,
         })
       )
diff --git a/packages/cli/src/api/extract.ts b/packages/cli/src/api/extract.ts
index 38012608..63d02935 100644
--- a/packages/cli/src/api/extract.ts
+++ b/packages/cli/src/api/extract.ts
@@ -6,13 +6,12 @@ import * as R from "ramda"
 
 import { prettyOrigin } from "./utils"
 
-import * as extractors from "./extractors"
-import { ExtractorType } from "./extractors"
-
+import cliExtractor, { ExtractorType } from "./extractors"
 
 type ExtractOptions = {
   ignore?: Array<string>
   verbose?: boolean
+  extractors?: ExtractorType[]
   projectType?: string
   babelOptions?: Object
 }
@@ -44,7 +43,7 @@ export function extract(
   targetPath: string,
   options: ExtractOptions = {}
 ) {
-  const { ignore = [], verbose = false } = options
+  const { ignore = [] } = options
   const ignorePattern = ignore.length ? new RegExp(ignore.join("|"), "i") : null
 
   srcPaths.forEach((srcFilename) => {
@@ -64,17 +63,7 @@ export function extract(
       return
     }
 
-    R.values(extractors).some((ext: ExtractorType) => {
-      if (!ext.match || !ext.match(srcFilename)) return false
-
-      let spinner
-      if (verbose) spinner = ora().start(srcFilename)
-
-      ext.extract(srcFilename, targetPath, options)
-      if (verbose && spinner) spinner.succeed()
-
-      return true
-    })
+    cliExtractor(srcFilename, targetPath, options)
   })
 }
 
diff --git a/packages/cli/src/api/extractors/index.ts b/packages/cli/src/api/extractors/index.ts
index 76c03d34..cb5d23b0 100644
--- a/packages/cli/src/api/extractors/index.ts
+++ b/packages/cli/src/api/extractors/index.ts
@@ -1,10 +1,7 @@
 import ora from "ora"
 import babel from "./babel"
-import typescript from "./typescript"
-import * as R from "ramda"
-
-const extractors = { babel, typescript }
 
+const DEFAULT_EXTRACTORS: ExtractorType[] = [babel]
 
 export type BabelOptions = {
   plugins?: Array<string>
@@ -14,6 +11,7 @@ export type BabelOptions = {
 export type ExtractOptions = {
   verbose?: boolean
   projectType?: string
+  extractors?: ExtractorType[]
   babelOptions?: BabelOptions
 }
 
@@ -27,7 +25,9 @@ export default function extract(
   targetPath: string,
   options: ExtractOptions
 ): boolean {
-  return R.values(extractors).some((ext: ExtractorType) => {
+  const extractorsToExtract = options.extractors ?? DEFAULT_EXTRACTORS
+
+  return extractorsToExtract.some((ext) => {
     if (!ext.match(filename)) return false
 
     let spinner
@@ -48,5 +48,3 @@ export default function extract(
     return true
   })
 }
-
-export { babel, typescript }
diff --git a/packages/cli/src/api/formats/lingui.ts b/packages/cli/src/api/formats/lingui.ts
index c06c70a7..b010d8a4 100644
--- a/packages/cli/src/api/formats/lingui.ts
+++ b/packages/cli/src/api/formats/lingui.ts
@@ -16,7 +16,10 @@ const removeOrigins = (R.map(
 const removeLineNumbers = (R.map(
   (message: ExtractedMessageType) => {
     if (message.origin) {
-      message.origin.map(originValue => originValue.pop())
+      message.origin.map(originValue => {
+        originValue.pop()
+        return originValue
+      })
     }
     return message
   }
diff --git a/packages/cli/src/api/formats/po.ts b/packages/cli/src/api/formats/po.ts
index 33f8e118..9794af5c 100644
--- a/packages/cli/src/api/formats/po.ts
+++ b/packages/cli/src/api/formats/po.ts
@@ -27,10 +27,7 @@ const serialize = (items: CatalogType, options) =>
       item.extractedComments = message.extractedComments || []
       if (options.origins !== false) {
         if (message.origin && options.lineNumbers === false) {
-          item.references = message.origin.map(msg => {
-            msg.pop()
-            return msg
-          }).map(joinOrigin)
+          item.references = message.origin.map(msg => msg.slice(0, -1)).map(joinOrigin)
         } else {
           item.references = message.origin ? message.origin.map(joinOrigin) : []
         }
diff --git a/packages/cli/src/lingui-extract.ts b/packages/cli/src/lingui-extract.ts
index 8bc186a9..b88eb9ec 100644
--- a/packages/cli/src/lingui-extract.ts
+++ b/packages/cli/src/lingui-extract.ts
@@ -42,6 +42,7 @@ export default function command(
     catalog.make({
       ...options,
       orderBy: config.orderBy,
+      extractors: config.extractors,
       projectType: detect(),
     })
 
diff --git a/packages/conf/index.d.ts b/packages/conf/index.d.ts
index 408c1520..fe47871e 100644
--- a/packages/conf/index.d.ts
+++ b/packages/conf/index.d.ts
@@ -23,6 +23,11 @@ export type DefaultLocaleObject = {
 
 export declare type FallbackLocales = LocaleObject | DefaultLocaleObject
 
+declare type ExtractorType = {
+    match(filename: string): boolean;
+    extract(filename: string, targetDir: string, options?: any): void;
+}
+
 export declare type LinguiConfig = {
     catalogs: CatalogConfig[];
     compileNamespace: "es" | "cjs" | "ts" | string;
@@ -30,6 +35,7 @@ export declare type LinguiConfig = {
     compilerBabelOptions: GeneratorOptions;
     fallbackLocales: FallbackLocales;
     format: CatalogFormat;
+    extractors?: ExtractorType[];
     prevFormat: CatalogFormat;
     formatOptions: CatalogFormatOptions;
     localeDir: string;
diff --git a/packages/conf/src/index.ts b/packages/conf/src/index.ts
index 688295ea..f57ff115 100644
--- a/packages/conf/src/index.ts
+++ b/packages/conf/src/index.ts
@@ -32,12 +32,18 @@ export type FallbackLocales = LocaleObject | DefaultLocaleObject | false
 
 type ModuleSource = [string, string?]
 
+type ExtractorType = {
+  match(filename: string): boolean
+  extract(filename: string, targetDir: string, options?: any): void
+}
+
 export type LinguiConfig = {
   catalogs: CatalogConfig[]
   compileNamespace: "es" | "ts" | "cjs" | string
   extractBabelOptions: Record<string, unknown>
   compilerBabelOptions: GeneratorOptions
   fallbackLocales?: FallbackLocales
+  extractors?: ExtractorType[]
   format: CatalogFormat
   formatOptions: CatalogFormatOptions
   locales: string[]
diff --git a/packages/core/npm/esm/dev.js b/packages/core/npm/esm/dev.js
new file mode 100644
index 00000000..988a5a2f
--- /dev/null
+++ b/packages/core/npm/esm/dev.js
@@ -0,0 +1,18 @@
+import {
+  i18n as i18nProd,
+  setupI18n as setupI18nProd,
+  formats as formatsProd,
+  I18n as I18nProd
+} from './dev.production.min';
+
+import {
+  i18n as i18nDev,
+  setupI18n as setupI18nDev,
+  formats as formatsDev,
+  I18n as I18nDev
+} from './dev.development';
+
+export const i18n = process.env.NODE_ENV === 'production' ? i18nProd : i18nDev;
+export const setupI18n = process.env.NODE_ENV === 'production' ? setupI18nProd : setupI18nDev;
+export const formats = process.env.NODE_ENV === 'production' ? formatsProd : formatsDev;
+export const I18n = process.env.NODE_ENV === 'production' ? I18nProd : I18nDev;
\ No newline at end of file
diff --git a/packages/core/npm/esm/index.js b/packages/core/npm/esm/index.js
index 3b280003..01cf71e7 100644
--- a/packages/core/npm/esm/index.js
+++ b/packages/core/npm/esm/index.js
@@ -3,14 +3,14 @@ import {
     setupI18n as setupI18nProd,
     formats as formatsProd,
     I18n as I18nProd
-} from './core.esm';
+} from './core.production.min';
 
 import {
     i18n as i18nDev,
     setupI18n as setupI18nDev,
     formats as formatsDev,
     I18n as I18nDev
-} from './dev.esm';
+} from './core.development';
 
 export const i18n = process.env.NODE_ENV === 'production' ? i18nProd : i18nDev;
 export const setupI18n = process.env.NODE_ENV === 'production' ? setupI18nProd : setupI18nDev;
diff --git a/scripts/build/rollup.js b/scripts/build/rollup.js
index 71944eb0..c804bf1e 100644
--- a/scripts/build/rollup.js
+++ b/scripts/build/rollup.js
@@ -19,7 +19,6 @@ const codeFrame = require("babel-code-frame")
 const babelConfig = require("./babel.config")
 
 const Modules = require("./modules")
-const Bundles = require("./bundles")
 const Packaging = require("./packaging")
 
 const UMD_DEV = "UMD_DEV"
@@ -27,6 +26,7 @@ const UMD_PROD = "UMD_PROD"
 const NODE_DEV = "NODE_DEV"
 const NODE_PROD = "NODE_PROD"
 const ESM_PROD = "ESM_PROD"
+const ESM_DEV = "ESM_DEV"
 
 const extensions = [".js", ".ts", ".tsx"]
 
@@ -71,6 +71,7 @@ function getFormat(bundleType) {
     case NODE_PROD:
       return 'cjs'
     case ESM_PROD:
+    case ESM_DEV:
       return 'es'
   }
 }
@@ -79,13 +80,13 @@ function getFilename(bundle, bundleType) {
   const filename = bundle.label || bundle.entry
   switch (bundleType) {
     case NODE_DEV:
+    case ESM_DEV:
     case UMD_DEV:
       return `${filename}.development.js`
     case UMD_PROD:
     case NODE_PROD:
-      return `${filename}.production.min.js`
     case ESM_PROD:
-      return `${filename}.esm.js`
+      return `${filename}.production.min.js`
   }
 }
 
@@ -93,6 +94,7 @@ function isProductionBundleType(bundleType) {
   switch (bundleType) {
     case UMD_DEV:
     case NODE_DEV:
+    case ESM_DEV:
       return false
     case UMD_PROD:
     case NODE_PROD:
@@ -291,7 +293,7 @@ async function build(bundle, bundleType) {
     ),
   }
   const [mainOutputPath, ...otherOutputPaths] = Packaging.getBundleOutputPaths(
-    bundleType === ESM_PROD ? "ESM" : "UNIVERSAL",
+    bundleType === ESM_PROD || bundleType === ESM_DEV ? "ESM" : "UNIVERSAL",
     filename,
     packageName
   )
@@ -320,6 +322,7 @@ async function build(bundle, bundleType) {
 module.exports = async function (bundle) {
   await build(bundle, NODE_DEV)
   await build(bundle, NODE_PROD)
+  await build(bundle, ESM_DEV)
   await build(bundle, ESM_PROD)
   // await build(bundle, UMD_DEV)
   // await build(bundle, UMD_PROD)


Test patch:
diff --git a/packages/cli/src/api/extract.test.ts b/packages/cli/src/api/extract.test.ts
index f1442c74..ccc9b3dc 100644
--- a/packages/cli/src/api/extract.test.ts
+++ b/packages/cli/src/api/extract.test.ts
@@ -53,6 +53,9 @@ describe("extract", function () {
     })
   })
 
+  afterEach(() => {
+    jest.clearAllMocks()
+  })
   afterAll(() => {
     mockFs.restore()
   })
@@ -60,6 +63,10 @@ describe("extract", function () {
   it("should traverse directory and call extractors", function () {
     extract(["src"], "locale", {
       ignore: ["forbidden"],
+      extractors: [
+        babel,
+        typescript
+      ],
       babelOptions: {},
     })
 
@@ -92,8 +99,8 @@ describe("extract", function () {
     )
 
     const extractArgs = [
-      "locale", 
-      { babelOptions: {}, ignore: ["forbidden"]}
+      "locale",
+      { extractors: [babel, typescript], babelOptions: {}, ignore: ["forbidden"]}
     ]
     expect(babel.extract).toHaveBeenCalledWith(
       path.join("src", "components", "Babel.js"),
@@ -145,6 +152,95 @@ describe("extract", function () {
       ...extractArgs
     )
   })
+
+  it("by default the traverse directory only uses babel", function () {
+    extract(["src"], "locale", {
+      ignore: ["forbidden"],
+      babelOptions: {},
+    })
+
+    expect(typescript.match).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Typescript.ts")
+    )
+    expect(babel.match).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.js")
+    )
+    expect(babel.match).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.jsx")
+    )
+    expect(babel.match).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.es6")
+    )
+    expect(babel.match).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.es")
+    )
+    expect(babel.match).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.mjs")
+    )
+		
+    expect(babel.match).toHaveBeenCalledWith(
+			path.join("src", "index.html")
+    )
+
+    // This file is ignored
+    expect(babel.extract).not.toHaveBeenCalledWith(
+      path.join("src", "index.html")
+    )
+
+    const extractArgs = [
+      "locale",
+      { babelOptions: {}, ignore: ["forbidden"]}
+    ]
+    expect(babel.extract).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.js"),
+      ...extractArgs
+    )
+    expect(babel.extract).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.jsx"),
+      ...extractArgs
+    )
+    expect(babel.extract).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.es6"),
+      ...extractArgs
+    )
+    expect(babel.extract).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.es"),
+      ...extractArgs
+    )
+    expect(babel.extract).toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.mjs"),
+      ...extractArgs
+    )
+    expect(babel.extract).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Typescript.ts"),
+      ...extractArgs
+    )
+
+    expect(typescript.extract).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.js"),
+      ...extractArgs
+    )
+    expect(typescript.extract).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.jsx"),
+      ...extractArgs
+    )
+    expect(typescript.extract).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.es6"),
+      ...extractArgs
+    )
+    expect(typescript.extract).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.es"),
+      ...extractArgs
+    )
+    expect(typescript.extract).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Babel.mjs"),
+      ...extractArgs
+    )
+    expect(typescript.extract).not.toHaveBeenCalledWith(
+      path.join("src", "components", "Typescript.ts"),
+      ...extractArgs
+    )
+  })
 })
 
 describe("collect", function () {