Repository: lingui/js-lingui (ts)

Issue:
Bug with utf chars
**Describe the bug**
My compiled messages files contains encoded UTF characters instead of simple utf

<img width="981" alt="image" src="https://user-images.githubusercontent.com/1213198/101289992-6a585600-3808-11eb-8d8e-91f3c7639437.png">

**To Reproduce**
Compile `po` file with Cyrillic characters

**Expected behavior**
Simple not encoded utf chars

- jsLingui version
```
@lingui/cli": "3.2.3",
"@lingui/macro": "3.2.3",
"@lingui/react": "3.2.3",
```
- Babel version ``@babel/core@7.12.9``


Pull request:
feat: config accepts compilerBabelOptions
Will close #902 

We won't break babel 6 users, but babel 7 could use this to modify that ASCII configuration :)

Patch:
diff --git a/docs/ref/conf.rst b/docs/ref/conf.rst
index a027a74e..f47bf5e5 100644
--- a/docs/ref/conf.rst
+++ b/docs/ref/conf.rst
@@ -20,6 +20,7 @@ Default config:
     }],
     "compileNamespace": "cjs",
     "extractBabelOptions": {},
+    "compilerBabelOptions": {},
     "fallbackLocales": {},
     "format": "po",
     "locales": [],
@@ -230,7 +231,37 @@ extracted. This is required when project doesn't use standard Babel config
      }
    }
 
-.. config:: fallbackLocales
+.. config:: extractBabelOptions
+
+compilerBabelOptions
+-------------------
+
+Default: ``{
+   "minified": true,
+   "jsescOption": {
+      "minimal": true
+   }
+}``
+
+
+Specify extra babel options used to generate files when messages are being
+compiled. We use internaly ``@babel/generator`` that accepts some configuration for generating code with/out ASCII characters.
+These are all the options available: https://github.com/mathiasbynens/jsesc
+
+.. code-block:: json
+
+   {
+     "compilerBabelOptions": {
+       "jsescOption": {
+         "minimal": false
+      }
+     }
+   }
+
+This example configuration will compile with scaped ASCII characters. https://github.com/mathiasbynens/jsesc#minimal
+
+.. config:: compilerBabelOptions
+
 
 fallbackLocales
 --------------
diff --git a/examples/create-react-app/package.json b/examples/create-react-app/package.json
index 4f9f460f..e2aa3312 100644
--- a/examples/create-react-app/package.json
+++ b/examples/create-react-app/package.json
@@ -20,6 +20,7 @@
     "start": "yarn extract && yarn compile && react-scripts start",
     "build": "yarn extract && yarn compile && react-scripts build",
     "test": "react-scripts test",
+    "test:ci": "yarn extract && yarn compile && react-scripts test --watchAll=false",
     "eject": "react-scripts eject",
     "extract": "lingui extract",
     "compile": "lingui compile"
diff --git a/packages/cli/src/api/compile.ts b/packages/cli/src/api/compile.ts
index 36abb07e..de37bfde 100644
--- a/packages/cli/src/api/compile.ts
+++ b/packages/cli/src/api/compile.ts
@@ -15,6 +15,7 @@ export type CreateCompileCatalogOptions = {
   strict?: boolean
   namespace?: CompiledCatalogNamespace
   pseudoLocale?: string
+  compilerBabelOptions?: Record<string, unknown>
 }
 
 export function createCompiledCatalog(
@@ -22,7 +23,7 @@ export function createCompiledCatalog(
   messages: CompiledCatalogType,
   options: CreateCompileCatalogOptions
 ) {
-  const { strict = false, namespace = "cjs", pseudoLocale } = options
+  const { strict = false, namespace = "cjs", pseudoLocale, compilerBabelOptions = {} } = options
   const compiledMessages = R.keys(messages).map((key: string) => {
     // Don't use `key` as a fallback translation in strict mode.
     let translation = messages[key] || (!strict ? key : "")
@@ -41,8 +42,12 @@ export function createCompiledCatalog(
 
   return (
     "/*eslint-disable*/" +
-    generate(ast as any, {
+    generate(ast, {
       minified: true,
+      jsescOption: {
+        minimal: true,
+      },
+      ...compilerBabelOptions,
     }).code
   )
 }
diff --git a/packages/cli/src/lingui-compile.ts b/packages/cli/src/lingui-compile.ts
index f4a328ec..27057750 100644
--- a/packages/cli/src/lingui-compile.ts
+++ b/packages/cli/src/lingui-compile.ts
@@ -4,7 +4,7 @@ import * as R from "ramda"
 import program from "commander"
 import * as plurals from "make-plural"
 
-import { getConfig } from "@lingui/conf"
+import { getConfig, LinguiConfig } from "@lingui/conf"
 
 import { getCatalogs } from "./api/catalog"
 import { createCompiledCatalog } from "./api/compile"
@@ -16,7 +16,7 @@ const noMessages: (catalogs: Object[]) => boolean = R.pipe(
   R.all(R.equals<any>(true))
 )
 
-function command(config, options) {
+function command(config: LinguiConfig, options) {
   const catalogs = getCatalogs(config)
 
   if (noMessages(catalogs)) {
@@ -87,6 +87,7 @@ function command(config, options) {
           strict: false,
           namespace,
           pseudoLocale: config.pseudoLocale,
+          compilerBabelOptions: config.compilerBabelOptions,
         })
 
         const compiledPath = catalog.writeCompiled(
diff --git a/packages/conf/index.d.ts b/packages/conf/index.d.ts
index aed99cd6..02606c5c 100644
--- a/packages/conf/index.d.ts
+++ b/packages/conf/index.d.ts
@@ -23,14 +23,15 @@ export declare type FallbackLocales = LocaleObject | DefaultLocaleObject
 export declare type LinguiConfig = {
     catalogs: CatalogConfig[];
     compileNamespace: string;
-    extractBabelOptions: Object;
+    extractBabelOptions: Record<string, unknown>;
+    compilerBabelOptions: Record<string, unknown>;
     fallbackLocales: FallbackLocales;
     format: CatalogFormat;
     prevFormat: CatalogFormat;
     formatOptions: CatalogFormatOptions;
     localeDir: string;
     locales: string[];
-    catalogsMergePath?: string;
+    catalogsMergePath: string;
     orderBy: OrderBy;
     pseudoLocale: string;
     rootDir: string;
@@ -51,6 +52,7 @@ export declare const configValidation: {
             plugins: string[];
             presets: string[];
         };
+        compilerBabelOptions: Record<string, unknown>;
         catalogs: CatalogConfig[];
         compileNamespace: string;
         fallbackLocales: FallbackLocales;
diff --git a/packages/conf/src/index.ts b/packages/conf/src/index.ts
index 335f3fb0..2fdd483c 100644
--- a/packages/conf/src/index.ts
+++ b/packages/conf/src/index.ts
@@ -32,12 +32,13 @@ type ModuleSource = [string, string?];
 export type LinguiConfig = {
   catalogs: CatalogConfig[]
   compileNamespace: string
-  extractBabelOptions: Object
+  extractBabelOptions: Record<string, unknown>
+  compilerBabelOptions: Record<string, unknown>
   fallbackLocales?: FallbackLocales
   format: CatalogFormat
   formatOptions: CatalogFormatOptions
   locales: string[]
-  catalogsMergePath?: string
+  catalogsMergePath: string
   orderBy: OrderBy
   pseudoLocale: string
   rootDir: string
@@ -62,7 +63,9 @@ export const defaultConfig: LinguiConfig = {
       exclude: ["*/node_modules/*"],
     },
   ],
+  catalogsMergePath: "",
   compileNamespace: "cjs",
+  compilerBabelOptions: {},
   extractBabelOptions: { plugins: [], presets: [] },
   fallbackLocales: {},
   format: "po",
diff --git a/scripts/verdaccio-integration.js b/scripts/verdaccio-integration.js
index 1d467a5c..715e5fc6 100644
--- a/scripts/verdaccio-integration.js
+++ b/scripts/verdaccio-integration.js
@@ -18,7 +18,7 @@ async function main() {
 
   spinner.start("Running tests")
   try {
-    await exec('yarn test --watchAll=false', OPTS)
+    await exec('yarn test:ci', OPTS)
     spinner.succeed()
   } catch (error) {
     spinner.fail()


Test patch:
diff --git a/packages/cli/src/api/__snapshots__/compile.test.ts.snap b/packages/cli/src/api/__snapshots__/compile.test.ts.snap
index 5d6841d3..5ad75d26 100644
--- a/packages/cli/src/api/__snapshots__/compile.test.ts.snap
+++ b/packages/cli/src/api/__snapshots__/compile.test.ts.snap
@@ -2,6 +2,10 @@
 
 exports[`compile should report failed message on error 1`] = `Can't parse message. Please check correct syntax: "{value, plural, one {Book} other {Books"`;
 
+exports[`createCompiledCatalog options.compilerBabelOptions by default should return catalog without ASCII chars 1`] = `/*eslint-disable*/module.exports={messages:{"Hello":"Alohà"}};`;
+
+exports[`createCompiledCatalog options.compilerBabelOptions should return catalog without ASCII chars 1`] = `/*eslint-disable*/module.exports={messages:{"Hello":"Aloh\\xE0"}};`;
+
 exports[`createCompiledCatalog options.namespace should compile with es 1`] = `/*eslint-disable*/export const messages={};`;
 
 exports[`createCompiledCatalog options.namespace should compile with global 1`] = `/*eslint-disable*/global.test={messages:{}};`;
@@ -10,7 +14,7 @@ exports[`createCompiledCatalog options.namespace should compile with window 1`]
 
 exports[`createCompiledCatalog options.namespace should error with invalid value 1`] = `Invalid namespace param: "global"`;
 
-exports[`createCompiledCatalog options.pseudoLocale should return catalog with pseudolocalized messages 1`] = `/*eslint-disable*/module.exports={messages:{"Hello":"\\xC0\\u0125\\u014D\\u0134"}};`;
+exports[`createCompiledCatalog options.pseudoLocale should return catalog with pseudolocalized messages 1`] = `/*eslint-disable*/module.exports={messages:{"Hello":"ÀĥōĴ"}};`;
 
 exports[`createCompiledCatalog options.pseudoLocale should return compiled catalog when pseudoLocale doesn't match current locale 1`] = `/*eslint-disable*/module.exports={messages:{"Hello":"Ahoj"}};`;
 
diff --git a/packages/cli/src/api/compile.test.ts b/packages/cli/src/api/compile.test.ts
index 8f950cd0..6dac4faf 100644
--- a/packages/cli/src/api/compile.test.ts
+++ b/packages/cli/src/api/compile.test.ts
@@ -1,14 +1,14 @@
 import generate from "@babel/generator"
 import { compile, createCompiledCatalog } from "./compile"
 
-describe("compile", function () {
+describe("compile", () => {
   const getSource = (message) =>
     generate(compile(message) as any, {
       compact: true,
       minified: true,
     }).code
 
-  it("should optimize string only messages", function () {
+  it("should optimize string only messages", () => {
     expect(getSource("Hello World")).toEqual('"Hello World"')
   })
 
@@ -17,13 +17,13 @@ describe("compile", function () {
     expect(getSource("''")).toEqual('"\'"')
   })
 
-  it("should compile arguments", function () {
+  it("should compile arguments", () => {
     expect(getSource("{name}")).toEqual('[["name"]]')
 
     expect(getSource("B4 {name} A4")).toEqual('["B4 ",["name"]," A4"]')
   })
 
-  it("should compile arguments with formats", function () {
+  it("should compile arguments with formats", () => {
     expect(getSource("{name, number}")).toEqual('[["name","number"]]')
 
     expect(getSource("{name, number, percent}")).toEqual(
@@ -31,7 +31,7 @@ describe("compile", function () {
     )
   })
 
-  it("should compile plural", function () {
+  it("should compile plural", () => {
     expect(getSource("{name, plural, one {Book} other {Books}}")).toEqual(
       '[["name","plural",{one:"Book",other:"Books"}]]'
     )
@@ -53,7 +53,7 @@ describe("compile", function () {
     )
   })
 
-  it("should compile select", function () {
+  it("should compile select", () => {
     expect(getSource("{name, select, male {He} female {She}}")).toEqual(
       '[["name","select",{male:"He",female:"She"}]]'
     )
@@ -67,15 +67,15 @@ describe("compile", function () {
     )
   })
 
-  it("should report failed message on error", function () {
+  it("should report failed message on error", () => {
     expect(() =>
       getSource("{value, plural, one {Book} other {Books")
     ).toThrowErrorMatchingSnapshot()
   })
 })
 
-describe("createCompiledCatalog", function () {
-  describe("options.namespace", function () {
+describe("createCompiledCatalog", () => {
+  describe("options.namespace", () => {
     const getCompiledCatalog = (namespace) =>
       createCompiledCatalog(
         "fr",
@@ -85,24 +85,24 @@ describe("createCompiledCatalog", function () {
         }
       )
 
-    it("should compile with es", function () {
+    it("should compile with es", () => {
       expect(getCompiledCatalog("es")).toMatchSnapshot()
     })
 
-    it("should compile with window", function () {
+    it("should compile with window", () => {
       expect(getCompiledCatalog("window.test")).toMatchSnapshot()
     })
 
-    it("should compile with global", function () {
+    it("should compile with global", () => {
       expect(getCompiledCatalog("global.test")).toMatchSnapshot()
     })
 
-    it("should error with invalid value", function () {
+    it("should error with invalid value", () => {
       expect(() => getCompiledCatalog("global")).toThrowErrorMatchingSnapshot()
     })
   })
 
-  describe("options.strict", function () {
+  describe("options.strict", () => {
     const getCompiledCatalog = (strict) =>
       createCompiledCatalog(
         "cs",
@@ -115,16 +115,16 @@ describe("createCompiledCatalog", function () {
         }
       )
 
-    it("should return message key as a fallback translation", function () {
+    it("should return message key as a fallback translation", () => {
       expect(getCompiledCatalog(false)).toMatchSnapshot()
     })
 
-    it("should't return message key as a fallback in strict mode", function () {
+    it("should't return message key as a fallback in strict mode", () => {
       expect(getCompiledCatalog(true)).toMatchSnapshot()
     })
   })
 
-  describe("options.pseudoLocale", function () {
+  describe("options.pseudoLocale", () => {
     const getCompiledCatalog = (pseudoLocale) =>
       createCompiledCatalog(
         "ps",
@@ -136,12 +136,37 @@ describe("createCompiledCatalog", function () {
         }
       )
 
-    it("should return catalog with pseudolocalized messages", function () {
+    it("should return catalog with pseudolocalized messages", () => {
       expect(getCompiledCatalog("ps")).toMatchSnapshot()
     })
 
-    it("should return compiled catalog when pseudoLocale doesn't match current locale", function () {
+    it("should return compiled catalog when pseudoLocale doesn't match current locale", () => {
       expect(getCompiledCatalog("en")).toMatchSnapshot()
     })
   })
+
+  describe("options.compilerBabelOptions", () => {
+    const getCompiledCatalog = (opts = {}) =>
+      createCompiledCatalog(
+        "ru",
+        {
+          Hello: "Alohà",
+        },
+        opts
+      )
+
+    it("by default should return catalog without ASCII chars", () => {
+      expect(getCompiledCatalog()).toMatchSnapshot()
+    })
+
+    it("should return catalog without ASCII chars", () => {
+      expect(getCompiledCatalog({
+        compilerBabelOptions: { 
+          jsescOption: {
+            minimal: false,
+          }
+        }
+      })).toMatchSnapshot()
+    })
+  })
 })
diff --git a/packages/conf/src/__snapshots__/index.test.ts.snap b/packages/conf/src/__snapshots__/index.test.ts.snap
index e043dca7..08b1bd90 100644
--- a/packages/conf/src/__snapshots__/index.test.ts.snap
+++ b/packages/conf/src/__snapshots__/index.test.ts.snap
@@ -108,7 +108,9 @@ Object {
       path: ./locale/{locale}/messages,
     },
   ],
+  catalogsMergePath: ,
   compileNamespace: cjs,
+  compilerBabelOptions: Object {},
   extractBabelOptions: Object {
     plugins: Array [],
     presets: Array [],