Repository: i18next/i18next-parser
Pull request #223 (2020-06-26 11:12:53)

Extract translation from comment for both Javascript and Jsx Lexers
### Why am I submitting this PR
Adds support for extracting translations from comments for both JavascriptLexer and JsxLexer
Fixes https://github.com/i18next/i18next-parser/issues/166

This PR is an extension of https://github.com/i18next/i18next-parser/pull/222 and is to be merged after https://github.com/i18next/i18next-parser/pull/222. In https://github.com/i18next/i18next-parser/pull/222 the JavascriptLexer is extended to add support for extracting translations from comments. This PR extends it and adds the same support for JsxLexer.

I wanted to add comment extraction support to JsxLexer few days ago. Before seeing @vincaslt 's PR, I went with a quite similar approach to support parsing comments in JsxLexer. You can check the approach in this commit https://github.com/i18next/i18next-parser/commit/3337bb2428b480112fdbf344cc6aab0d640be5c4 if you are interested, but my approach only handles single line comment, @vincaslt 's approach is more extensive that it handles also multiline comment, so I switched to using his approach.

```js
// My initial approach
const sourceFile = ts.createSourceFile(
      filename,
      content,
      ts.ScriptTarget.Latest
    )
const comments = ts.getLeadingCommentRanges(
        sourceFile.getText(),
        node.getFullStart()
      )
 if (comments) {
     comments.map((comment) => {
         const commentEntry = sourceFile
            .getFullText()
            .slice(comment.pos, comment.end)

           if (comment.kind === ts.SyntaxKind.SingleLineCommentTrivia) {
            const regExp = / t\(([^)]+)\)/
            var matches = regExp.exec(commentEntry)
            if (matches) {
              const key = matches[1].replace(/['"\\]+/g, '')
              keys.push({ key })
            }
          }
    })
}

```
### Checklist

- [x] only relevant code is changed (make a diff before you submit the PR)
- [x] tests are included and pass for jsx-lexer.test.js


vincaslt: Good catch, I thought JS lexer would implicitly handle this for Jsx too, but I didn't bother to make sure... I guess ideally it would reuse the same logic, since it seems pretty similar. As an idea - something like a closure in JS lexer which would keep visited comments' state, that Jsx lexer can reuse:

```
  createCommentNodeParser(node) {
    const visitedComments = new Set()
    const keys = []

    return () => {
      s.forEachLeadingCommentRange(
        ...
      )
    }
  }

extract(content, filename = '__default.js') {
    const parseCommentNode = this.createCommentNodeParser.call(this, node)

    const parseTree = (node) => {
      let entry

      parseCommentNode(node)
      ...
    }
  }
```
karellm: @tienpham94 Thanks for the PR!

@vincaslt I like your suggestion, I'll let you review the code and let me know when it's ready to merge and publish.
tienpham94: @vincaslt Updated based on your comment!

Patch:
diff --git a/dist/lexers/javascript-lexer.js b/dist/lexers/javascript-lexer.js
index 4ece9f5..b03f4b3 100644
--- a/dist/lexers/javascript-lexer.js
+++ b/dist/lexers/javascript-lexer.js
@@ -8,15 +8,12 @@ JavascriptLexer = function (_BaseLexer) {_inherits(JavascriptLexer, _BaseLexer);
     _this.callPattern = '(?<=^|\\s|\\.)' + _this.functionPattern() + '\\(.*\\)';
     _this.functions = options.functions || ['t'];
     _this.attr = options.attr || 'i18nKey';return _this;
-  }_createClass(JavascriptLexer, [{ key: 'extract', value: function extract(
+  }_createClass(JavascriptLexer, [{ key: 'createCommentNodeParser', value: function createCommentNodeParser()
 
-    content) {var _this2 = this;var filename = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : '__default.js';
+    {var _this2 = this;
       var visitedComments = new Set();
-      var keys = [];
-
-      var parseTree = function parseTree(node) {
-        var entry = void 0;
 
+      return function (keys, node, content) {
         ts.forEachLeadingCommentRange(
         content,
         node.getFullStart(),
@@ -36,9 +33,21 @@ JavascriptLexer = function (_BaseLexer) {_inherits(JavascriptLexer, _BaseLexer);
           }
         });
 
+      };
+    } }, { key: 'extract', value: function extract(
+
+    content) {var _this3 = this;var filename = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : '__default.js';
+      var keys = [];
+
+      var parseCommentNode = this.createCommentNodeParser();
+
+      var parseTree = function parseTree(node) {
+        var entry = void 0;
+
+        parseCommentNode(keys, node, content);
 
         if (node.kind === ts.SyntaxKind.CallExpression) {
-          entry = _this2.expressionExtractor.call(_this2, node);
+          entry = _this3.expressionExtractor.call(_this3, node);
         }
 
         if (entry) {
@@ -149,7 +158,7 @@ JavascriptLexer = function (_BaseLexer) {_inherits(JavascriptLexer, _BaseLexer);
       return null;
     } }, { key: 'commentExtractor', value: function commentExtractor(
 
-    commentText) {var _this3 = this;
+    commentText) {var _this4 = this;
       var regexp = new RegExp(this.callPattern, 'g');
       var expressions = commentText.match(regexp);
 
@@ -159,7 +168,7 @@ JavascriptLexer = function (_BaseLexer) {_inherits(JavascriptLexer, _BaseLexer);
 
       var keys = [];
       expressions.forEach(function (expression) {
-        var expressionKeys = _this3.extract(expression);
+        var expressionKeys = _this4.extract(expression);
         if (expressionKeys) {
           keys.push.apply(keys, _toConsumableArray(expressionKeys));
         }
diff --git a/dist/lexers/jsx-lexer.js b/dist/lexers/jsx-lexer.js
index ec3c44c..5dc5c0d 100644
--- a/dist/lexers/jsx-lexer.js
+++ b/dist/lexers/jsx-lexer.js
@@ -18,9 +18,13 @@ JsxLexer = function (_JavascriptLexer) {_inherits(JsxLexer, _JavascriptLexer);
     content) {var _this2 = this;var filename = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : '__default.jsx';
       var keys = [];
 
+      var parseCommentNode = this.createCommentNodeParser();
+
       var parseTree = function parseTree(node) {
         var entry = void 0;
 
+        parseCommentNode(keys, node, content);
+
         switch (node.kind) {
           case ts.SyntaxKind.CallExpression:
             entry = _this2.expressionExtractor.call(_this2, node);
diff --git a/src/lexers/javascript-lexer.js b/src/lexers/javascript-lexer.js
index f92d81b..1a7595a 100644
--- a/src/lexers/javascript-lexer.js
+++ b/src/lexers/javascript-lexer.js
@@ -10,13 +10,10 @@ export default class JavascriptLexer extends BaseLexer {
     this.attr = options.attr || 'i18nKey'
   }
 
-  extract(content, filename = '__default.js') {
+  createCommentNodeParser() {
     const visitedComments = new Set()
-    const keys = []
-
-    const parseTree = (node) => {
-      let entry
 
+    return (keys, node, content) => {
       ts.forEachLeadingCommentRange(
         content,
         node.getFullStart(),
@@ -36,6 +33,18 @@ export default class JavascriptLexer extends BaseLexer {
           }
         }
       )
+    }
+  }
+
+  extract(content, filename = '__default.js') {
+    const keys = []
+
+    const parseCommentNode = this.createCommentNodeParser()
+
+    const parseTree = (node) => {
+      let entry
+
+      parseCommentNode(keys, node, content)
 
       if (node.kind === ts.SyntaxKind.CallExpression) {
         entry = this.expressionExtractor.call(this, node)
diff --git a/src/lexers/jsx-lexer.js b/src/lexers/jsx-lexer.js
index 4fdaa37..1ffc28a 100644
--- a/src/lexers/jsx-lexer.js
+++ b/src/lexers/jsx-lexer.js
@@ -18,9 +18,13 @@ export default class JsxLexer extends JavascriptLexer {
   extract(content, filename = '__default.jsx') {
     const keys = []
 
+    const parseCommentNode = this.createCommentNodeParser()
+
     const parseTree = (node) => {
       let entry
 
+      parseCommentNode(keys, node, content)
+
       switch (node.kind) {
         case ts.SyntaxKind.CallExpression:
           entry = this.expressionExtractor.call(this, node)


Test patch:
diff --git a/test/lexers/jsx-lexer.test.js b/test/lexers/jsx-lexer.test.js
index 9055ebd..a52ba87 100644
--- a/test/lexers/jsx-lexer.test.js
+++ b/test/lexers/jsx-lexer.test.js
@@ -1,5 +1,4 @@
-import { assert, expect } from 'chai'
-import { spy } from 'sinon'
+import { assert } from 'chai'
 import JsxLexer from '../../src/lexers/jsx-lexer'
 
 describe('JsxLexer', () => {
@@ -76,6 +75,43 @@ describe('JsxLexer', () => {
       done()
     })
 
+    it('extracts keys from single line comments', (done) => {
+      const Lexer = new JsxLexer()
+      const content = `
+      // i18n.t('commentKey1')
+      i18n.t('commentKey' + i)
+      // i18n.t('commentKey2')
+      i18n.t(\`commentKey\${i}\`)
+      // Irrelevant comment
+      // i18n.t('commentKey3')
+      `
+      assert.deepEqual(Lexer.extract(content), [
+        { key: 'commentKey1' },
+        { key: 'commentKey2' },
+        { key: 'commentKey3' },
+      ])
+      done()
+    })
+
+    it('extracts keys from multiline comments', (done) => {
+      const Lexer = new JsxLexer()
+      const content = `
+      /*
+        i18n.t('commentKey1')
+        i18n.t('commentKey2')
+      */
+      i18n.t(\`commentKey\${i}\`)
+      // Irrelevant comment
+      /* i18n.t('commentKey3') */
+      `
+      assert.deepEqual(Lexer.extract(content), [
+        { key: 'commentKey1' },
+        { key: 'commentKey2' },
+        { key: 'commentKey3' },
+      ])
+      done()
+    })
+
     it('invalid interpolation gets stripped', (done) => {
       const Lexer = new JsxLexer()
       const content = '<Trans count={count}>before{{ key1, key2 }}after</Trans>'