Repository: atom/atom
File: spec/tree-sitter-language-mode-spec.js (JavaScript)

Code before:

        buffer.setText('<% // js comment %>\n<% b() %>')
        const languageMode = new TreeSitterLanguageMode({
          buffer,
          grammar: ejsGrammar,
          grammars: atom.grammars
        })
        buffer.setLanguageMode(languageMode)

        expectTokensToEqual(editor, [
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' // js comment ', scopes: [] },
            { text: '%>', scopes: ['directive'] },
            { text: '', scopes: ['html'] }
          ],
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' ', scopes: [] },
            { text: 'b', scopes: ['function'] },
            { text: '() ', scopes: [] },
            { text: '%>', scopes: ['directive'] }
          ]
        ])
      })

      it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
        const promise = new Promise(resolve => {
          editor.onDidTokenize(event => {
            expectTokensToEqual(editor, [
              [
                { text: '<', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ],
              [
                { text: 'hello', scopes: ['html', 'function'] },
                { text: '();', scopes: ['html'] }
              ],
              [
                { text: '</', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ]
            ])
            resolve()
          })
        })

        atom.grammars.addGrammar(jsGrammar)
        atom.grammars.addGrammar(htmlGrammar)

Review comment (suggestion):
Prior to the changes to `src/tree-sitter-language-mode.js` introduced in this pull request, Atom would produce the following tokens when executing this test:

```javascript
[
  [
    { text: '<%', scopes: ['directive'] },
    { text: ' // js comment ', scopes: [] },
    { text: '%>', scopes: ['directive'] },
    { text: '', scopes: ['html'] }
  ],
  [
    { text: '<%', scopes: ['directive'] },
    { text: ' b() ', scopes: [] },
    { text: '%>', scopes: ['directive'] }
  ]
]
```

Note that in this tokenization ☝️, `b()` is not tokenized as a function. Instead, `b()` is treated as a continuation of the comment on the preceding line:

```
<% // js comment %>
<% b() %>
```

The enhancements in this pull request (specifically ef94657438f4b3a70f0e0a7cc7895939626f1c72) allow Atom to correctly identify that the JavaScript comment is implicitly terminated at the end of the first line, and the second line begins a new JavaScript statement.


Code after:
        atom.grammars.addGrammar(htmlGrammar)

        buffer.setText('<% // js comment %>\n<% b() %>')
        const languageMode = new TreeSitterLanguageMode({
          buffer,
          grammar: ejsGrammar,
          grammars: atom.grammars
        })
        buffer.setLanguageMode(languageMode)

        expectTokensToEqual(editor, [
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' ', scopes: [] },
            { text: '// js comment ', scopes: ['comment'] },
            { text: '%>', scopes: ['directive'] },
            { text: '', scopes: ['html'] }
          ],
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' ', scopes: [] },
            { text: 'b', scopes: ['function'] },
            { text: '() ', scopes: [] },
            { text: '%>', scopes: ['directive'] }
          ]
        ])
      })

      it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
        const promise = new Promise(resolve => {
          editor.onDidTokenize(event => {
            expectTokensToEqual(editor, [
              [
                { text: '<', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ],
              [
                { text: 'hello', scopes: ['html', 'function'] },
                { text: '();', scopes: ['html'] }
              ],
              [
                { text: '</', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ]
            ])
            resolve()
          })
        })