Repository: atom/atom
File: spec/tree-sitter-language-mode-spec.js (JavaScript)

Code before:
        buffer.setText('<% // js comment%>\n<% b() %>')
        const languageMode = new TreeSitterLanguageMode({
          buffer,
          grammar: ejsGrammar,
          grammars: atom.grammars
        })
        buffer.setLanguageMode(languageMode)

        expectTokensToEqual(editor, [
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' ', scopes: [] },
            { text: '// js comment ', scopes: ['comment'] },
            { text: '%>', scopes: ['directive'] }
          ],
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' ', scopes: [] },
            { text: 'b', scopes: ['function'] },
            { text: '() ', scopes: [] },
            { text: '%>', scopes: ['directive'] }
          ]
        ])
      })

      it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
        const promise = new Promise(resolve => {
          editor.onDidTokenize(event => {
            expectTokensToEqual(editor, [
              [
                { text: '<', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ],
              [
                { text: 'hello', scopes: ['html', 'function'] },
                { text: '();', scopes: ['html'] }
              ],
              [
                { text: '</', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ]
            ])
            resolve()
          })
        })

        atom.grammars.addGrammar(jsGrammar)
        atom.grammars.addGrammar(htmlGrammar)
        buffer.setText('<script>\nhello();\n</script>')

Review comment (bug):
@maxbrunsfeld: This test borrows heavily from [the preceding test](https://github.com/atom/atom/blob/c163564a9e97820d82b66a4934fbb52999a9ecd0/spec/tree-sitter-language-mode-spec.js#L690-L767), and attempts to cover the updates introduced in ef94657438f4b3a70f0e0a7cc7895939626f1c72.

I would expect to end up with the tokens seen above, but I'm actually ending up with a slightly different set of tokens:

```javascript
[
  { text: '<%', scopes: ['directive'] },
  { text: ' // js comment', scopes: [] },
  { text: '%>', scopes: ['directive'] },
  { text: '', scopes: ['html'] }
],
[
  { text: '<%', scopes: ['directive'] },
  { text: ' ', scopes: [] },
  { text: 'b', scopes: ['function'] },
  { text: '() ', scopes: [] },
  { text: '%>', scopes: ['directive'] }
]
```

A couple questions for you:

- In general, how does this approach compare to what you had in mind for testing the updates introduced in ef94657438f4b3a70f0e0a7cc7895939626f1c72?
- If this approach is consistent with what you had in mind, do you happen to see anything that I might be doing wrong, so that I can understand why I'm getting different results than I'm expecting? I'm wondering if there's some change I need to make to the way the EJS grammar gets set up around [here](https://github.com/atom/atom/blob/c163564a9e97820d82b66a4934fbb52999a9ecd0/spec/tree-sitter-language-mode-spec.js#L776-L779).

(By the way, if it's easier to push changes to this branch than to suggest them via prose, feel free to push directly to this pull request. I'll study any changes you push up to make sure I understand what needed to change and why.)

Code after:

        buffer.setText('<% // js comment %>\n<% b() %>')
        const languageMode = new TreeSitterLanguageMode({
          buffer,
          grammar: ejsGrammar,
          grammars: atom.grammars
        })
        buffer.setLanguageMode(languageMode)

        expectTokensToEqual(editor, [
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' ', scopes: [] },
            { text: '// js comment ', scopes: ['comment'] },
            { text: '%>', scopes: ['directive'] },
            { text: '', scopes: ['html'] }
          ],
          [
            { text: '<%', scopes: ['directive'] },
            { text: ' ', scopes: [] },
            { text: 'b', scopes: ['function'] },
            { text: '() ', scopes: [] },
            { text: '%>', scopes: ['directive'] }
          ]
        ])
      })

      it('notifies onDidTokenize listeners the first time all syntax highlighting is done', async () => {
        const promise = new Promise(resolve => {
          editor.onDidTokenize(event => {
            expectTokensToEqual(editor, [
              [
                { text: '<', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ],
              [
                { text: 'hello', scopes: ['html', 'function'] },
                { text: '();', scopes: ['html'] }
              ],
              [
                { text: '</', scopes: ['html'] },
                { text: 'script', scopes: ['html', 'tag'] },
                { text: '>', scopes: ['html'] }
              ]
            ])
            resolve()
          })
        })

        atom.grammars.addGrammar(jsGrammar)