Repository: vscode-neovim/vscode-neovim
Pull request #1904 (2024-04-24 13:19:47)

refactor: remove use of nvim_call_atomic

nvim_call_atomic is deprecated since nvim_exec_lua can be used to batch calls and handle errors.

btw, i get this when i run `npm run lint` locally, now

```
> vscode-neovim@1.10.0 format
> pre-commit run --all prettier && pre-commit run --all stylua-github
sh: pre-commit: command not found
```

even after running `npm install -g pre-commit`  (which, really, I shouldn't need to do)

justinmk: CI not triggering...

Patch:
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index a0f9801..373b0f8 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -138,8 +138,7 @@ VSCode-Neovim uses neovim's [`node-client`](https://neovim.io/node-client/index.
 neovim. This library is a wrapper around neovim's [msgpack-rpc API](https://neovim.io/doc/user/api.html). The library is
 used to send and receive messages to and from neovim. The library is also used to spawn neovim processes. Many functions
 are exposed, listed [here](https://neovim.io/node-client/classes/NeovimClient.html). However, the library is not kept up
-to date, so many API wrappers are missing. In these cases, the raw `call`, `callAtomic`, `lua`, and `command` commands
-are directly used.
+to date, so many API wrappers are missing. In those cases just use `call`, `lua`, and `command` instead.
 
 ### Neovim UI Protocol
 
@@ -194,7 +193,7 @@ editor with nvim, and roughly matches the nvim ui API. Managers can do the follo
 
 -   Listen to ui and custom events from nvim through `eventBus.on` (`src/eventBus.ts`).
 -   Listen to vscode events such as `window.onDidChangeTextEditorSelection` and `workspace.onDidChangeTextDocument`.
--   Access the lua API through `this.client.call` and `callAtomic`.
+-   Access the lua API through `this.client.call` and `this.client.lua`.
 
 ### ModeManager
 
diff --git a/package.json b/package.json
index 36b210c..cc1d584 100644
--- a/package.json
+++ b/package.json
@@ -2090,7 +2090,7 @@
         "format": "pre-commit run --all prettier && pre-commit run --all stylua-github",
         "keybind": "node ./scripts/keybindings/index.cjs",
         "test": "npm run test:unit && npm run test:integration",
-        "test:unit": "mocha ./out/test/unit/**/*.js",
+        "test:unit": "mocha ./out/test/unit/*.js ./out/test/unit/**/*.js",
         "test:integration": "node ./out/test/runIntegrationTest.js",
         "pretest:unit": "npm run test-compile",
         "pretest:integration": "npm run test-compile",
diff --git a/src/buffer_manager.ts b/src/buffer_manager.ts
index a15524f..b8ec763 100644
--- a/src/buffer_manager.ts
+++ b/src/buffer_manager.ts
@@ -29,7 +29,7 @@ import { config } from "./config";
 import { EventBusData, eventBus } from "./eventBus";
 import { LogLevel, createLogger } from "./logger";
 import { MainController } from "./main_controller";
-import { ManualPromise, callAtomic, convertByteNumToCharNum, disposeAll, wait } from "./utils";
+import { ManualPromise, convertByteNumToCharNum, disposeAll, wait } from "./utils";
 
 // NOTE: document and editors in vscode events and namespace are reference stable
 // Integration notes:
@@ -602,22 +602,34 @@ export class BufferManager implements Disposable {
                     : docUri.fsPath
                 : docUri.toString();
 
-        const requests: [string, unknown[]][] = [
-            ["nvim_buf_set_lines", [bufId, 0, -1, false, lines]],
-            // set vscode controlled flag so we can check it neovim
-            ["nvim_buf_set_var", [bufId, "vscode_controlled", true]],
-            // In vscode same document can have different insertSpaces/tabSize settings per editor
-            // however in neovim it's per buffer. We make assumption here that these settings are same for all editors
-            ["nvim_buf_set_var", [bufId, "vscode_editor_options", makeEditorOptionsVariable(editor?.options)]],
-            ["nvim_buf_set_var", [bufId, "vscode_uri", docUri.toString()]],
-            ["nvim_buf_set_var", [bufId, "vscode_uri_data", docUri.toJSON()]],
-            ["nvim_buf_set_name", [bufId, bufname]],
-            ["nvim_buf_set_option", [bufId, "modifiable", !this.isExternalTextDocument(document)]],
-            // force nofile, just in case if the buffer was created externally
-            ["nvim_buf_set_option", [bufId, "buftype", "nofile"]],
-            ["nvim_buf_set_option", [bufId, "buflisted", true]],
-        ];
-        await callAtomic(this.client, requests, logger);
+        await this.client.lua(
+            `
+            local bufId, lines, vscode_editor_options, docUri, docUriJson, bufname, isExternalDoc = ...
+            vim.api.nvim_buf_set_lines(bufId, 0, -1, false, lines)
+            -- set vscode controlled flag so we can check it neovim
+            vim.api.nvim_buf_set_var(bufId, "vscode_controlled", true)
+            -- In vscode same document can have different insertSpaces/tabSize settings per editor
+            -- however in neovim it's per buffer. We make assumption here that these settings are same for all editors
+            vim.api.nvim_buf_set_var(bufId, "vscode_editor_options", vscode_editor_options)
+            vim.api.nvim_buf_set_var(bufId, "vscode_uri", docUri)
+            vim.api.nvim_buf_set_var(bufId, "vscode_uri_data", docUriJson)
+            vim.api.nvim_buf_set_name(bufId, bufname)
+            vim.api.nvim_buf_set_option(bufId, "modifiable", not isExternalDoc)
+            -- force nofile, just in case if the buffer was created externally
+            vim.api.nvim_buf_set_option(bufId, "buftype", "nofile")
+            vim.api.nvim_buf_set_option(bufId, "buflisted", true)
+        `,
+            [
+                bufId,
+                lines,
+                makeEditorOptionsVariable(editor?.options),
+                docUri.toString(),
+                docUri.toJSON(),
+                bufname,
+                this.isExternalTextDocument(document),
+            ],
+        );
+
         // Looks like need to be in separate request
         if (!this.isExternalTextDocument(document)) {
             await this.client.callFunction("VSCodeClearUndo", bufId);
diff --git a/src/highlight_manager.ts b/src/highlight_manager.ts
index d6fab67..68405f0 100644
--- a/src/highlight_manager.ts
+++ b/src/highlight_manager.ts
@@ -1,7 +1,7 @@
 import { Disposable, TextEditor } from "vscode";
 
 import { EventBusData, eventBus } from "./eventBus";
-import { HighlightProvider } from "./highlight_provider";
+import { HighlightProvider } from "./highlights/highlight_provider";
 import { MainController } from "./main_controller";
 import { disposeAll } from "./utils";
 import { PendingUpdates } from "./pending_updates";
diff --git a/src/highlights/highlight_grid.ts b/src/highlights/highlight_grid.ts
new file mode 100644
index 0000000..3a0bf3f
--- /dev/null
+++ b/src/highlights/highlight_grid.ts
@@ -0,0 +1,332 @@
+import wcswidth from "ts-wcwidth";
+import GraphemeSplitter from "grapheme-splitter";
+
+import { calculateEditorColFromVimScreenCol, expandTabs } from "../utils/text";
+
+export interface ValidCell {
+    text: string;
+    hlId: number;
+}
+
+export interface Highlight extends ValidCell {
+    virtText?: string;
+}
+
+export type HighlightRange = NormalTextHighlightRange | VirtualTextHighlightRange;
+
+export interface NormalTextHighlightRange {
+    textType: "normal";
+    hlId: number;
+    line: number;
+    startCol: number;
+    endCol: number;
+}
+
+export interface VirtualTextHighlightRange {
+    textType: "virtual";
+    highlights: Highlight[];
+    line: number;
+    col: number;
+}
+
+export interface HighlightCellsEvent {
+    row: number;
+    vimCol: number;
+    validCells: ValidCell[];
+    lineText: string;
+    tabSize: number;
+}
+
+export class HighlightGrid {
+    /**
+     * a three-dimensional array representing rows and columns.
+     * Each column can contain multiple highlights. e.g. double-width character, tab
+     */
+    private grid: Highlight[][][];
+
+    private static splitter = new GraphemeSplitter();
+
+    constructor() {
+        this.grid = [];
+    }
+
+    /**
+     * Checks whether or not the contents of a text cell should actually span two cells. For instance:
+     * a  length:1 width:1
+     * 你 length:1 width:2
+     * 🚀 length:2 width:2
+     * 🕵️ length:3 width:2
+     * ❤️ length:2 width:2
+     *
+     * @param cellText The cell text to check
+     * @returns Whether or not this cell is a double-width cell
+     */
+    static isDouble(cellText: string): boolean {
+        return wcswidth(cellText) === 2 || cellText.length > 1;
+    }
+
+    /**
+     * Get the width of a piece of text, in cells. Note that the use of tabs is
+     * position dependent, and this function must assume a line with tabs starts
+     * at position 0in the editor.
+     *
+     * @param text The text to get the width of
+     * @param tabSize The size of tab-widths
+     * @returns The width of the text, in cells
+     */
+    static getWidth(text: string, tabSize: number): number {
+        const t = expandTabs(text, tabSize);
+        return HighlightGrid.splitGraphemes(t).reduce((p, c) => p + (HighlightGrid.isDouble(c) ? 2 : 1), 0);
+    }
+
+    static splitGraphemes(str: string): string[] {
+        return HighlightGrid.splitter.splitGraphemes(str);
+    }
+
+    cleanRow(row: number) {
+        delete this.grid[row];
+    }
+
+    processHighlightCellsEvent({ row, vimCol, validCells, lineText, tabSize }: HighlightCellsEvent): boolean {
+        let hasUpdates = false;
+
+        if (!this.grid[row]) {
+            this.grid[row] = [];
+        }
+
+        const gridRow = this.grid[row];
+        const lineChars = HighlightGrid.splitGraphemes(lineText);
+
+        // Calculates the number of spaces occupied by the tab
+        const calcTabCells = (tabCol: number) => {
+            let nearestTabIdx = lineChars.slice(0, tabCol).lastIndexOf("\t");
+            nearestTabIdx = nearestTabIdx === -1 ? 0 : nearestTabIdx + 1;
+            const center = lineChars.slice(nearestTabIdx, tabCol).join("");
+            return tabSize - (HighlightGrid.getWidth(center, tabSize) % tabSize);
+        };
+
+        const editorCol = calculateEditorColFromVimScreenCol(lineText, vimCol, tabSize);
+        const cellIter = new CellIter(validCells);
+
+        // #region
+        // If the previous column can contain multiple cells,
+        // then the redraw cells may contain cells from the previous column.
+        if (editorCol > 0) {
+            const prevCol = editorCol - 1;
+            const prevChar: string | undefined = lineChars[prevCol];
+            const expectedCells =
+                prevChar === "\t" ? calcTabCells(prevCol) : HighlightGrid.getWidth(prevChar ?? "", tabSize);
+            if (expectedCells > 1) {
+                const expectedVimCol = HighlightGrid.getWidth(lineChars.slice(0, editorCol).join(""), tabSize);
+                if (expectedVimCol > vimCol) {
+                    const rightHls: Highlight[] = [];
+                    for (let i = 0; i < expectedVimCol - vimCol; i++) {
+                        const cell = cellIter.next();
+                        cell && rightHls.push({ ...cell, virtText: cell.text });
+                    }
+                    const leftHls: Highlight[] = [];
+                    if (expectedCells - rightHls.length) {
+                        leftHls.push(...(gridRow[prevCol] ?? []).slice(0, expectedCells - rightHls.length));
+                    }
+                    gridRow[prevCol] = [...leftHls, ...rightHls];
+                    hasUpdates = true;
+                }
+            }
+        }
+        // #endregion
+
+        // Insert additional columns for characters with length greater than 1.
+        const filledLineText = HighlightGrid.splitGraphemes(lineText).reduce(
+            (p, c) => p + c + " ".repeat(c.length - 1),
+            "",
+        );
+
+        const filledLineChars = HighlightGrid.splitGraphemes(filledLineText);
+        let currCharCol = editorCol;
+        let cell = cellIter.next();
+        while (cell) {
+            const hls: Highlight[] = [];
+            const add = (cell: ValidCell, virtText?: string) => hls.push({ ...cell, virtText });
+            const currChar = filledLineChars[currCharCol];
+            const extraCols = currChar ? currChar.length - 1 : 0;
+            currCharCol += extraCols;
+            // ... some emojis have text versions e.g. [..."❤️"] == ['❤', '️']
+            const hlCol = currCharCol - (currChar ? [...currChar].length - 1 : 0);
+
+            do {
+                if (currChar === "\t") {
+                    add(cell, cell.text);
+                    for (let i = 0; i < calcTabCells(currCharCol) - 1; i++) {
+                        cell = cellIter.next();
+                        cell && add(cell, cell.text);
+                    }
+
+                    break;
+                }
+
+                if (currChar && HighlightGrid.isDouble(currChar)) {
+                    if (currChar === cell.text) {
+                        add(cell);
+                        break;
+                    }
+
+                    add(cell, cell.text);
+                    if (!HighlightGrid.isDouble(cell.text)) {
+                        const nextCell = cellIter.next();
+                        nextCell && add(nextCell, nextCell.text);
+                        extraCols && add(nextCell ?? cell, " ".repeat(extraCols));
+                    }
+
+                    break;
+                }
+
+                if (currChar === cell.text) {
+                    add(cell);
+                } else {
+                    add(cell, cell.text);
+                    if (HighlightGrid.isDouble(cell.text)) {
+                        currCharCol++;
+                    }
+                }
+
+                // eslint-disable-next-line no-constant-condition
+            } while (false);
+
+            if (!hls.length || !hls.some((d) => d.hlId !== 0)) {
+                if (gridRow[hlCol]) {
+                    hasUpdates = true;
+                    delete gridRow[hlCol];
+                }
+            } else {
+                hasUpdates = true;
+                gridRow[hlCol] = hls;
+            }
+            /////////////////////////////////////////////
+            currCharCol++;
+            cell = cellIter.next();
+        }
+
+        return hasUpdates;
+    }
+
+    buildHighlightRanges(topLine: number): HighlightRange[] {
+        const res: HighlightRange[] = [];
+        this.grid.forEach((rowHighlights, row) => {
+            const line = row + topLine;
+            let currHighlight: Highlight | null = null;
+            let currStartCol = 0;
+            let currEndCol = 0;
+            rowHighlights.forEach((colHighlights, col) => {
+                if (colHighlights.length > 1 || colHighlights[0].virtText) {
+                    res.push({
+                        textType: "virtual",
+                        highlights: colHighlights,
+                        line,
+                        col,
+                    });
+                } else {
+                    // Extend range highlights
+                    const highlight = colHighlights[0];
+                    if (currHighlight?.hlId === highlight.hlId && currEndCol === col - 1) {
+                        currEndCol = col;
+                    } else {
+                        if (currHighlight) {
+                            res.push({
+                                textType: "normal",
+                                hlId: currHighlight.hlId,
+                                line,
+                                startCol: currStartCol,
+                                endCol: currEndCol + 1,
+                            });
+                        }
+
+                        currHighlight = highlight;
+                        currStartCol = col;
+                        currEndCol = col;
+                    }
+                }
+            });
+            if (currHighlight) {
+                res.push({
+                    textType: "normal",
+                    // @ts-expect-error Typescript is wrong here. It asserts that currHighlight can never be non-null,
+                    //                  which is flagrantly incorrect. This is an artifact of the use of forEach.
+                    hlId: currHighlight.hlId,
+                    line,
+                    startCol: currStartCol,
+                    endCol: currEndCol + 1,
+                });
+            }
+        });
+
+        return res;
+    }
+
+    shiftHighlights(by: number, from: number): void {
+        if (by > 0) {
+            // remove clipped out rows
+            for (let i = 0; i < by; i++) {
+                delete this.grid[from + i];
+            }
+            // first get non empty indexes, then process, seems faster than iterating whole array
+            const idxs: number[] = [];
+            this.grid.forEach((_row, idx) => {
+                idxs.push(idx);
+            });
+            // shift
+            for (const idx of idxs) {
+                if (idx <= from) {
+                    continue;
+                }
+                this.grid[idx - by] = this.grid[idx];
+                delete this.grid[idx];
+            }
+        } else if (by < 0) {
+            // remove clipped out rows
+            for (let i = 0; i < Math.abs(by); i++) {
+                delete this.grid[from !== 0 ? from + i : this.grid.length - 1 - i];
+            }
+            const idxs: number[] = [];
+            this.grid.forEach((_row, idx) => {
+                idxs.push(idx);
+            });
+            for (const idx of idxs.reverse()) {
+                if (idx <= from) {
+                    continue;
+                }
+                this.grid[idx + Math.abs(by)] = this.grid[idx];
+                delete this.grid[idx];
+            }
+        }
+    }
+
+    maxColInRow(row: number) {
+        const gridRow = this.grid[row];
+        if (!gridRow) {
+            return 0;
+        }
+
+        let currMaxCol = 0;
+        gridRow.forEach((_, col) => {
+            if (col > currMaxCol) currMaxCol = col;
+        });
+
+        return currMaxCol;
+    }
+}
+
+class CellIter {
+    private _index = 0;
+    constructor(private _cells: ValidCell[]) {}
+    next(): { text: string; hlId: number } | undefined {
+        return this._cells[this._index++];
+    }
+    getNext(): { text: string; hlId: number } | undefined {
+        return this._cells[this._index];
+    }
+    setNext(hlId: number, text: string) {
+        if (this._index < this._cells.length) {
+            this._cells[this._index] = { hlId, text };
+        }
+    }
+}
diff --git a/src/highlight_provider.ts b/src/highlights/highlight_provider.ts
similarity index 58%
rename from src/highlight_provider.ts
rename to src/highlights/highlight_provider.ts
index 67a91dd..acd3e9c 100644
--- a/src/highlight_provider.ts
+++ b/src/highlights/highlight_provider.ts
@@ -1,6 +1,4 @@
-import GraphemeSplitter from "grapheme-splitter";
 import { cloneDeep } from "lodash-es";
-import wcswidth from "ts-wcwidth";
 import {
     DecorationOptions,
     DecorationRangeBehavior,
@@ -13,8 +11,9 @@ import {
     window,
 } from "vscode";
 
-import { calculateEditorColFromVimScreenCol, expandTabs } from "./utils";
-import { config } from "./config";
+import { config } from "../config";
+
+import { Highlight, HighlightGrid, ValidCell } from "./highlight_grid";
 
 export interface VimHighlightUIAttributes {
     foreground?: number;
@@ -42,13 +41,6 @@ export interface HighlightConfiguration {
 }
 
 type Cell = [string, number?, number?];
-interface ValidCell {
-    text: string;
-    hlId: number;
-}
-interface Highlight extends ValidCell {
-    virtText?: string;
-}
 
 /**
  * Convert VIM HL attributes to vscode text decoration attributes
@@ -106,38 +98,11 @@ function normalizeDecorationConfig(config: ThemableDecorationRenderOptions): The
     return newConfig;
 }
 
-// 你 length:1 width:2
-// 🚀 length:2 width:2
-// 🕵️ length:3 width:2
-// ❤️ length:2 width:1
-const isDouble = (c?: string) => wcswidth(c) === 2 || (c ?? "").length > 1;
-const segment: (str: string) => string[] = (() => {
-    const splitter = new GraphemeSplitter();
-    return (str) => splitter.splitGraphemes(str);
-})();
-
-class CellIter {
-    private _index = 0;
-    constructor(private _cells: ValidCell[]) {}
-    next(): { text: string; hlId: number } | undefined {
-        return this._cells[this._index++];
-    }
-    getNext(): { text: string; hlId: number } | undefined {
-        return this._cells[this._index];
-    }
-    setNext(hlId: number, text: string) {
-        if (this._index < this._cells.length) {
-            this._cells[this._index] = { hlId, text };
-        }
-    }
-}
-
 export class HighlightProvider implements Disposable {
     /**
-     * key is the grid id and values is a three-dimensional array representing rows and columns.
-     * Each column can contain multiple highlights. e.g. double-width character, tab
+     * key is the grid id and values is a grid representing those highlights
      */
-    private highlights: Map<number, Highlight[][][]> = new Map();
+    private highlights: Map<number, HighlightGrid> = new Map();
     private prevGridHighlightsIds: Map<number, Set<number>> = new Map();
     /**
      * HL group id to text decorator
@@ -215,7 +180,8 @@ export class HighlightProvider implements Disposable {
         if (!gridHl) {
             return;
         }
-        delete gridHl[row];
+
+        gridHl.cleanRow(row);
     }
 
     public processHLCellsEvent(
@@ -226,40 +192,16 @@ export class HighlightProvider implements Disposable {
         lineText: string,
         tabSize: number,
     ): boolean {
-        let hasUpdates = false;
-
         if (!this.highlights.has(grid)) {
-            this.highlights.set(grid, []);
+            this.highlights.set(grid, new HighlightGrid());
         }
         const gridHl = this.highlights.get(grid)!;
-        if (!gridHl[row]) {
-            gridHl[row] = [];
-        }
-
-        const getWidth = (text?: string) => {
-            const t = expandTabs(text ?? "", tabSize);
-            return segment(t).reduce((p, c) => p + (isDouble(c) ? 2 : 1), 0);
-        };
-
-        const lineChars = segment(lineText);
-
-        // Calculates the number of spaces occupied by the tab
-        const calcTabCells = (tabCol: number) => {
-            let nearestTabIdx = lineChars.slice(0, tabCol).lastIndexOf("\t");
-            nearestTabIdx = nearestTabIdx === -1 ? 0 : nearestTabIdx + 1;
-            const center = lineChars.slice(nearestTabIdx, tabCol).join("");
-            return tabSize - (getWidth(center) % tabSize);
-        };
-
-        const editorCol = calculateEditorColFromVimScreenCol(lineText, vimCol, tabSize);
 
+        // TODO: Break this out somehow
         const validCells: ValidCell[] = [];
         {
-            const idealMaxCells = Math.max(0, getWidth(lineText) - vimCol);
-            let currMaxCol = 0;
-            gridHl[row].forEach((_, col) => {
-                if (col > currMaxCol) currMaxCol = col;
-            });
+            const idealMaxCells = Math.max(0, HighlightGrid.getWidth(lineText, tabSize) - vimCol);
+            const currMaxCol = gridHl.maxColInRow(row);
             const maxValidCells = Math.max(idealMaxCells, currMaxCol);
             const eolCells: ValidCell[] = [];
             let currHlId = 0;
@@ -297,102 +239,8 @@ export class HighlightProvider implements Disposable {
             }
             validCells.push(...finalEolCells);
         }
-        const cellIter = new CellIter(validCells);
 
-        // #region
-        // If the previous column can contain multiple cells,
-        // then the redraw cells may contain cells from the previous column.
-        if (editorCol > 0) {
-            const prevCol = editorCol - 1;
-            const prevChar = lineChars[prevCol];
-            const expectedCells = prevChar === "\t" ? calcTabCells(prevCol) : getWidth(prevChar);
-            if (expectedCells > 1) {
-                const expectedVimCol = getWidth(lineChars.slice(0, editorCol).join(""));
-                if (expectedVimCol > vimCol) {
-                    const rightHls: Highlight[] = [];
-                    for (let i = 0; i < expectedVimCol - vimCol; i++) {
-                        const cell = cellIter.next();
-                        cell && rightHls.push({ ...cell, virtText: cell.text });
-                    }
-                    const leftHls: Highlight[] = [];
-                    if (expectedCells - rightHls.length) {
-                        leftHls.push(...(gridHl[row][prevCol] ?? []).slice(0, expectedCells - rightHls.length));
-                    }
-                    gridHl[row][prevCol] = [...leftHls, ...rightHls];
-                }
-            }
-        }
-        // #endregion
-
-        // Insert additional columns for characters with length greater than 1.
-        const filledLineText = segment(lineText).reduce((p, c) => p + c + " ".repeat(c.length - 1), "");
-
-        const filledLineChars = segment(filledLineText);
-        let currCharCol = editorCol;
-        let cell = cellIter.next();
-        while (cell) {
-            const hls: Highlight[] = [];
-            const add = (cell: ValidCell, virtText?: string) => hls.push({ ...cell, virtText });
-            const currChar = filledLineChars[currCharCol];
-            const extraCols = currChar ? currChar.length - 1 : 0;
-            currCharCol += extraCols;
-            // ... some emojis have text versions e.g. [..."❤️"] == ['❤', '️']
-            const hlCol = currCharCol - (currChar ? [...currChar].length - 1 : 0);
-
-            do {
-                if (currChar === "\t") {
-                    add(cell, cell.text);
-                    for (let i = 0; i < calcTabCells(currCharCol) - 1; i++) {
-                        cell = cellIter.next();
-                        cell && add(cell, cell.text);
-                    }
-
-                    break;
-                }
-
-                if (currChar && isDouble(currChar)) {
-                    if (currChar === cell.text) {
-                        add(cell);
-                        break;
-                    }
-
-                    add(cell, cell.text);
-                    if (!isDouble(cell.text)) {
-                        const nextCell = cellIter.next();
-                        nextCell && add(nextCell, nextCell.text);
-                        extraCols && add(nextCell ?? cell, " ".repeat(extraCols));
-                    }
-
-                    break;
-                }
-
-                if (currChar === cell.text) {
-                    add(cell);
-                } else {
-                    add(cell, cell.text);
-                    if (isDouble(cell.text)) {
-                        currCharCol++;
-                    }
-                }
-
-                // eslint-disable-next-line no-constant-condition
-            } while (false);
-
-            if (!hls.length || !hls.some((d) => d.hlId !== 0)) {
-                if (gridHl[row][hlCol]) {
-                    hasUpdates = true;
-                    delete gridHl[row][hlCol];
-                }
-            } else {
-                hasUpdates = true;
-                gridHl[row][hlCol] = hls;
-            }
-            /////////////////////////////////////////////
-            currCharCol++;
-            cell = cellIter.next();
-        }
-
-        return hasUpdates;
+        return gridHl.processHighlightCellsEvent({ row, vimCol, validCells, lineText, tabSize });
     }
 
     public shiftGridHighlights(grid: number, by: number, from: number): void {
@@ -400,41 +248,8 @@ export class HighlightProvider implements Disposable {
         if (!gridHl) {
             return;
         }
-        if (by > 0) {
-            // remove clipped out rows
-            for (let i = 0; i < by; i++) {
-                delete gridHl[from + i];
-            }
-            // first get non empty indexes, then process, seems faster than iterating whole array
-            const idxs: number[] = [];
-            gridHl.forEach((_row, idx) => {
-                idxs.push(idx);
-            });
-            // shift
-            for (const idx of idxs) {
-                if (idx <= from) {
-                    continue;
-                }
-                gridHl[idx - by] = gridHl[idx];
-                delete gridHl[idx];
-            }
-        } else if (by < 0) {
-            // remove clipped out rows
-            for (let i = 0; i < Math.abs(by); i++) {
-                delete gridHl[from !== 0 ? from + i : gridHl.length - 1 - i];
-            }
-            const idxs: number[] = [];
-            gridHl.forEach((_row, idx) => {
-                idxs.push(idx);
-            });
-            for (const idx of idxs.reverse()) {
-                if (idx <= from) {
-                    continue;
-                }
-                gridHl[idx + Math.abs(by)] = gridHl[idx];
-                delete gridHl[idx];
-            }
-        }
+
+        gridHl.shiftHighlights(by, from);
     }
 
     public getGridHighlights(
@@ -451,41 +266,26 @@ export class HighlightProvider implements Disposable {
         };
 
         const gridHl = this.highlights.get(grid);
-        if (gridHl) {
-            gridHl.forEach((rowHighlights, row) => {
-                const line = row + topLine;
+        const highlightRanges = gridHl?.buildHighlightRanges(topLine) ?? [];
+        highlightRanges.forEach((range) => {
+            if (range.textType === "virtual") {
                 // FIXME: Possibly due to viewport desync
-                if (line >= editor.document.lineCount) {
+                if (range.line >= editor.document.lineCount) {
                     return;
                 }
-                const lineText = editor.document.lineAt(line).text;
-                let currHlId = 0;
-                let currStartCol = 0;
-                let currEndCol = 0;
-                rowHighlights.forEach((colHighlights, col) => {
-                    if (colHighlights.length > 1 || colHighlights[0].virtText) {
-                        this.createColVirtTextOptions(line, col, colHighlights, lineText).forEach((options, hlId) =>
-                            pushOptions(hlId, ...options),
-                        );
-                    } else {
-                        // Extend range highlights
-                        const { hlId } = colHighlights[0];
-                        if (currHlId === hlId && currEndCol === col - 1) {
-                            currEndCol = col;
-                        } else {
-                            if (currHlId)
-                                pushOptions(currHlId, { range: new Range(line, currStartCol, line, currEndCol + 1) });
-                            currHlId = hlId;
-                            currStartCol = col;
-                            currEndCol = col;
-                        }
-                    }
+
+                const lineText = editor.document.lineAt(range.line).text;
+                this.createColVirtTextOptions(range.line, range.col, range.highlights, lineText).forEach(
+                    (options, hlId) => {
+                        pushOptions(hlId, ...options);
+                    },
+                );
+            } else {
+                pushOptions(range.hlId, {
+                    range: new Range(range.line, range.startCol, range.line, range.endCol),
                 });
-                if (currHlId) {
-                    pushOptions(currHlId, { range: new Range(line, currStartCol, line, currEndCol + 1) });
-                }
-            });
-        }
+            }
+        });
 
         const result: [TextEditorDecorationType, DecorationOptions[]][] = [];
         hlId_options.forEach((options, hlId) => {
diff --git a/src/utils.ts b/src/utils.ts
index 002771b..5d2252c 100644
--- a/src/utils.ts
+++ b/src/utils.ts
@@ -1,4 +1,3 @@
 export * from "./utils/text";
 export * from "./utils/vscode";
-export * from "./utils/nvim";
 export * from "./utils/async";
diff --git a/src/utils/nvim.ts b/src/utils/nvim.ts
deleted file mode 100644
index 6bc66fd..0000000
--- a/src/utils/nvim.ts
+++ /dev/null
@@ -1,38 +0,0 @@
-import { NeovimClient } from "neovim";
-
-import { ILogger, LogLevel } from "../logger";
-
-/**
- * Wrap nvim callAtomic and check for any errors in result
- * @param client
- * @param requests
- * @param logger
- * @param prefix
- */
-export async function callAtomic(
-    client: NeovimClient,
-    requests: [string, unknown[]][],
-    logger: ILogger,
-): Promise<void> {
-    // The type annotation in the Neovim node client seems to be wrong
-    // (see https://neovim.io/doc/user/api.html for the correct type for nvim_call_atomic)
-    const res = (await client.callAtomic(requests)) as unknown as [unknown[], [number, unknown, string] | null];
-    // Should never reach here if neovim is behaving correctly
-    if (!(res && Array.isArray(res) && Array.isArray(res[0]))) {
-        logger.log(undefined, LogLevel.error, `Unexpected result from nvim_call_atomic`);
-        return;
-    }
-    const returned_errors = res[1];
-    if (returned_errors) {
-        const [failing_call_idx, err_type, err_msg] = returned_errors;
-        const call = requests[failing_call_idx];
-        const requestName = call[0];
-        const errMsg = `${requestName}: ${err_msg} (Error type: ${err_type})`;
-        // TODO: Determine cause for errors for both of these requests
-        if (requestName === "nvim_input" || requestName === "nvim_win_close") {
-            logger.log(undefined, LogLevel.warn, errMsg);
-        } else {
-            logger.log(undefined, LogLevel.error, errMsg);
-        }
-    }
-}
diff --git a/src/utils/vscode.ts b/src/utils/vscode.ts
index c947c25..226274d 100644
--- a/src/utils/vscode.ts
+++ b/src/utils/vscode.ts
@@ -13,10 +13,9 @@ import {
     commands,
 } from "vscode";
 
-import { convertCharNumToByteNum } from "../utils";
 import { config } from "../config";
 
-import { convertByteNumToCharNum } from "./text";
+import { convertByteNumToCharNum, convertCharNumToByteNum } from "./text";
 
 /**
  * Stores last changes information for dot repeat


Test patch:
diff --git a/src/test/integ/options.test.ts b/src/test/integ/options.test.ts
index c1844b2..a6649dc 100644
--- a/src/test/integ/options.test.ts
+++ b/src/test/integ/options.test.ts
@@ -72,10 +72,11 @@ describe("Synchronize editor options", () => {
 
     async function checkTab(editor: vscode.TextEditor): Promise<void> {
         const { insertSpaces, tabSize } = editor.options;
-        const [[expandtab, tabstop]] = await client.callAtomic([
-            ["nvim_buf_get_option", [0, "expandtab"]],
-            ["nvim_buf_get_option", [0, "tabstop"]],
-        ]);
+        const [expandtab, tabstop] = (await client.lua(`
+            local expandtab = vim.api.nvim_get_option_value('expandtab', { buf = 0 })
+            local tabstop = vim.api.nvim_get_option_value('tabstop', { buf = 0 })
+            return { expandtab, tabstop }
+        `)) as any[];
         assert.equal(insertSpaces, expandtab, "insertSpaces should be equal to expandtab");
         assert.equal(tabSize, tabstop, "tabSize should be equal to tabstop");
     }
diff --git a/src/test/unit/highlights/highlight_grid.test.ts b/src/test/unit/highlights/highlight_grid.test.ts
new file mode 100644
index 0000000..73cec25
--- /dev/null
+++ b/src/test/unit/highlights/highlight_grid.test.ts
@@ -0,0 +1,1060 @@
+import { strict as assert } from "assert";
+
+import { HighlightCellsEvent, HighlightGrid, HighlightRange } from "../../../highlights/highlight_grid";
+
+function sortRanges(ranges: HighlightRange[]) {
+    ranges.sort((a, b) => {
+        const aCol = a.textType === "normal" ? a.startCol : a.col;
+        const bCol = b.textType === "normal" ? b.startCol : b.col;
+
+        return a.line + aCol - (b.line + bCol);
+    });
+}
+
+describe("processHighlightCellsEvent", () => {
+    [
+        {
+            testName: "can highlight single-width text",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 1, text: "h" },
+                        { hlId: 1, text: "e" },
+                        { hlId: 1, text: "l" },
+                        { hlId: 1, text: "l" },
+                        { hlId: 1, text: "o" },
+                    ],
+                    lineText: "hello world",
+                    tabSize: 4,
+                },
+                {
+                    row: 3,
+                    vimCol: 2,
+                    validCells: [
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "e" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "o" },
+                    ],
+                    lineText: "> hello world",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 1,
+                    line: 12,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 13,
+                    startCol: 2,
+                    endCol: 7,
+                },
+            ],
+        },
+        {
+            testName: "text after start-of-line tabs should have a startCol equal to the number of tabs",
+            events: [
+                {
+                    row: 2,
+                    // column 8 because we have two tabs, and our tab size is four
+                    vimCol: 8,
+                    validCells: [
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "e" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "o" },
+                    ],
+                    lineText: "\t\thello world",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 12,
+                    startCol: 2,
+                    endCol: 7,
+                },
+            ],
+        },
+        {
+            testName:
+                "text after middle-of-line tabs should have a startCol equal to the number of tabs plus the intermediate text width",
+            events: [
+                {
+                    row: 2,
+                    // column 12 because we have two tabs, the first one has width four, the text between them is width five,
+                    // the second has width three (due to its position relative to the tabstop)
+                    vimCol: 12,
+                    validCells: [
+                        { hlId: 2, text: "w" },
+                        { hlId: 2, text: "o" },
+                        { hlId: 2, text: "r" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "d" },
+                    ],
+                    lineText: "\thello\tworld",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 12,
+                    startCol: 7,
+                    endCol: 12,
+                },
+            ],
+        },
+        {
+            testName: "searching for tabs should highlight their full width with a virtual highlight",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 2, text: " " },
+                        { hlId: 2, text: " " },
+                        { hlId: 2, text: " " },
+                        { hlId: 2, text: " " },
+                    ],
+                    lineText: "\thello\tworld",
+                    tabSize: 4,
+                },
+                {
+                    row: 2,
+                    // column 9 because we have one tab, plus the text width of five
+                    vimCol: 9,
+                    validCells: [
+                        { hlId: 3, text: " " },
+                        { hlId: 3, text: " " },
+                        { hlId: 3, text: " " },
+                    ],
+                    lineText: "\thello\tworld",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "virtual" as const,
+                    highlights: [
+                        { hlId: 2, text: " ", virtText: " " },
+                        { hlId: 2, text: " ", virtText: " " },
+                        { hlId: 2, text: " ", virtText: " " },
+                        { hlId: 2, text: " ", virtText: " " },
+                    ],
+                    line: 12,
+                    col: 0,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [
+                        { hlId: 3, text: " ", virtText: " " },
+                        { hlId: 3, text: " ", virtText: " " },
+                        { hlId: 3, text: " ", virtText: " " },
+                    ],
+                    line: 12,
+                    col: 6,
+                },
+            ],
+        },
+        {
+            testName: "searching for double-wide characters should include their full width in the output",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 6,
+                    validCells: [
+                        { hlId: 2, text: "s" },
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "i" },
+                        { hlId: 2, text: "p" },
+                        { hlId: 2, text: " " },
+                        { hlId: 2, text: "i" },
+                        { hlId: 2, text: "t" },
+                        { hlId: 2, text: " " },
+                        { hlId: 2, text: "🚀" },
+                    ],
+                    lineText: "Let's ship it 🚀",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 12,
+                    startCol: 6,
+                    endCol: 14,
+                },
+                // The rocket has a separate highlight range
+                // TODO: should it?
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 12,
+                    startCol: 15,
+                    endCol: 16,
+                },
+            ],
+        },
+        {
+            testName: "removes highlights when they are reassigned to hl group zero",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 6,
+                    validCells: [
+                        { hlId: 2, text: "w" },
+                        { hlId: 2, text: "o" },
+                        { hlId: 2, text: "r" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "d" },
+                    ],
+                    lineText: "hello world",
+                    tabSize: 4,
+                },
+                {
+                    row: 2,
+                    vimCol: 6,
+                    validCells: [
+                        { hlId: 0, text: "w" },
+                        { hlId: 0, text: "o" },
+                        { hlId: 0, text: "r" },
+                        { hlId: 0, text: "l" },
+                        { hlId: 0, text: "d" },
+                    ],
+                    lineText: "hello world",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [],
+        },
+        {
+            testName: "removes highlights when part of the selection is reassigned to hl group zero",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 6,
+                    validCells: [
+                        { hlId: 2, text: "w" },
+                        { hlId: 2, text: "o" },
+                        { hlId: 2, text: "r" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "d" },
+                    ],
+                    lineText: "hello world",
+                    tabSize: 4,
+                },
+                {
+                    row: 2,
+                    vimCol: 6,
+                    validCells: [
+                        { hlId: 0, text: "w" },
+                        { hlId: 0, text: "o" },
+                        { hlId: 0, text: "r" },
+                        // l and d are kept
+                    ],
+                    lineText: "hello world",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 12,
+                    startCol: 9,
+                    endCol: 11,
+                },
+            ],
+        },
+        {
+            testName: "removes highlights from deleted characters",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 6,
+                    validCells: [
+                        { hlId: 2, text: "w" },
+                        { hlId: 2, text: "o" },
+                        { hlId: 2, text: "r" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "d" },
+                    ],
+                    lineText: "hello world",
+                    tabSize: 4,
+                },
+                {
+                    row: 2,
+                    vimCol: 6,
+                    validCells: [
+                        { hlId: 2, text: "o" },
+                        { hlId: 2, text: "r" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "d" },
+                    ],
+                    lineText: "hello orld",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 12,
+                    startCol: 6,
+                    endCol: 11,
+                },
+            ],
+        },
+        {
+            testName: "allows adding virtual text to the end of an empty line",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "e" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "o" },
+                    ],
+                    lineText: "",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "h", virtText: "h" }],
+                    line: 12,
+                    col: 0,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "e", virtText: "e" }],
+                    line: 12,
+                    col: 1,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "l", virtText: "l" }],
+                    line: 12,
+                    col: 2,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "l", virtText: "l" }],
+                    line: 12,
+                    col: 3,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "o", virtText: "o" }],
+                    line: 12,
+                    col: 4,
+                },
+            ],
+        },
+        {
+            testName: "allows adding virtual text with double-wide characters to the end of an empty line",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 2, text: "👋" },
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "e" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "o" },
+                    ],
+                    lineText: "",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "👋", virtText: "👋" }],
+                    line: 12,
+                    col: 0,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "h", virtText: "h" }],
+                    line: 12,
+                    col: 2,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "e", virtText: "e" }],
+                    line: 12,
+                    col: 3,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "l", virtText: "l" }],
+                    line: 12,
+                    col: 4,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "l", virtText: "l" }],
+                    line: 12,
+                    col: 5,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "o", virtText: "o" }],
+                    line: 12,
+                    col: 6,
+                },
+            ],
+        },
+        {
+            testName: "allows overlaying virtual text on an existing line",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "e" },
+                        { hlId: 2, text: "y" },
+                        { hlId: 2, text: "!" },
+                        { hlId: 2, text: "!" },
+                    ],
+                    lineText: "world",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "h", virtText: "h" }],
+                    line: 12,
+                    col: 0,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "e", virtText: "e" }],
+                    line: 12,
+                    col: 1,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "y", virtText: "y" }],
+                    line: 12,
+                    col: 2,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "!", virtText: "!" }],
+                    line: 12,
+                    col: 3,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "!", virtText: "!" }],
+                    line: 12,
+                    col: 4,
+                },
+            ],
+        },
+        {
+            testName: "allows overlaying virtual text on an existing line with double-wide characters",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "e" },
+                        { hlId: 2, text: "y" },
+                        { hlId: 2, text: "!" },
+                        { hlId: 2, text: "!" },
+                    ],
+                    lineText: "👋 yo",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "virtual" as const,
+                    highlights: [
+                        { hlId: 2, text: "h", virtText: "h" },
+                        { hlId: 2, text: "e", virtText: "e" },
+                        // TODO: why is e repeated? Something to do with the double-wide char...
+                        { hlId: 2, text: "e", virtText: " " },
+                    ],
+                    line: 12,
+                    col: 1,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "y", virtText: "y" }],
+                    line: 12,
+                    col: 2,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "!", virtText: "!" }],
+                    line: 12,
+                    col: 3,
+                },
+                {
+                    textType: "virtual" as const,
+                    highlights: [{ hlId: 2, text: "!", virtText: "!" }],
+                    line: 12,
+                    col: 4,
+                },
+            ],
+        },
+        {
+            testName:
+                "allows overlaying virtual text over double-wide chars with an early end by adding an extra space",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [{ hlId: 2, text: "h" }],
+                    lineText: "👋",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "virtual" as const,
+                    highlights: [
+                        { hlId: 2, text: "h", virtText: "h" },
+                        { hlId: 2, text: "h", virtText: " " },
+                    ],
+                    line: 12,
+                    col: 1,
+                },
+            ],
+        },
+        {
+            testName: "redrawing a virtual highlight over a double-wide character redraws both cells",
+            events: [
+                {
+                    row: 2,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 2, text: "a" },
+                        { hlId: 2, text: "1" },
+                    ],
+                    lineText: "你",
+                    tabSize: 4,
+                },
+                {
+                    row: 2,
+                    vimCol: 1,
+                    validCells: [{ hlId: 2, text: "2" }],
+                    lineText: "你",
+                    tabSize: 4,
+                },
+            ],
+            expectedRanges: [
+                {
+                    textType: "virtual" as const,
+                    highlights: [
+                        { hlId: 2, text: "a", virtText: "a" },
+                        { hlId: 2, text: "2", virtText: "2" },
+                    ],
+                    line: 12,
+                    col: 0,
+                },
+            ],
+        },
+    ].forEach(
+        ({
+            testName,
+            events,
+            expectedRanges,
+        }: {
+            testName: string;
+            events: HighlightCellsEvent[];
+            expectedRanges: HighlightRange[];
+        }) => {
+            it(testName, () => {
+                const grid = new HighlightGrid();
+                events.forEach((event) => {
+                    const updated = grid.processHighlightCellsEvent(event);
+                    assert.equal(true, updated);
+                });
+
+                // 10 is arbitrary, it's just to demonstrate that this offset works as expected
+                const ranges = grid.buildHighlightRanges(10);
+                sortRanges(ranges);
+
+                assert.deepEqual(ranges, expectedRanges);
+            });
+        },
+    );
+
+    it("returns false when no update is applied", () => {
+        const grid = new HighlightGrid();
+        // Deselect something that was never highlighted
+        const updated = grid.processHighlightCellsEvent({
+            row: 2,
+            vimCol: 0,
+            validCells: [
+                { hlId: 0, text: "h" },
+                { hlId: 0, text: "e" },
+                { hlId: 0, text: "l" },
+                { hlId: 0, text: "l" },
+                { hlId: 0, text: "o" },
+            ],
+            lineText: "hello world",
+            tabSize: 4,
+        });
+
+        assert.equal(updated, false);
+    });
+});
+
+describe("cleanRow", () => {
+    it("clears selected lines from the highlighted ranges", () => {
+        const grid = new HighlightGrid();
+
+        for (let row = 1; row <= 5; row++) {
+            grid.processHighlightCellsEvent({
+                row,
+                vimCol: 0,
+                validCells: [
+                    { hlId: row, text: "h" },
+                    { hlId: row, text: "e" },
+                    { hlId: row, text: "l" },
+                    { hlId: row, text: "l" },
+                    { hlId: row, text: "o" },
+                ],
+                lineText: "hello world",
+                tabSize: 4,
+            });
+        }
+
+        grid.cleanRow(2);
+        grid.cleanRow(4);
+
+        const ranges = grid.buildHighlightRanges(0);
+        sortRanges(ranges);
+
+        assert.deepEqual(ranges, [
+            {
+                textType: "normal",
+                hlId: 1,
+                line: 1,
+                startCol: 0,
+                endCol: 5,
+            },
+            {
+                textType: "normal",
+                hlId: 3,
+                line: 3,
+                startCol: 0,
+                endCol: 5,
+            },
+            {
+                textType: "normal",
+                hlId: 5,
+                line: 5,
+                startCol: 0,
+                endCol: 5,
+            },
+        ]);
+    });
+});
+
+describe("shiftHighlights", () => {
+    [
+        {
+            testName: "should no nothing when scrolling by zero",
+            scrollAmount: 0,
+            scrollFrom: 0,
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 4,
+                    startCol: 1,
+                    endCol: 6,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 3,
+                    line: 6,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 4,
+                    line: 7,
+                    startCol: 2,
+                    endCol: 4,
+                },
+            ],
+        },
+        {
+            testName: "should shift items up by the scroll amount",
+            scrollAmount: 2,
+            scrollFrom: 0,
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    // 4 -> 2
+                    line: 2,
+                    startCol: 1,
+                    endCol: 6,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 3,
+                    // 6 -> 4
+                    line: 4,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 4,
+                    // 7 -> 5
+                    line: 5,
+                    startCol: 2,
+                    endCol: 4,
+                },
+            ],
+        },
+        {
+            testName: "should shift items down by the scroll amount",
+            scrollAmount: -2,
+            scrollFrom: 2,
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    // 4 -> 6
+                    line: 6,
+                    startCol: 1,
+                    endCol: 6,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 3,
+                    // 6 -> 8
+                    line: 8,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 4,
+                    // 7 -> 9
+                    line: 9,
+                    startCol: 2,
+                    endCol: 4,
+                },
+            ],
+        },
+        {
+            testName: "should clip out of bounds items when scrolling up",
+            scrollAmount: 5,
+            scrollFrom: 0,
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 3,
+                    // 6 -> 1
+                    line: 1,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 4,
+                    // 7 -> 2
+                    line: 2,
+                    startCol: 2,
+                    endCol: 4,
+                },
+            ],
+        },
+        {
+            testName: "should clip out of bounds items when scrolling down",
+            scrollAmount: -5,
+            scrollFrom: 1,
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 3,
+                    // 6 -> 11
+                    line: 11,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 4,
+                    // 7 -> 12
+                    line: 12,
+                    startCol: 2,
+                    endCol: 4,
+                },
+            ],
+        },
+        {
+            testName: "should clip out of bounds items when scrolling down when scrolling down from position zero",
+            scrollAmount: -3,
+            scrollFrom: 0,
+            expectedRanges: [
+                // This is based on the number of rows we already have highlighted,
+                // so we remove anything 3 above original line 7
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    // 4 -> 7
+                    line: 7,
+                    startCol: 1,
+                    endCol: 6,
+                },
+            ],
+        },
+        {
+            testName: "should not alter highlight ranges that are out of scope of the scroll up event",
+            scrollAmount: 1,
+            scrollFrom: 100,
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 4,
+                    startCol: 1,
+                    endCol: 6,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 3,
+                    line: 6,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 4,
+                    line: 7,
+                    startCol: 2,
+                    endCol: 4,
+                },
+            ],
+        },
+        {
+            testName: "should not alter highlight ranges that are out of scope of the scroll down event",
+            scrollAmount: -1,
+            scrollFrom: 100,
+            expectedRanges: [
+                {
+                    textType: "normal" as const,
+                    hlId: 2,
+                    line: 4,
+                    startCol: 1,
+                    endCol: 6,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 3,
+                    line: 6,
+                    startCol: 0,
+                    endCol: 5,
+                },
+                {
+                    textType: "normal" as const,
+                    hlId: 4,
+                    line: 7,
+                    startCol: 2,
+                    endCol: 4,
+                },
+            ],
+        },
+    ].forEach(
+        ({
+            testName,
+            scrollAmount,
+            scrollFrom,
+            expectedRanges,
+        }: {
+            testName: string;
+            scrollAmount: number;
+            expectedRanges: HighlightRange[];
+            scrollFrom: number;
+        }) => {
+            it(testName, () => {
+                const grid = new HighlightGrid();
+
+                grid.processHighlightCellsEvent({
+                    row: 4,
+                    vimCol: 4,
+                    validCells: [
+                        { hlId: 2, text: "h" },
+                        { hlId: 2, text: "e" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "l" },
+                        { hlId: 2, text: "o" },
+                    ],
+                    lineText: "\thello,",
+                    tabSize: 4,
+                });
+
+                grid.processHighlightCellsEvent({
+                    row: 6,
+                    vimCol: 0,
+                    validCells: [
+                        { hlId: 3, text: "w" },
+                        { hlId: 3, text: "o" },
+                        { hlId: 3, text: "r" },
+                        { hlId: 3, text: "l" },
+                        { hlId: 3, text: "d" },
+                    ],
+                    lineText: "world",
+                    tabSize: 4,
+                });
+                grid.processHighlightCellsEvent({
+                    row: 7,
+                    vimCol: 8,
+                    validCells: [
+                        { hlId: 4, text: "!" },
+                        { hlId: 4, text: "!" },
+                    ],
+                    lineText: "\t\t!!",
+                    tabSize: 4,
+                });
+
+                grid.shiftHighlights(scrollAmount, scrollFrom);
+                const ranges = grid.buildHighlightRanges(0);
+                sortRanges(ranges);
+
+                assert.deepEqual(ranges, expectedRanges);
+            });
+        },
+    );
+});
+
+describe("maxColInRow", () => {
+    it("should return zero for an empty row", () => {
+        const grid = new HighlightGrid();
+        assert.equal(grid.maxColInRow(2), 0);
+    });
+
+    it("should return the last column of the last highlight in the given row for an empty row", () => {
+        const grid = new HighlightGrid();
+        grid.processHighlightCellsEvent({
+            row: 4,
+            vimCol: 0,
+            validCells: [
+                { hlId: 2, text: "h" },
+                { hlId: 2, text: "e" },
+                { hlId: 2, text: "l" },
+                { hlId: 2, text: "l" },
+                { hlId: 2, text: "o" },
+            ],
+            lineText: "hello hello!",
+            tabSize: 4,
+        });
+
+        grid.processHighlightCellsEvent({
+            row: 4,
+            vimCol: 6,
+            validCells: [
+                { hlId: 3, text: "h" },
+                { hlId: 3, text: "e" },
+                { hlId: 3, text: "l" },
+                { hlId: 3, text: "l" },
+                { hlId: 3, text: "o" },
+            ],
+            lineText: "hello hello!",
+            tabSize: 4,
+        });
+
+        assert.equal(grid.maxColInRow(4), 10);
+    });
+});
+
+describe("isDouble", () => {
+    [
+        {
+            char: "a", // length 1
+            expectedDouble: false,
+        },
+        {
+            char: "₪", // length 1
+            expectedDouble: false,
+        },
+        {
+            char: "你", // length 1
+            expectedDouble: true,
+        },
+        {
+            char: "🚀", // length 2
+            expectedDouble: true,
+        },
+        {
+            char: "🕵️", // length 3
+            expectedDouble: true,
+        },
+        {
+            char: "❤️", // length 2
+            expectedDouble: true,
+        },
+    ].forEach(({ char, expectedDouble }) => {
+        it(`${expectedDouble ? "should" : "should not"} consider '${char}' to be double-width`, () => {
+            assert.equal(HighlightGrid.isDouble(char), expectedDouble);
+        });
+    });
+});
+
+describe("getWidth", () => {
+    [
+        {
+            testName: "normal ascii should have a width equal to the length",
+            text: "hello world",
+            expectedWidth: 11,
+        },
+        {
+            testName: "double-wide chars should count twice in the length",
+            text: "ship it 🚀🚀", // rockets are double-wide, so count twice
+            expectedWidth: 12,
+        },
+        {
+            testName: "tabs at the start of a line should count for their full tab width",
+            text: "\treturn 0;",
+            expectedWidth: 13,
+        },
+        {
+            // see expandTabs for more tests like this
+            testName: "tabs in the middle of a line should only count up to their next tabstop",
+            text: "\treturn\t0;",
+            expectedWidth: 14,
+        },
+    ].forEach(({ testName, text, expectedWidth }) => {
+        it(testName, () => {
+            assert.equal(HighlightGrid.getWidth(text, 4), expectedWidth);
+        });
+    });
+});