Repository: microsoft/vscode (TypeScript)
Date: 2022-12-06 11:52:32+00:00
Task type: Bug Fix

Issue:
Unicode highlight: Non-breaking space not highlighted
Non-breaking space is not highlighted when it should've been.

The character 'å' is a common letter in the Norwegian alphabet and can also be used as a word. The phrase "for å gi et eksempel" (which means "to give an example") is an example of this. A non-breaking space after the word 'å' is not highlighted by vscode as can be seen in the screenshot where all spaces are non-breaking spaces (U+00A0):

![image](https://user-images.githubusercontent.com/1660659/205630244-11d19c5f-11bf-43e3-abd5-e6f6c791be93.png)

I'm guessing this is a regression introduced by #140960. I see the reason for why it's behaving like this, but it would be nice if it was possible to make it work in both scenarios.

Does this issue occur when all extensions are disabled?: Yes

<!-- 🪓 If you answered No above, use 'Help: Start Extension Bisect' from Command Palette to try to identify the cause. -->
<!-- 📣 Issues caused by an extension need to be reported directly to the extension publisher. The 'Help > Report Issue' dialog can assist with this. -->
- VS Code Version: 1.73.1
- OS Version: macOS 13.0.1

Steps to Reproduce:

1. Configure vscode with `"editor.unicodeHighlight.allowedCharacters": { "å": true }`
2. Test with this example text: "For å gi et eksempel"
3. Verify that a non-breaking space after 'å' is not highlighted


Issue discussion:

Solution patch:
diff --git a/src/vs/editor/common/services/unicodeTextModelHighlighter.ts b/src/vs/editor/common/services/unicodeTextModelHighlighter.ts
--- a/src/vs/editor/common/services/unicodeTextModelHighlighter.ts
+++ b/src/vs/editor/common/services/unicodeTextModelHighlighter.ts
@@ -61,7 +61,11 @@ export class UnicodeTextModelHighlighter {
 						}
 					}
 					const str = lineContent.substring(startIndex, endIndex);
-					const word = getWordAtText(startIndex + 1, DEFAULT_WORD_REGEXP, lineContent, 0);
+					let word = getWordAtText(startIndex + 1, DEFAULT_WORD_REGEXP, lineContent, 0);
+					if (word && word.endColumn <= startIndex + 1) {
+						// The word does not include the problematic character, ignore the word
+						word = null;
+					}
 					const highlightReason = codePointHighlighter.shouldHighlightNonBasicASCII(str, word ? word.word : null);
 
 					if (highlightReason !== SimpleHighlightReason.None) {


Test patch:
diff --git a/src/vs/editor/test/common/services/unicodeTextModelHighlighter.test.ts b/src/vs/editor/test/common/services/unicodeTextModelHighlighter.test.ts
new file mode 100644
--- /dev/null
+++ b/src/vs/editor/test/common/services/unicodeTextModelHighlighter.test.ts
@@ -0,0 +1,53 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert = require('assert');
+import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils';
+import { Range } from 'vs/editor/common/core/range';
+import { UnicodeHighlighterOptions, UnicodeTextModelHighlighter } from 'vs/editor/common/services/unicodeTextModelHighlighter';
+import { createTextModel } from 'vs/editor/test/common/testTextModel';
+
+suite('UnicodeTextModelHighlighter', () => {
+	ensureNoDisposablesAreLeakedInTestSuite();
+
+	function t(text: string, options: UnicodeHighlighterOptions): unknown {
+		const m = createTextModel(text);
+		const r = UnicodeTextModelHighlighter.computeUnicodeHighlights(m, options);
+		m.dispose();
+
+		return {
+			...r,
+			ranges: r.ranges.map(r => Range.lift(r).toString())
+		};
+	}
+
+	test('computeUnicodeHighlights (#168068)', () => {
+		assert.deepStrictEqual(
+			t(`
+	For å gi et eksempel
+`, {
+				allowedCodePoints: [],
+				allowedLocales: [],
+				ambiguousCharacters: true,
+				invisibleCharacters: true,
+				includeComments: false,
+				includeStrings: false,
+				nonBasicASCII: false
+			}),
+			{
+				ambiguousCharacterCount: 0,
+				hasMore: false,
+				invisibleCharacterCount: 4,
+				nonBasicAsciiCharacterCount: 0,
+				ranges: [
+					'[2,5 -> 2,6]',
+					'[2,7 -> 2,8]',
+					'[2,10 -> 2,11]',
+					'[2,13 -> 2,14]'
+				]
+			}
+		);
+	});
+});