Repository: microsoft/vscode (TypeScript)
Date: 2022-03-02 11:02:03+00:00
Task type: Bug Fix

Issue:
Issue in RTL languages in recent versions
![Untitled](https://user-images.githubusercontent.com/34407036/141463706-0fa2ff35-a6fa-43b9-9178-385f9121b55a.png)

this issue was not exist in previous versions


Issue discussion:
@YassineXM Can you please provide the content also in text form?
`<option value="العربية">العربية</option>`
Hi! @alexdima 
Can I solve this issue as a good-first-issue?
Is it easy or hard for a first-time contributor?
In vscode editor if we inspect, we have this code:
```
<span dir="ltr">
    <span class="mtk1">
          &lt;option&nbsp;
    </span>
    <span class="mtk5">
          value
     </span>
    <span class="mtk1">
        =
     </span>
    <span class="mtk11">
        "العربية"
    </span>
    <span class="mtk1">
        &gt;العربية&lt;/option&gt;
    </span>
</span>
```
And as a result we see this:
![image](https://user-images.githubusercontent.com/45296858/147136504-b0582a7d-c395-4dec-b085-fe1de77d4711.png)

If we write this HTML code instead:
```
<span dir="ltr">
    <span class="mtk1">
          &lt;option&nbsp;
    </span>
    <span class="mtk5">
          value
     </span>
    <span class="mtk1">
        =
     </span>
    <span class="mtk11" dir="auto">
        "العربية"
    </span>
    <span class="mtk1">
        &gt;العربية&lt;/option&gt;
    </span>
</span>`
```
Then we see the result like this:
![image](https://user-images.githubusercontent.com/45296858/147137243-c9302f9d-ec2a-4977-8c63-74406bb44acd.png)

I've just added `dir="auto"` to the `mtk11` class.


Solution patch:
diff --git a/src/vs/editor/common/viewLayout/viewLineRenderer.ts b/src/vs/editor/common/viewLayout/viewLineRenderer.ts
--- a/src/vs/editor/common/viewLayout/viewLineRenderer.ts
+++ b/src/vs/editor/common/viewLayout/viewLineRenderer.ts
@@ -31,18 +31,15 @@ export const enum LinePartMetadata {
 class LinePart {
 	_linePartBrand: void = undefined;
 
-	/**
-	 * last char index of this token (not inclusive).
-	 */
-	public readonly endIndex: number;
-	public readonly type: string;
-	public readonly metadata: number;
-
-	constructor(endIndex: number, type: string, metadata: number) {
-		this.endIndex = endIndex;
-		this.type = type;
-		this.metadata = metadata;
-	}
+	constructor(
+		/**
+		 * last char index of this token (not inclusive).
+		 */
+		public readonly endIndex: number,
+		public readonly type: string,
+		public readonly metadata: number,
+		public readonly containsRTL: boolean
+	) { }
 
 	public isWhitespace(): boolean {
 		return (this.metadata & LinePartMetadata.IS_WHITESPACE_MASK ? true : false);
@@ -473,7 +470,7 @@ function resolveRenderLineInput(input: RenderLineInput): ResolvedRenderLineInput
 		len = lineContent.length;
 	}
 
-	let tokens = transformAndRemoveOverflowing(input.lineTokens, input.fauxIndentLength, len);
+	let tokens = transformAndRemoveOverflowing(lineContent, input.containsRTL, input.lineTokens, input.fauxIndentLength, len);
 	if (input.renderControlCharacters && !input.isBasicASCII) {
 		// Calling `extractControlCharacters` before adding (possibly empty) line parts
 		// for inline decorations. `extractControlCharacters` removes empty line parts.
@@ -529,15 +526,15 @@ function resolveRenderLineInput(input: RenderLineInput): ResolvedRenderLineInput
  * In the rendering phase, characters are always looped until token.endIndex.
  * Ensure that all tokens end before `len` and the last one ends precisely at `len`.
  */
-function transformAndRemoveOverflowing(tokens: IViewLineTokens, fauxIndentLength: number, len: number): LinePart[] {
+function transformAndRemoveOverflowing(lineContent: string, lineContainsRTL: boolean, tokens: IViewLineTokens, fauxIndentLength: number, len: number): LinePart[] {
 	const result: LinePart[] = [];
 	let resultLen = 0;
 
 	// The faux indent part of the line should have no token type
 	if (fauxIndentLength > 0) {
-		result[resultLen++] = new LinePart(fauxIndentLength, '', 0);
+		result[resultLen++] = new LinePart(fauxIndentLength, '', 0, false);
 	}
-
+	let startOffset = fauxIndentLength;
 	for (let tokenIndex = 0, tokensLen = tokens.getCount(); tokenIndex < tokensLen; tokenIndex++) {
 		const endIndex = tokens.getEndOffset(tokenIndex);
 		if (endIndex <= fauxIndentLength) {
@@ -546,10 +543,13 @@ function transformAndRemoveOverflowing(tokens: IViewLineTokens, fauxIndentLength
 		}
 		const type = tokens.getClassName(tokenIndex);
 		if (endIndex >= len) {
-			result[resultLen++] = new LinePart(len, type, 0);
+			const tokenContainsRTL = (lineContainsRTL ? strings.containsRTL(lineContent.substring(startOffset, len)) : false);
+			result[resultLen++] = new LinePart(len, type, 0, tokenContainsRTL);
 			break;
 		}
-		result[resultLen++] = new LinePart(endIndex, type, 0);
+		const tokenContainsRTL = (lineContainsRTL ? strings.containsRTL(lineContent.substring(startOffset, endIndex)) : false);
+		result[resultLen++] = new LinePart(endIndex, type, 0, tokenContainsRTL);
+		startOffset = endIndex;
 	}
 
 	return result;
@@ -580,6 +580,7 @@ function splitLargeTokens(lineContent: string, tokens: LinePart[], onlyAtSpaces:
 			if (lastTokenEndIndex + Constants.LongToken < tokenEndIndex) {
 				const tokenType = token.type;
 				const tokenMetadata = token.metadata;
+				const tokenContainsRTL = token.containsRTL;
 
 				let lastSpaceOffset = -1;
 				let currTokenStart = lastTokenEndIndex;
@@ -589,13 +590,13 @@ function splitLargeTokens(lineContent: string, tokens: LinePart[], onlyAtSpaces:
 					}
 					if (lastSpaceOffset !== -1 && j - currTokenStart >= Constants.LongToken) {
 						// Split at `lastSpaceOffset` + 1
-						result[resultLen++] = new LinePart(lastSpaceOffset + 1, tokenType, tokenMetadata);
+						result[resultLen++] = new LinePart(lastSpaceOffset + 1, tokenType, tokenMetadata, tokenContainsRTL);
 						currTokenStart = lastSpaceOffset + 1;
 						lastSpaceOffset = -1;
 					}
 				}
 				if (currTokenStart !== tokenEndIndex) {
-					result[resultLen++] = new LinePart(tokenEndIndex, tokenType, tokenMetadata);
+					result[resultLen++] = new LinePart(tokenEndIndex, tokenType, tokenMetadata, tokenContainsRTL);
 				}
 			} else {
 				result[resultLen++] = token;
@@ -612,12 +613,13 @@ function splitLargeTokens(lineContent: string, tokens: LinePart[], onlyAtSpaces:
 			if (diff > Constants.LongToken) {
 				const tokenType = token.type;
 				const tokenMetadata = token.metadata;
+				const tokenContainsRTL = token.containsRTL;
 				const piecesCount = Math.ceil(diff / Constants.LongToken);
 				for (let j = 1; j < piecesCount; j++) {
 					const pieceEndIndex = lastTokenEndIndex + (j * Constants.LongToken);
-					result[resultLen++] = new LinePart(pieceEndIndex, tokenType, tokenMetadata);
+					result[resultLen++] = new LinePart(pieceEndIndex, tokenType, tokenMetadata, tokenContainsRTL);
 				}
-				result[resultLen++] = new LinePart(tokenEndIndex, tokenType, tokenMetadata);
+				result[resultLen++] = new LinePart(tokenEndIndex, tokenType, tokenMetadata, tokenContainsRTL);
 			} else {
 				result[resultLen++] = token;
 			}
@@ -664,7 +666,7 @@ function isControlCharacter(charCode: number): boolean {
 
 function extractControlCharacters(lineContent: string, tokens: LinePart[]): LinePart[] {
 	const result: LinePart[] = [];
-	let lastLinePart: LinePart = new LinePart(0, '', 0);
+	let lastLinePart: LinePart = new LinePart(0, '', 0, false);
 	let charOffset = 0;
 	for (const token of tokens) {
 		const tokenEndIndex = token.endIndex;
@@ -673,16 +675,16 @@ function extractControlCharacters(lineContent: string, tokens: LinePart[]): Line
 			if (isControlCharacter(charCode)) {
 				if (charOffset > lastLinePart.endIndex) {
 					// emit previous part if it has text
-					lastLinePart = new LinePart(charOffset, token.type, token.metadata);
+					lastLinePart = new LinePart(charOffset, token.type, token.metadata, token.containsRTL);
 					result.push(lastLinePart);
 				}
-				lastLinePart = new LinePart(charOffset + 1, 'mtkcontrol', token.metadata);
+				lastLinePart = new LinePart(charOffset + 1, 'mtkcontrol', token.metadata, false);
 				result.push(lastLinePart);
 			}
 		}
 		if (charOffset > lastLinePart.endIndex) {
 			// emit previous part if it has text
-			lastLinePart = new LinePart(tokenEndIndex, token.type, token.metadata);
+			lastLinePart = new LinePart(tokenEndIndex, token.type, token.metadata, token.containsRTL);
 			result.push(lastLinePart);
 		}
 	}
@@ -710,6 +712,7 @@ function _applyRenderWhitespace(input: RenderLineInput, lineContent: string, len
 	let resultLen = 0;
 	let tokenIndex = 0;
 	let tokenType = tokens[tokenIndex].type;
+	let tokenContainsRTL = tokens[tokenIndex].containsRTL;
 	let tokenEndIndex = tokens[tokenIndex].endIndex;
 	const tokensLength = tokens.length;
 
@@ -777,17 +780,17 @@ function _applyRenderWhitespace(input: RenderLineInput, lineContent: string, len
 				if (generateLinePartForEachWhitespace) {
 					const lastEndIndex = (resultLen > 0 ? result[resultLen - 1].endIndex : fauxIndentLength);
 					for (let i = lastEndIndex + 1; i <= charIndex; i++) {
-						result[resultLen++] = new LinePart(i, 'mtkw', LinePartMetadata.IS_WHITESPACE);
+						result[resultLen++] = new LinePart(i, 'mtkw', LinePartMetadata.IS_WHITESPACE, false);
 					}
 				} else {
-					result[resultLen++] = new LinePart(charIndex, 'mtkw', LinePartMetadata.IS_WHITESPACE);
+					result[resultLen++] = new LinePart(charIndex, 'mtkw', LinePartMetadata.IS_WHITESPACE, false);
 				}
 				tmpIndent = tmpIndent % tabSize;
 			}
 		} else {
 			// was in regular token
 			if (charIndex === tokenEndIndex || (isInWhitespace && charIndex > fauxIndentLength)) {
-				result[resultLen++] = new LinePart(charIndex, tokenType, 0);
+				result[resultLen++] = new LinePart(charIndex, tokenType, 0, tokenContainsRTL);
 				tmpIndent = tmpIndent % tabSize;
 			}
 		}
@@ -806,6 +809,7 @@ function _applyRenderWhitespace(input: RenderLineInput, lineContent: string, len
 			tokenIndex++;
 			if (tokenIndex < tokensLength) {
 				tokenType = tokens[tokenIndex].type;
+				tokenContainsRTL = tokens[tokenIndex].containsRTL;
 				tokenEndIndex = tokens[tokenIndex].endIndex;
 			} else {
 				break;
@@ -832,13 +836,13 @@ function _applyRenderWhitespace(input: RenderLineInput, lineContent: string, len
 		if (generateLinePartForEachWhitespace) {
 			const lastEndIndex = (resultLen > 0 ? result[resultLen - 1].endIndex : fauxIndentLength);
 			for (let i = lastEndIndex + 1; i <= len; i++) {
-				result[resultLen++] = new LinePart(i, 'mtkw', LinePartMetadata.IS_WHITESPACE);
+				result[resultLen++] = new LinePart(i, 'mtkw', LinePartMetadata.IS_WHITESPACE, false);
 			}
 		} else {
-			result[resultLen++] = new LinePart(len, 'mtkw', LinePartMetadata.IS_WHITESPACE);
+			result[resultLen++] = new LinePart(len, 'mtkw', LinePartMetadata.IS_WHITESPACE, false);
 		}
 	} else {
-		result[resultLen++] = new LinePart(len, tokenType, 0);
+		result[resultLen++] = new LinePart(len, tokenType, 0, tokenContainsRTL);
 	}
 
 	return result;
@@ -862,31 +866,32 @@ function _applyInlineDecorations(lineContent: string, len: number, tokens: LineP
 		const tokenEndIndex = token.endIndex;
 		const tokenType = token.type;
 		const tokenMetadata = token.metadata;
+		const tokenContainsRTL = token.containsRTL;
 
 		while (lineDecorationIndex < lineDecorationsLen && lineDecorations[lineDecorationIndex].startOffset < tokenEndIndex) {
 			const lineDecoration = lineDecorations[lineDecorationIndex];
 
 			if (lineDecoration.startOffset > lastResultEndIndex) {
 				lastResultEndIndex = lineDecoration.startOffset;
-				result[resultLen++] = new LinePart(lastResultEndIndex, tokenType, tokenMetadata);
+				result[resultLen++] = new LinePart(lastResultEndIndex, tokenType, tokenMetadata, tokenContainsRTL);
 			}
 
 			if (lineDecoration.endOffset + 1 <= tokenEndIndex) {
 				// This line decoration ends before this token ends
 				lastResultEndIndex = lineDecoration.endOffset + 1;
-				result[resultLen++] = new LinePart(lastResultEndIndex, tokenType + ' ' + lineDecoration.className, tokenMetadata | lineDecoration.metadata);
+				result[resultLen++] = new LinePart(lastResultEndIndex, tokenType + ' ' + lineDecoration.className, tokenMetadata | lineDecoration.metadata, tokenContainsRTL);
 				lineDecorationIndex++;
 			} else {
 				// This line decoration continues on to the next token
 				lastResultEndIndex = tokenEndIndex;
-				result[resultLen++] = new LinePart(lastResultEndIndex, tokenType + ' ' + lineDecoration.className, tokenMetadata | lineDecoration.metadata);
+				result[resultLen++] = new LinePart(lastResultEndIndex, tokenType + ' ' + lineDecoration.className, tokenMetadata | lineDecoration.metadata, tokenContainsRTL);
 				break;
 			}
 		}
 
 		if (tokenEndIndex > lastResultEndIndex) {
 			lastResultEndIndex = tokenEndIndex;
-			result[resultLen++] = new LinePart(lastResultEndIndex, tokenType, tokenMetadata);
+			result[resultLen++] = new LinePart(lastResultEndIndex, tokenType, tokenMetadata, tokenContainsRTL);
 		}
 	}
 
@@ -894,7 +899,7 @@ function _applyInlineDecorations(lineContent: string, len: number, tokens: LineP
 	if (lineDecorationIndex < lineDecorationsLen && lineDecorations[lineDecorationIndex].startOffset === lastTokenEndIndex) {
 		while (lineDecorationIndex < lineDecorationsLen && lineDecorations[lineDecorationIndex].startOffset === lastTokenEndIndex) {
 			const lineDecoration = lineDecorations[lineDecorationIndex];
-			result[resultLen++] = new LinePart(lastResultEndIndex, lineDecoration.className, lineDecoration.metadata);
+			result[resultLen++] = new LinePart(lastResultEndIndex, lineDecoration.className, lineDecoration.metadata, false);
 			lineDecorationIndex++;
 		}
 	}
@@ -946,12 +951,17 @@ function _renderLine(input: ResolvedRenderLineInput, sb: IStringBuilder): Render
 		const part = parts[partIndex];
 		const partEndIndex = part.endIndex;
 		const partType = part.type;
+		const partContainsRTL = part.containsRTL;
 		const partRendersWhitespace = (renderWhitespace !== RenderWhitespace.None && part.isWhitespace());
 		const partRendersWhitespaceWithWidth = partRendersWhitespace && !fontIsMonospace && (partType === 'mtkw'/*only whitespace*/ || !containsForeignElements);
 		const partIsEmptyAndHasPseudoAfter = (charIndex === partEndIndex && part.isPseudoAfter());
 		charOffsetInPart = 0;
 
-		sb.appendASCIIString('<span class="');
+		sb.appendASCIIString('<span ');
+		if (partContainsRTL) {
+			sb.appendASCIIString('dir="auto" ');
+		}
+		sb.appendASCIIString('class="');
 		sb.appendASCIIString(partRendersWhitespaceWithWidth ? 'mtkz' : partType);
 		sb.appendASCII(CharCode.DoubleQuote);
 


Test patch:
diff --git a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts
--- a/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts
+++ b/src/vs/editor/test/common/viewLayout/viewLineRenderer.test.ts
@@ -465,8 +465,8 @@ suite('viewLineRenderer.renderLine', () => {
 
 		const expectedOutput = [
 			'<span class="mtk6">var</span>',
-			'<span class="mtk1">\u00a0קודמות\u00a0=\u00a0</span>',
-			'<span class="mtk20">"מיותר\u00a0קודמות\u00a0צ\'ט\u00a0של,\u00a0אם\u00a0לשון\u00a0העברית\u00a0שינויים\u00a0ויש,\u00a0אם"</span>',
+			'<span dir="auto" class="mtk1">\u00a0קודמות\u00a0=\u00a0</span>',
+			'<span dir="auto" class="mtk20">"מיותר\u00a0קודמות\u00a0צ\'ט\u00a0של,\u00a0אם\u00a0לשון\u00a0העברית\u00a0שינויים\u00a0ויש,\u00a0אם"</span>',
 			'<span class="mtk1">;</span>'
 		].join('');
 
@@ -496,6 +496,63 @@ suite('viewLineRenderer.renderLine', () => {
 		assert.strictEqual(_actual.containsRTL, true);
 	});
 
+	test('issue #137036: Issue in RTL languages in recent versions', () => {
+		const lineText = '<option value=\"العربية\">العربية</option>';
+
+		const lineParts = createViewLineTokens([
+			createPart(1, 2),
+			createPart(7, 3),
+			createPart(8, 4),
+			createPart(13, 5),
+			createPart(14, 4),
+			createPart(23, 6),
+			createPart(24, 2),
+			createPart(31, 4),
+			createPart(33, 2),
+			createPart(39, 3),
+			createPart(40, 2),
+		]);
+
+		const expectedOutput = [
+			'<span class="mtk2">&lt;</span>',
+			'<span class="mtk3">option</span>',
+			'<span class="mtk4">\u00a0</span>',
+			'<span class="mtk5">value</span>',
+			'<span class="mtk4">=</span>',
+			'<span dir="auto" class="mtk6">"العربية"</span>',
+			'<span class="mtk2">&gt;</span>',
+			'<span dir="auto" class="mtk4">العربية</span>',
+			'<span class="mtk2">&lt;/</span>',
+			'<span class="mtk3">option</span>',
+			'<span class="mtk2">&gt;</span>',
+		].join('');
+
+		const _actual = renderViewLine(new RenderLineInput(
+			false,
+			true,
+			lineText,
+			false,
+			false,
+			true,
+			0,
+			lineParts,
+			[],
+			4,
+			0,
+			10,
+			10,
+			10,
+			-1,
+			'none',
+			false,
+			false,
+			null
+		));
+
+		assert.strictEqual(_actual.html, '<span dir="ltr">' + expectedOutput + '</span>');
+		assert.strictEqual(_actual.containsRTL, true);
+	});
+
 	test('issue #6885: Splits large tokens', () => {
 		//                                                                                                                  1         1         1
 		//                        1         2         3         4         5         6         7         8         9         0         1         2
@@ -681,7 +738,7 @@ suite('viewLineRenderer.renderLine', () => {
 		const lineText = 'את גרמנית בהתייחסות שמו, שנתי המשפט אל חפש, אם כתב אחרים ולחבר. של התוכן אודות בויקיפדיה כלל, של עזרה כימיה היא. על עמוד יוצרים מיתולוגיה סדר, אם שכל שתפו לעברית שינויים, אם שאלות אנגלית עזה. שמות בקלות מה סדר.';
 		const lineParts = createViewLineTokens([createPart(lineText.length, 1)]);
 		const expectedOutput = [
-			'<span class="mtk1">את\u00a0גרמנית\u00a0בהתייחסות\u00a0שמו,\u00a0שנתי\u00a0המשפט\u00a0אל\u00a0חפש,\u00a0אם\u00a0כתב\u00a0אחרים\u00a0ולחבר.\u00a0של\u00a0התוכן\u00a0אודות\u00a0בויקיפדיה\u00a0כלל,\u00a0של\u00a0עזרה\u00a0כימיה\u00a0היא.\u00a0על\u00a0עמוד\u00a0יוצרים\u00a0מיתולוגיה\u00a0סדר,\u00a0אם\u00a0שכל\u00a0שתפו\u00a0לעברית\u00a0שינויים,\u00a0אם\u00a0שאלות\u00a0אנגלית\u00a0עזה.\u00a0שמות\u00a0בקלות\u00a0מה\u00a0סדר.</span>'
+			'<span dir="auto" class="mtk1">את\u00a0גרמנית\u00a0בהתייחסות\u00a0שמו,\u00a0שנתי\u00a0המשפט\u00a0אל\u00a0חפש,\u00a0אם\u00a0כתב\u00a0אחרים\u00a0ולחבר.\u00a0של\u00a0התוכן\u00a0אודות\u00a0בויקיפדיה\u00a0כלל,\u00a0של\u00a0עזרה\u00a0כימיה\u00a0היא.\u00a0על\u00a0עמוד\u00a0יוצרים\u00a0מיתולוגיה\u00a0סדר,\u00a0אם\u00a0שכל\u00a0שתפו\u00a0לעברית\u00a0שינויים,\u00a0אם\u00a0שאלות\u00a0אנגלית\u00a0עזה.\u00a0שמות\u00a0בקלות\u00a0מה\u00a0סדר.</span>'
 		];
 		const actual = renderViewLine(new RenderLineInput(
 			false,