Repository: KaTeX/KaTeX (js)

Issue:
Error with single backslash at end of line
**Describe the bug:**
(Copying from [a fixed pandoc issue](https://github.com/jgm/pandoc/issues/4134):)

> In LaTeX you can use a backslash `\` followed by any whitespace to produce a regular space, as opposed to an inter-sentence space. The whitespace needn't be a space; it could be a line break.

KaTeX treats a single `\` at the end of the line as an error.

**To Reproduce:**
Steps to reproduce the behavior:
1. Go to https://katex.org
2. Type a single `\` followed immediately by a line break.
3. See `KaTeX parse error: Undefined control sequence: \ at position 1: \̲ ̲`.

**Expected behavior:**
It should produce a regular space instead of a parse error.

**Screenshots:**

<img width="791" alt="Screenshot" src="https://user-images.githubusercontent.com/18308236/112727613-90267980-8f23-11eb-83e9-080c8eef33b7.png">

**Environment:**
 - KaTeX Version: 0.13.0

**Additional context:**
MathJax allows a single backslash at the end of the line.

Pull request description:
fix: Correctly parse \ followed by whitespace
LaTeX parses `\` followed by whitespace including up to one newline as equivalent to `\ `.  (With multiple newlines, you get paragraph breaks. These aren't supported by KaTeX, but I figured it might one day...)

Fix #2860.

Patch:
diff --git a/src/Lexer.js b/src/Lexer.js
index 7cf96edd..849eee63 100644
--- a/src/Lexer.js
+++ b/src/Lexer.js
@@ -28,8 +28,16 @@ import type Settings from "./Settings";
  * - does not match bare surrogate code units
  * - matches any BMP character except for those just described
  * - matches any valid Unicode surrogate pair
- * - matches a backslash followed by one or more letters
- * - matches a backslash followed by any BMP character, including newline
+ * - matches a backslash followed by one or more whitespace characters
+ * - matches a backslash followed by one or more letters then whitespace
+ * - matches a backslash followed by any BMP character
+ * Capturing groups:
+ *   [1] regular whitespace
+ *   [2] backslash followed by whitespace
+ *   [3] anything else, which may include:
+ *     [4] left character of \verb*
+ *     [5] left character of \verb
+ *     [6] backslash followed by word, excluding any trailing whitespace
  * Just because the Lexer matches something doesn't mean it's valid input:
  * If there is no matching function or symbol definition, the Parser will
  * still reject the input.
@@ -38,19 +46,19 @@ const spaceRegexString = "[ \r\n\t]";
 const controlWordRegexString = "\\\\[a-zA-Z@]+";
 const controlSymbolRegexString = "\\\\[^\uD800-\uDFFF]";
 const controlWordWhitespaceRegexString =
-    `${controlWordRegexString}${spaceRegexString}*`;
-const controlWordWhitespaceRegex = new RegExp(
-    `^(${controlWordRegexString})${spaceRegexString}*$`);
+    `(${controlWordRegexString})${spaceRegexString}*`;
+const controlSpaceRegexString = "\\\\(\n|[ \r\t]+\n?)[ \r\t]*";
 const combiningDiacriticalMarkString = "[\u0300-\u036f]";
 export const combiningDiacriticalMarksEndRegex: RegExp =
     new RegExp(`${combiningDiacriticalMarkString}+$`);
 const tokenRegexString = `(${spaceRegexString}+)|` +  // whitespace
+    `${controlSpaceRegexString}|` +                   // \whitespace
     "([!-\\[\\]-\u2027\u202A-\uD7FF\uF900-\uFFFF]" +  // single codepoint
     `${combiningDiacriticalMarkString}*` +            // ...plus accents
     "|[\uD800-\uDBFF][\uDC00-\uDFFF]" +               // surrogate pair
     `${combiningDiacriticalMarkString}*` +            // ...plus accents
-    "|\\\\verb\\*([^]).*?\\3" +                       // \verb*
-    "|\\\\verb([^*a-zA-Z]).*?\\4" +                   // \verb unstarred
+    "|\\\\verb\\*([^]).*?\\4" +                       // \verb*
+    "|\\\\verb([^*a-zA-Z]).*?\\5" +                   // \verb unstarred
     "|\\\\operatorname\\*" +                          // \operatorname*
     `|${controlWordWhitespaceRegexString}` +          // \macroName + spaces
     `|${controlSymbolRegexString})`;                  // \\, \', etc.
@@ -94,7 +102,7 @@ export default class Lexer implements LexerInterface {
                 `Unexpected character: '${input[pos]}'`,
                 new Token(input[pos], new SourceLocation(this, pos, pos + 1)));
         }
-        let text = match[2] || " ";
+        const text = match[6] || match[3] || (match[2] ? "\\ " : " ");
 
         if (this.catcodes[text] === 14) { // comment character
             const nlIndex = input.indexOf('\n', this.tokenRegex.lastIndex);
@@ -109,12 +117,6 @@ export default class Lexer implements LexerInterface {
             return this.lex();
         }
 
-        // Trim any trailing whitespace from control word match
-        const controlMatch = text.match(controlWordWhitespaceRegex);
-        if (controlMatch) {
-            text = controlMatch[1];
-        }
-
         return new Token(text, new SourceLocation(this, pos,
             this.tokenRegex.lastIndex));
     }


Test patch:
diff --git a/test/katex-spec.js b/test/katex-spec.js
index 421e977f..51979469 100644
--- a/test/katex-spec.js
+++ b/test/katex-spec.js
@@ -678,7 +678,7 @@ describe("A text parser", function() {
     const noBraceTextExpression = r`\text x`;
     const nestedTextExpression =
         r`\text{a {b} \blue{c} \textcolor{#fff}{x} \llap{x}}`;
-    const spaceTextExpression = r`\text{  a \ }`;
+    const spaceTextExpression = r`\text{  a \  }`;
     const leadingSpaceTextExpression = r`\text {moo}`;
     const badTextExpression = r`\text{a b%}`;
     const badFunctionExpression = r`\text{\sqrt{x}}`;
@@ -722,12 +722,17 @@ describe("A text parser", function() {
         const parse = getParsed(spaceTextExpression)[0];
         const group = parse.body;
 
+        expect(group.length).toEqual(4);
         expect(group[0].type).toEqual("spacing");
         expect(group[1].type).toEqual("textord");
         expect(group[2].type).toEqual("spacing");
         expect(group[3].type).toEqual("spacing");
     });
 
+    it("should handle backslash followed by newline", () => {
+        expect("\\text{\\ \t\r \n \t\r  }").toParseLike("\\text{\\ }");
+    });
+
     it("should accept math mode tokens after its argument", function() {
         expect(mathTokenAfterText).toParse();
     });