Repository: KaTeX/KaTeX (js)

Issue:
problem with parsing text with mixed $$ and $ delimiters
> Before reporting a bug
- [x] Check [common issues](https://katex.org/docs/issues.html).
- [x] Check the bug is reproducible in [the demo](https://katex.org). If not, check KaTeX is up-to-date and installed correctly.
- [x] Search for [existing issues](https://github.com/KaTeX/KaTeX/issues).

**Describe the bug:**
In LaTeX source texts it is common to see inline math like `$x$$y$` which is equivalent to `$xy$`
mixed with display math.

However, such math does not render correctly with the autorender extension, when the delimiter option is set to
```
{delimiters: [
  {left: '$$', right: '$$', display: true},
  {left: '$', right: '$', display: false},
  {left: '\\(', right: '\\)', display: false},
  {left: '\\[', right: '\\]', display: true}
]}
```

**To Reproduce:**
Steps to reproduce the behavior:

display this file in a supported browser
```html
<!DOCTYPE html>
<!-- KaTeX requires the use of the HTML5 doctype. Without it, KaTeX may not render properly -->
<html>
  <head>
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.12.0/dist/katex.min.css" integrity="sha384-AfEj0r4/OFrOo5t7NnNe46zW/tFg\
W6x/bCJG8FqQCEo3+Aro6EYUG4+cU+KJWu/X" crossorigin="anonymous">

    <!-- The loading of KaTeX is deferred to speed up page rendering -->
    <script defer src="https://cdn.jsdelivr.net/npm/katex@0.12.0/dist/katex.min.js" integrity="sha384-g7c+Jr9ZivxKLnZTDUhnkOnsh30B4H0rpLUpJ4j\
AIKs4fnJI+sEnkvrMWph2EDg4" crossorigin="anonymous"></script>

    <!-- To automatically render math in text elements, include the auto-render extension: -->
    <script defer
    src="https://cdn.jsdelivr.net/npm/katex@0.12.0/dist/contrib/auto-render.min.js"
    integrity="sha384-mll67QQFJfxn0IYznZYonOWZ644AWYC+Pt2cHqMaRhXVrursRwvLnLaebdGIlYNa"
            crossorigin="anonymous" onload="renderMathInElement(document.body, {delimiters: [
  {left: '$$', right: '$$', display: true},
  {left: '$', right: '$', display: false},
  {left: '\\(', right: '\\)', display: false},
  {left: '\\[', right: '\\]', display: true}
]});"></script>
  </head>
  <body>
    <h1>A note on discrete $$\mathcal{R}$$ symmetries in $\mathbb{Z}$$_{6}$-II orbifolds with Wilson lines</h1>

  </body>
</html>
```

**Expected behavior:**
both the display math and the inline math should be rendered. It is not, but the `$$` in the middle of  `$\mathbb{Z}$$_{6}$` disappears. 

Omitting the `$$` (which is a NOOP) in the middle results in correct rendering.
Introducing a space `$\mathbb{Z}$ $_{6}$` also renders the inline math, albeit with a gratuitous space.


**Screenshots:**
![katex-autorender](https://user-images.githubusercontent.com/766159/93270138-af22a280-f76d-11ea-85de-e2f06b7efa83.png)

**Environment (please complete the following information):**
 - KaTeX Version: v0.12.0
 - Device: Linux desktop
 - OS: Fedora 32
 - Browser: Chrome
 - Version: 85.0.4183.102 (Official Build) (64-bit)


**Additional Context:**

We use KaTeX for rendering in a large repository of scientific papers. We don't control the TeX source, and hence can't avoid this mix of delimiters. 


Pull request:
fix: rewrite of splitAtDelimiters.js -- new fix for #2523
Resolves #2523 as well as other related auto-render problems (mentioned in the comments of the issue).
The text is only parsed once for all delimiters at once (this is presumably faster than the old, sequential way).
Everything works fine now according to my limited testing, but this is not my project so probably requires more testing.


Patch:
diff --git a/contrib/auto-render/auto-render.js b/contrib/auto-render/auto-render.js
index 1eed7dac..d1c830d1 100644
--- a/contrib/auto-render/auto-render.js
+++ b/contrib/auto-render/auto-render.js
@@ -3,22 +3,11 @@
 import katex from "katex";
 import splitAtDelimiters from "./splitAtDelimiters";
 
-const splitWithDelimiters = function(text, delimiters) {
-    let data = [{type: "text", data: text}];
-    for (let i = 0; i < delimiters.length; i++) {
-        const delimiter = delimiters[i];
-        data = splitAtDelimiters(
-            data, delimiter.left, delimiter.right,
-            delimiter.display || false);
-    }
-    return data;
-};
-
 /* Note: optionsCopy is mutated by this method. If it is ever exposed in the
  * API, we should copy it before mutating.
  */
 const renderMathInText = function(text, optionsCopy) {
-    const data = splitWithDelimiters(text, optionsCopy.delimiters);
+    const data = splitAtDelimiters(text, optionsCopy.delimiters);
     if (data.length === 1 && data[0].type === 'text') {
         // There is no formula in the text.
         // Let's return null which means there is no need to replace
@@ -48,7 +37,7 @@ const renderMathInText = function(text, optionsCopy) {
                 }
                 optionsCopy.errorCallback(
                     "KaTeX auto-render: Failed to parse `" + data[i].data +
-                    "` with ",
+                        "` with ",
                     e
                 );
                 fragment.appendChild(document.createTextNode(data[i].rawData));
@@ -76,8 +65,8 @@ const renderElem = function(elem, optionsCopy) {
             const className = ' ' + childNode.className + ' ';
             const shouldRender = optionsCopy.ignoredTags.indexOf(
                 childNode.nodeName.toLowerCase()) === -1 &&
-                    optionsCopy.ignoredClasses.every(
-                        x => className.indexOf(' ' + x + ' ') === -1);
+                  optionsCopy.ignoredClasses.every(
+                      x => className.indexOf(' ' + x + ' ') === -1);
 
             if (shouldRender) {
                 renderElem(childNode, optionsCopy);
@@ -107,11 +96,7 @@ const renderMathInElement = function(elem, options) {
         {left: "\\(", right: "\\)", display: false},
         // LaTeX uses $…$, but it ruins the display of normal `$` in text:
         // {left: "$", right: "$", display: false},
-
-        //  \[…\] must come last in this array. Otherwise, renderMathInElement
-        //  will search for \[ before it searches for $$ or  \(
-        // That makes it susceptible to finding a \\[0.3em] row delimiter and
-        // treating it as if it were the start of a KaTeX math zone.
+	// $ must come after $$
         {left: "\\[", right: "\\]", display: true},
     ];
     optionsCopy.ignoredTags = optionsCopy.ignoredTags || [
diff --git a/contrib/auto-render/splitAtDelimiters.js b/contrib/auto-render/splitAtDelimiters.js
index ef8f3eed..b310489c 100644
--- a/contrib/auto-render/splitAtDelimiters.js
+++ b/contrib/auto-render/splitAtDelimiters.js
@@ -27,76 +27,53 @@ const findEndOfMath = function(delimiter, text, startIndex) {
     return -1;
 };
 
-const splitAtDelimiters = function(startData, leftDelim, rightDelim, display) {
-    const finalData = [];
-
-    for (let i = 0; i < startData.length; i++) {
-        if (startData[i].type === "text") {
-            const text = startData[i].data;
-
-            let lookingForLeft = true;
-            let currIndex = 0;
-            let nextIndex;
-
-            nextIndex = text.indexOf(leftDelim);
-            if (nextIndex !== -1) {
-                currIndex = nextIndex;
-                finalData.push({
-                    type: "text",
-                    data: text.slice(0, currIndex),
-                });
-                lookingForLeft = false;
-            }
-
-            while (true) {
-                if (lookingForLeft) {
-                    nextIndex = text.indexOf(leftDelim, currIndex);
-                    if (nextIndex === -1) {
-                        break;
-                    }
-
-                    finalData.push({
-                        type: "text",
-                        data: text.slice(currIndex, nextIndex),
-                    });
-
-                    currIndex = nextIndex;
-                } else {
-                    nextIndex = findEndOfMath(
-                        rightDelim,
-                        text,
-                        currIndex + leftDelim.length);
-                    if (nextIndex === -1) {
-                        break;
-                    }
-
-                    finalData.push({
-                        type: "math",
-                        data: text.slice(
-                            currIndex + leftDelim.length,
-                            nextIndex),
-                        rawData: text.slice(
-                            currIndex,
-                            nextIndex + rightDelim.length),
-                        display: display,
-                    });
+const escapeRegex = function(string) {
+    return string.replace(/[-/\\^$*+?.()|[\]{}]/g, "\\$&");
+};
 
-                    currIndex = nextIndex + rightDelim.length;
-                }
+const splitAtDelimiters = function(text, delimiters) {
+    let index;
+    const data = [];
 
-                lookingForLeft = !lookingForLeft;
-            }
+    const regexLeft = new RegExp(
+        "(" + delimiters.map((x) => escapeRegex(x.left)).join("|") + ")"
+    );
 
-            finalData.push({
+    while (true) {
+        index = text.search(regexLeft);
+        if (index === -1) {
+            break;
+        }
+        if (index > 0) {
+            data.push({
                 type: "text",
-                data: text.slice(currIndex),
+                data: text.slice(0, index),
             });
-        } else {
-            finalData.push(startData[i]);
+            text = text.slice(index); // now text starts with delimiter
         }
+        // ... so this always succeeds:
+        const i = delimiters.findIndex((delim) => text.startsWith(delim.left));
+        index = findEndOfMath(delimiters[i].right, text, delimiters[i].left.length);
+        if (index === -1) {
+            break;
+        }
+        data.push({
+            type: "math",
+            data: text.slice(delimiters[i].left.length, index),
+            rawData: text.slice(0, index + delimiters[i].right.length),
+            display: delimiters[i].display,
+        });
+        text = text.slice(index + delimiters[i].right.length);
+    }
+
+    if (text !== "") {
+        data.push({
+            type: "text",
+            data: text,
+        });
     }
 
-    return finalData;
+    return data;
 };
 
 export default splitAtDelimiters;


Test patch:
diff --git a/contrib/auto-render/test/auto-render-spec.js b/contrib/auto-render/test/auto-render-spec.js
index 01eb314f..2665754b 100644
--- a/contrib/auto-render/test/auto-render-spec.js
+++ b/contrib/auto-render/test/auto-render-spec.js
@@ -6,17 +6,16 @@ beforeEach(function() {
         toSplitInto: function(actual, left, right, result) {
             const message = {
                 pass: true,
-                message: "'" + actual + "' split correctly",
+                message: () => "'" + actual + "' split correctly",
             };
 
-            const startData = [{type: "text", data: actual}];
-
             const split =
-                splitAtDelimiters(startData, left, right, false);
+                  splitAtDelimiters(actual,
+                                    [{left: left, right: right, display: false}]);
 
             if (split.length !== result.length) {
                 message.pass = false;
-                message.message = "Different number of splits: " +
+                message.message = () => "Different number of splits: " +
                     split.length + " vs. " + result.length + " (" +
                     JSON.stringify(split) + " vs. " +
                     JSON.stringify(result) + ")";
@@ -43,7 +42,7 @@ beforeEach(function() {
 
                 if (!good) {
                     message.pass = false;
-                    message.message = "Difference at split " +
+                    message.message = () => "Difference at split " +
                         (i + 1) + ": " + JSON.stringify(real) +
                         " vs. " + JSON.stringify(correct) +
                         " (" + diff + " differs)";
@@ -146,6 +145,19 @@ describe("A delimiter splitter", function() {
             ]);
     });
 
+    it("correctly processes sequences of $..$", function() {
+        expect("$hello$$world$$boo$").toSplitInto(
+            "$", "$",
+            [
+                {type: "math", data: "hello",
+                    rawData: "$hello$", display: false},
+                {type: "math", data: "world",
+                    rawData: "$world$", display: false},
+                {type: "math", data: "boo",
+                    rawData: "$boo$", display: false},
+            ]);
+    });
+
     it("doesn't split at escaped delimiters", function() {
         expect("hello ( world \\) ) boo").toSplitInto(
             "(", ")",
@@ -157,14 +169,14 @@ describe("A delimiter splitter", function() {
             ]);
 
         /* TODO(emily): make this work maybe?
-        expect("hello \\( ( world ) boo").toSplitInto(
-            "(", ")",
-            [
-                {type: "text", data: "hello \\( "},
-                {type: "math", data: " world ",
-                    rawData: "( world )", display: false},
-                {type: "text", data: " boo"},
-            ]);
+           expect("hello \\( ( world ) boo").toSplitInto(
+           "(", ")",
+           [
+           {type: "text", data: "hello \\( "},
+           {type: "math", data: " world ",
+           rawData: "( world )", display: false},
+           {type: "text", data: " boo"},
+           ]);
         */
     });
 
@@ -179,10 +191,20 @@ describe("A delimiter splitter", function() {
             ]);
     });
 
+    it("ignores \\$", function() {
+        expect("$x = \\$5$").toSplitInto(
+            "$", "$",
+            [
+                {type: "math", data: "x = \\$5",
+                    rawData: "$x = \\$5$", display: false},
+            ]);
+    });
+
     it("remembers which delimiters are display-mode", function() {
-        const startData = [{type: "text", data: "hello ( world ) boo"}];
+        const startData = "hello ( world ) boo";
 
-        expect(splitAtDelimiters(startData, "(", ")", true)).toEqual(
+        expect(splitAtDelimiters(startData,
+                                 [{left:"(", right:")", display:true}])).toEqual(
             [
                 {type: "text", data: "hello "},
                 {type: "math", data: " world ",
@@ -191,42 +213,52 @@ describe("A delimiter splitter", function() {
             ]);
     });
 
-    it("works with more than one start datum", function() {
-        const startData = [
-            {type: "text", data: "hello ( world ) boo"},
-            {type: "math", data: "math", rawData: "(math)", display: true},
-            {type: "text", data: "hello ( world ) boo"},
-        ];
-
-        expect(splitAtDelimiters(startData, "(", ")", false)).toEqual(
+    it("handles nested delimiters irrespective of order", function() {
+        expect(splitAtDelimiters("$\\fbox{\\(hi\\)}$",
             [
-                {type: "text", data: "hello "},
-                {type: "math", data: " world ",
-                    rawData: "( world )", display: false},
-                {type: "text", data: " boo"},
-                {type: "math", data: "math", rawData: "(math)", display: true},
-                {type: "text", data: "hello "},
-                {type: "math", data: " world ",
-                    rawData: "( world )", display: false},
-                {type: "text", data: " boo"},
+                                     {left:"\\(", right:"\\)", display:false},
+                                     {left:"$", right:"$", display:false},
+            ])).toEqual(
+            [
+                {type: "math", data: "\\fbox{\\(hi\\)}",
+                    rawData: "$\\fbox{\\(hi\\)}$", display: false},
+            ]);
+        expect(splitAtDelimiters("\\(\\fbox{$hi$}\\)",
+            [
+                {left:"\\(", right:"\\)", display:false},
+                {left:"$", right:"$", display:false},
+            ])).toEqual(
+            [
+                {type: "math", data: "\\fbox{$hi$}",
+                    rawData: "\\(\\fbox{$hi$}\\)", display: false},
             ]);
     });
 
-    it("doesn't do splitting inside of math nodes", function() {
-        const startData = [
-            {type: "text", data: "hello ( world ) boo"},
-            {type: "math", data: "hello ( world ) boo",
-                rawData: "(hello ( world ) boo)", display: true},
-        ];
-
-        expect(splitAtDelimiters(startData, "(", ")", false)).toEqual(
+    it("handles a mix of $ and $$", function() {
+        expect(splitAtDelimiters("$hello$world$$boo$$",
             [
-                {type: "text", data: "hello "},
-                {type: "math", data: " world ",
-                    rawData: "( world )", display: false},
-                {type: "text", data: " boo"},
-                {type: "math", data: "hello ( world ) boo",
-                    rawData: "(hello ( world ) boo)", display: true},
+                {left:"$$", right:"$$", display:true},
+                {left:"$", right:"$", display:false},
+            ])).toEqual(
+            [
+                {type: "math", data: "hello",
+                    rawData: "$hello$", display: false},
+                                         {type: "text", data: "world"},
+                {type: "math", data: "boo",
+                    rawData: "$$boo$$", display: true},
+            ]);
+        expect(splitAtDelimiters("$hello$$world$$$boo$$",
+            [
+                {left:"$$", right:"$$", display:true},
+                {left:"$", right:"$", display:false},
+            ])).toEqual(
+            [
+                {type: "math", data: "hello",
+                    rawData: "$hello$", display: false},
+                {type: "math", data: "world",
+                    rawData: "$world$", display: false},
+                {type: "math", data: "boo",
+                    rawData: "$$boo$$", display: true},
             ]);
     });
 });