Repository: alecthomas/chroma (go)

Issue:
Unterminated Raku heredoc produces a panic
**Describe the bug**

An unterminated heredoc in Raku causes chroma to panic.

**To Reproduce**

```diff
--- a/lexers/testdata/raku.actual
+++ b/lexers/testdata/raku.actual
@@ -495,6 +495,8 @@ say Calculator.parse('2 + 3', actions => Calculations).made;
 
 put Date.today.later(:2years).year;
 
+say q:to/Foo/;
+
 =finish
 C<say> Date.today.year;
 # Output: 2020
```

```
$ go test -run ^TestLexers$ ./lexers
--- FAIL: TestLexers (13.90s)
    --- FAIL: TestLexers/Raku/testdata/raku.actual (4.86s)
panic: runtime error: slice bounds out of range [12136:12133] [recovered]
	panic: runtime error: slice bounds out of range [12136:12133]

goroutine 177 [running]:
testing.tRunner.func1.2({0x7934a0, 0xc0017bac30})
	/usr/lib/go/src/testing/testing.go:1209 +0x24e
testing.tRunner.func1()
	/usr/lib/go/src/testing/testing.go:1212 +0x218
panic({0x7934a0, 0xc0017bac30})
	/usr/lib/go/src/runtime/panic.go:1038 +0x215
github.com/alecthomas/chroma/lexers/r.rakuRules.func1.1(0xc001784210)
	/home/jraspass/chroma/lexers/r/raku.go:541 +0x1145
github.com/alecthomas/chroma.MutatorFunc.Mutate(0xc00132e950, 0x2fc0)
	/home/jraspass/chroma/mutators.go:24 +0x1f
github.com/alecthomas/chroma.(*LexerState).Iterator(0xc001784210)
	/home/jraspass/chroma/regexp.go:363 +0x3a2
github.com/alecthomas/chroma.(*coalescer).Tokenise.func1()
	/home/jraspass/chroma/coalesce.go:15 +0x2f
github.com/alecthomas/chroma.Tokenise({0x8e5188, 0xc000f4db60}, 0x3044, {0xc001254000, 0x0})
	/home/jraspass/chroma/regexp.go:191 +0x91
github.com/alecthomas/chroma/lexers_test.FileTest.func1(0x0)
	/home/jraspass/chroma/lexers/lexers_test.go:54 +0xdf
testing.tRunner(0xc001791520, 0xc0014e6600)
	/usr/lib/go/src/testing/testing.go:1259 +0x102
created by testing.(*T).Run
	/usr/lib/go/src/testing/testing.go:1306 +0x35a
FAIL	github.com/alecthomas/chroma/lexers	13.907s
FAIL
```

Pull request:
Raku: Fix unterminated heredoc, fixes #547
- Fix heredoc
- Move testdata to raku directory
- Add testdata for unterminated heredoc

Patch:
diff --git a/lexers/r/raku.go b/lexers/r/raku.go
index 5c4fea3..349cb98 100644
--- a/lexers/r/raku.go
+++ b/lexers/r/raku.go
@@ -513,14 +513,24 @@ func rakuRules() Rules {
 
 			adverbre := regexp.MustCompile(`:to\b|:heredoc\b`)
 			var heredocTerminator []rune
+			var endHeredocPos int
 			if adverbre.MatchString(string(adverbs)) {
-				heredocTerminator = text[state.Pos:endPos]
-				if len(heredocTerminator) > 0 {
-					endHeredocPos := indexAt(text[endPos:], heredocTerminator, 0)
+				if endPos != len(text) {
+					heredocTerminator = text[state.Pos:endPos]
 					nChars = len(heredocTerminator)
-					endPos += endHeredocPos
 				} else {
-					endPos = len(text)
+					endPos = state.Pos + 1
+					heredocTerminator = []rune{}
+					nChars = 0
+				}
+
+				if nChars > 0 {
+					endHeredocPos = indexAt(text[endPos:], heredocTerminator, 0)
+					if endHeredocPos > -1 {
+						endPos += endHeredocPos
+					} else {
+						endPos = len(text)
+					}
 				}
 			}
 
@@ -532,7 +542,7 @@ func rakuRules() Rules {
 			case rakuQuote:
 				if len(heredocTerminator) > 0 {
 					// Length of heredoc terminator + closing chars + `;`
-					heredocFristPunctuationLen := len(heredocTerminator) + len(openingChars) + 1
+					heredocFristPunctuationLen := nChars + len(openingChars) + 1
 
 					state.NamedGroups[`opening_delimiters`] = string(openingChars) +
 						string(text[state.Pos:state.Pos+heredocFristPunctuationLen])
@@ -540,10 +550,14 @@ func rakuRules() Rules {
 					state.NamedGroups[`value`] =
 						string(text[state.Pos+heredocFristPunctuationLen : endPos])
 
-					state.NamedGroups[`closing_delimiters`] = string(heredocTerminator)
+					if endHeredocPos > -1 {
+						state.NamedGroups[`closing_delimiters`] = string(heredocTerminator)
+					}
 				} else {
 					state.NamedGroups[`value`] = textBetweenBrackets
-					state.NamedGroups[`closing_delimiters`] = string(closingChars)
+					if nChars > 0 {
+						state.NamedGroups[`closing_delimiters`] = string(closingChars)
+					}
 				}
 			default:
 				state.Groups = []string{state.Groups[0] + string(text[state.Pos:endPos+nChars])}


Test patch:
diff --git a/lexers/testdata/raku.actual b/lexers/testdata/raku/raku.actual
similarity index 100%
rename from lexers/testdata/raku.actual
rename to lexers/testdata/raku/raku.actual
diff --git a/lexers/testdata/raku.expected b/lexers/testdata/raku/raku.expected
similarity index 100%
rename from lexers/testdata/raku.expected
rename to lexers/testdata/raku/raku.expected
diff --git a/lexers/testdata/raku/unterminated_heredoc.actual b/lexers/testdata/raku/unterminated_heredoc.actual
new file mode 100644
index 0000000..319757c
--- /dev/null
+++ b/lexers/testdata/raku/unterminated_heredoc.actual
@@ -0,0 +1,8 @@
+say q:to//;
+
+say 'something';
+
+# Unterminated heredoc
+say q:to/Foo/;
+
+say 'something';
diff --git a/lexers/testdata/raku/unterminated_heredoc.expected b/lexers/testdata/raku/unterminated_heredoc.expected
new file mode 100644
index 0000000..8f915f8
--- /dev/null
+++ b/lexers/testdata/raku/unterminated_heredoc.expected
@@ -0,0 +1,23 @@
+[
+  {"type":"NameBuiltin","value":"say"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"q"},
+  {"type":"LiteralStringAffix","value":":to"},
+  {"type":"Punctuation","value":"//"},
+  {"type":"LiteralString","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"NameBuiltin","value":"say"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"'"},
+  {"type":"LiteralStringSingle","value":"something"},
+  {"type":"Punctuation","value":"';"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"# Unterminated heredoc"},
+  {"type":"Text","value":"\n"},
+  {"type":"NameBuiltin","value":"say"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"q"},
+  {"type":"LiteralStringAffix","value":":to"},
+  {"type":"Punctuation","value":"/Foo/;"},
+  {"type":"LiteralString","value":"\n\nsay 'something';\n"}
+]