Repository: alecthomas/chroma
Pull request #370 (2020-06-04 20:08:54)

added curly braces to toml lexer
I added curly braces to the TOML lexer. This should fix the error formatting of inline tables.
https://github.com/toml-lang/toml#user-content-inline-table

I have no go experience and did not test the build, so please excuse if this brakes everything ;-)


Patch:
diff --git a/.golangci.yml b/.golangci.yml
index b1e51f3..9a73537 100644
--- a/.golangci.yml
+++ b/.golangci.yml
@@ -20,6 +20,11 @@ linters:
     - wsl
     - gomnd
     - gocognit
+    - goerr113
+    - nolintlint
+    - testpackage
+    - godot
+    - nestif
 
 linters-settings:
   govet:
diff --git a/.travis.yml b/.travis.yml
index 9216ec9..701c623 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,7 +4,7 @@ go:
   - "1.13.x"
 script:
   - go test -v ./...
-  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s v1.22.2
+  - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s 1.26.0
   - ./bin/golangci-lint run
   - git clean -fdx .
 after_success:
diff --git a/lexers/circular/php.go b/lexers/circular/php.go
index d7f8940..2107cb7 100644
--- a/lexers/circular/php.go
+++ b/lexers/circular/php.go
@@ -1,15 +1,12 @@
 package circular
 
 import (
-	"strings"
-
 	. "github.com/alecthomas/chroma" // nolint
-	"github.com/alecthomas/chroma/lexers/h"
 	"github.com/alecthomas/chroma/lexers/internal"
 )
 
-// PHP lexer.
-var PHP = internal.Register(DelegatingLexer(h.HTML, MustNewLexer(
+// PHP lexer for pure PHP code (not embedded in HTML).
+var PHP = internal.Register(MustNewLexer(
 	&Config{
 		Name:            "PHP",
 		Aliases:         []string{"php", "php3", "php4", "php5"},
@@ -19,73 +16,65 @@ var PHP = internal.Register(DelegatingLexer(h.HTML, MustNewLexer(
 		CaseInsensitive: true,
 		EnsureNL:        true,
 	},
-	Rules{
-		"root": {
-			{`<\?(php)?`, CommentPreproc, Push("php")},
-			{`[^<]+`, Other, nil},
-			{`<`, Other, nil},
-		},
-		"php": {
-			{`\?>`, CommentPreproc, Pop(1)},
-			{`(<<<)([\'"]?)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)(\2\n.*?\n\s*)(\3)(;?)(\n)`, ByGroups(LiteralString, LiteralString, LiteralStringDelimiter, LiteralString, LiteralStringDelimiter, Punctuation, Text), nil},
-			{`\s+`, Text, nil},
-			{`#.*?\n`, CommentSingle, nil},
-			{`//.*?\n`, CommentSingle, nil},
-			{`/\*\*/`, CommentMultiline, nil},
-			{`/\*\*.*?\*/`, LiteralStringDoc, nil},
-			{`/\*.*?\*/`, CommentMultiline, nil},
-			{`(->|::)(\s*)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)`, ByGroups(Operator, Text, NameAttribute), nil},
-			{`[~!%^&*+=|:.<>/@-]+`, Operator, nil},
-			{`\?`, Operator, nil},
-			{`[\[\]{}();,]+`, Punctuation, nil},
-			{`(class)(\s+)`, ByGroups(Keyword, Text), Push("classname")},
-			{`(function)(\s*)(?=\()`, ByGroups(Keyword, Text), nil},
-			{`(function)(\s+)(&?)(\s*)`, ByGroups(Keyword, Text, Operator, Text), Push("functionname")},
-			{`(const)(\s+)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)`, ByGroups(Keyword, Text, NameConstant), nil},
-			{`(and|E_PARSE|old_function|E_ERROR|or|as|E_WARNING|parent|eval|PHP_OS|break|exit|case|extends|PHP_VERSION|cfunction|FALSE|print|for|require|continue|foreach|require_once|declare|return|default|static|do|switch|die|stdClass|echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|virtual|endfor|include_once|while|endforeach|global|endif|list|endswitch|new|endwhile|not|array|E_ALL|NULL|final|php_user_filter|interface|implements|public|private|protected|abstract|clone|try|catch|throw|this|use|namespace|trait|yield|finally)\b`, Keyword, nil},
-			{`(true|false|null)\b`, KeywordConstant, nil},
-			Include("magicconstants"),
-			{`\$\{\$+(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*\}`, NameVariable, nil},
-			{`\$+(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameVariable, nil},
-			{`(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameOther, nil},
-			{`(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?`, LiteralNumberFloat, nil},
-			{`\d+e[+-]?[0-9]+`, LiteralNumberFloat, nil},
-			{`0[0-7]+`, LiteralNumberOct, nil},
-			{`0x[a-f0-9]+`, LiteralNumberHex, nil},
-			{`\d+`, LiteralNumberInteger, nil},
-			{`0b[01]+`, LiteralNumberBin, nil},
-			{`'([^'\\]*(?:\\.[^'\\]*)*)'`, LiteralStringSingle, nil},
-			{"`([^`\\\\]*(?:\\\\.[^`\\\\]*)*)`", LiteralStringBacktick, nil},
-			{`"`, LiteralStringDouble, Push("string")},
-		},
-		"magicfuncs": {
-			{Words(``, `\b`, `__construct`, `__destruct`, `__call`, `__callStatic`, `__get`, `__set`, `__isset`, `__unset`, `__sleep`, `__wakeup`, `__toString`, `__invoke`, `__set_state`, `__clone`, `__debugInfo`), NameFunctionMagic, nil},
-		},
-		"magicconstants": {
-			{Words(``, `\b`, `__LINE__`, `__FILE__`, `__DIR__`, `__FUNCTION__`, `__CLASS__`, `__TRAIT__`, `__METHOD__`, `__NAMESPACE__`), NameConstant, nil},
-		},
-		"classname": {
-			{`(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameClass, Pop(1)},
-		},
-		"functionname": {
-			Include("magicfuncs"),
-			{`(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameFunction, Pop(1)},
-			Default(Pop(1)),
-		},
-		"string": {
-			{`"`, LiteralStringDouble, Pop(1)},
-			{`[^{$"\\]+`, LiteralStringDouble, nil},
-			{`\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})`, LiteralStringEscape, nil},
-			{`\$(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*(\[\S+?\]|->(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)?`, LiteralStringInterpol, nil},
-			{`(\{\$\{)(.*?)(\}\})`, ByGroups(LiteralStringInterpol, UsingSelf("root"), LiteralStringInterpol), nil},
-			{`(\{)(\$.*?)(\})`, ByGroups(LiteralStringInterpol, UsingSelf("root"), LiteralStringInterpol), nil},
-			{`(\$\{)(\S+)(\})`, ByGroups(LiteralStringInterpol, NameVariable, LiteralStringInterpol), nil},
-			{`[${\\]`, LiteralStringDouble, nil},
-		},
+	phpCommonRules.Rename("php", "root"),
+))
+
+var phpCommonRules = Rules{
+	"php": {
+		{`\?>`, CommentPreproc, Pop(1)},
+		{`(<<<)([\'"]?)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)(\2\n.*?\n\s*)(\3)(;?)(\n)`, ByGroups(LiteralString, LiteralString, LiteralStringDelimiter, LiteralString, LiteralStringDelimiter, Punctuation, Text), nil},
+		{`\s+`, Text, nil},
+		{`#.*?\n`, CommentSingle, nil},
+		{`//.*?\n`, CommentSingle, nil},
+		{`/\*\*/`, CommentMultiline, nil},
+		{`/\*\*.*?\*/`, LiteralStringDoc, nil},
+		{`/\*.*?\*/`, CommentMultiline, nil},
+		{`(->|::)(\s*)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)`, ByGroups(Operator, Text, NameAttribute), nil},
+		{`[~!%^&*+=|:.<>/@-]+`, Operator, nil},
+		{`\?`, Operator, nil},
+		{`[\[\]{}();,]+`, Punctuation, nil},
+		{`(class)(\s+)`, ByGroups(Keyword, Text), Push("classname")},
+		{`(function)(\s*)(?=\()`, ByGroups(Keyword, Text), nil},
+		{`(function)(\s+)(&?)(\s*)`, ByGroups(Keyword, Text, Operator, Text), Push("functionname")},
+		{`(const)(\s+)((?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)`, ByGroups(Keyword, Text, NameConstant), nil},
+		{`(and|E_PARSE|old_function|E_ERROR|or|as|E_WARNING|parent|eval|PHP_OS|break|exit|case|extends|PHP_VERSION|cfunction|FALSE|print|for|require|continue|foreach|require_once|declare|return|default|static|do|switch|die|stdClass|echo|else|TRUE|elseif|var|empty|if|xor|enddeclare|include|virtual|endfor|include_once|while|endforeach|global|endif|list|endswitch|new|endwhile|not|array|E_ALL|NULL|final|php_user_filter|interface|implements|public|private|protected|abstract|clone|try|catch|throw|this|use|namespace|trait|yield|finally)\b`, Keyword, nil},
+		{`(true|false|null)\b`, KeywordConstant, nil},
+		Include("magicconstants"),
+		{`\$\{\$+(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*\}`, NameVariable, nil},
+		{`\$+(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameVariable, nil},
+		{`(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameOther, nil},
+		{`(\d+\.\d*|\d*\.\d+)(e[+-]?[0-9]+)?`, LiteralNumberFloat, nil},
+		{`\d+e[+-]?[0-9]+`, LiteralNumberFloat, nil},
+		{`0[0-7]+`, LiteralNumberOct, nil},
+		{`0x[a-f0-9]+`, LiteralNumberHex, nil},
+		{`\d+`, LiteralNumberInteger, nil},
+		{`0b[01]+`, LiteralNumberBin, nil},
+		{`'([^'\\]*(?:\\.[^'\\]*)*)'`, LiteralStringSingle, nil},
+		{"`([^`\\\\]*(?:\\\\.[^`\\\\]*)*)`", LiteralStringBacktick, nil},
+		{`"`, LiteralStringDouble, Push("string")},
+	},
+	"magicfuncs": {
+		{Words(``, `\b`, `__construct`, `__destruct`, `__call`, `__callStatic`, `__get`, `__set`, `__isset`, `__unset`, `__sleep`, `__wakeup`, `__toString`, `__invoke`, `__set_state`, `__clone`, `__debugInfo`), NameFunctionMagic, nil},
+	},
+	"magicconstants": {
+		{Words(``, `\b`, `__LINE__`, `__FILE__`, `__DIR__`, `__FUNCTION__`, `__CLASS__`, `__TRAIT__`, `__METHOD__`, `__NAMESPACE__`), NameConstant, nil},
+	},
+	"classname": {
+		{`(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameClass, Pop(1)},
+	},
+	"functionname": {
+		Include("magicfuncs"),
+		{`(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*`, NameFunction, Pop(1)},
+		Default(Pop(1)),
+	},
+	"string": {
+		{`"`, LiteralStringDouble, Pop(1)},
+		{`[^{$"\\]+`, LiteralStringDouble, nil},
+		{`\\([nrt"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})`, LiteralStringEscape, nil},
+		{`\$(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*(\[\S+?\]|->(?:[\\_a-z]|[^\x00-\x7f])(?:[\\\w]|[^\x00-\x7f])*)?`, LiteralStringInterpol, nil},
+		{`(\{\$\{)(.*?)(\}\})`, ByGroups(LiteralStringInterpol, UsingSelf("root"), LiteralStringInterpol), nil},
+		{`(\{)(\$.*?)(\})`, ByGroups(LiteralStringInterpol, UsingSelf("root"), LiteralStringInterpol), nil},
+		{`(\$\{)(\S+)(\})`, ByGroups(LiteralStringInterpol, NameVariable, LiteralStringInterpol), nil},
+		{`[${\\]`, LiteralStringDouble, nil},
 	},
-).SetAnalyser(func(text string) float32 {
-	if strings.Contains(text, "<?php") {
-		return 0.5
-	}
-	return 0.0
-})))
+}
diff --git a/lexers/circular/phtml.go b/lexers/circular/phtml.go
new file mode 100644
index 0000000..b9bffd3
--- /dev/null
+++ b/lexers/circular/phtml.go
@@ -0,0 +1,34 @@
+package circular
+
+import (
+	"strings"
+
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/h"
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// PHTML lexer is PHP in HTML.
+var PHTML = internal.Register(DelegatingLexer(h.HTML, MustNewLexer(
+	&Config{
+		Name:            "PHTML",
+		Aliases:         []string{"phtml"},
+		Filenames:       []string{"*.phtml"},
+		MimeTypes:       []string{"application/x-php", "application/x-httpd-php", "application/x-httpd-php3", "application/x-httpd-php4", "application/x-httpd-php5"},
+		DotAll:          true,
+		CaseInsensitive: true,
+		EnsureNL:        true,
+	},
+	Rules{
+		"root": {
+			{`<\?(php)?`, CommentPreproc, Push("php")},
+			{`[^<]+`, Other, nil},
+			{`<`, Other, nil},
+		},
+	}.Merge(phpCommonRules),
+).SetAnalyser(func(text string) float32 {
+	if strings.Contains(text, "<?php") {
+		return 0.5
+	}
+	return 0.0
+})))
diff --git a/lexers/p/pony.go b/lexers/p/pony.go
new file mode 100644
index 0000000..9696b20
--- /dev/null
+++ b/lexers/p/pony.go
@@ -0,0 +1,59 @@
+package p
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Pony lexer.
+var Pony = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "Pony",
+		Aliases:   []string{"pony"},
+		Filenames: []string{"*.pony"},
+		MimeTypes: []string{},
+	},
+	Rules{
+		"root": {
+			{`\n`, Text, nil},
+			{`[^\S\n]+`, Text, nil},
+			{`//.*\n`, CommentSingle, nil},
+			{`/\*`, CommentMultiline, Push("nested_comment")},
+			{`"""(?:.|\n)*?"""`, LiteralStringDoc, nil},
+			{`"`, LiteralString, Push("string")},
+			{`\'.*\'`, LiteralStringChar, nil},
+			{`=>|[]{}:().~;,|&!^?[]`, Punctuation, nil},
+			{Words(``, `\b`, `addressof`, `and`, `as`, `consume`, `digestof`, `is`, `isnt`, `not`, `or`), OperatorWord, nil},
+			{`!=|==|<<|>>|[-+/*%=<>]`, Operator, nil},
+			{Words(``, `\b`, `box`, `break`, `compile_error`, `compile_intrinsic`, `continue`, `do`, `else`, `elseif`, `embed`, `end`, `error`, `for`, `if`, `ifdef`, `in`, `iso`, `lambda`, `let`, `match`, `object`, `recover`, `ref`, `repeat`, `return`, `tag`, `then`, `this`, `trn`, `try`, `until`, `use`, `var`, `val`, `where`, `while`, `with`, `#any`, `#read`, `#send`, `#share`), Keyword, nil},
+			{`(actor|class|struct|primitive|interface|trait|type)((?:\s)+)`, ByGroups(Keyword, Text), Push("typename")},
+			{`(new|fun|be)((?:\s)+)`, ByGroups(Keyword, Text), Push("methodname")},
+			{Words(``, `\b`, `U8`, `U16`, `U32`, `U64`, `ULong`, `USize`, `U128`, `Unsigned`, `Stringable`, `String`, `StringBytes`, `StringRunes`, `InputNotify`, `InputStream`, `Stdin`, `ByteSeq`, `ByteSeqIter`, `OutStream`, `StdStream`, `SourceLoc`, `I8`, `I16`, `I32`, `I64`, `ILong`, `ISize`, `I128`, `Signed`, `Seq`, `RuntimeOptions`, `Real`, `Integer`, `SignedInteger`, `UnsignedInteger`, `FloatingPoint`, `Number`, `Int`, `ReadSeq`, `ReadElement`, `Pointer`, `Platform`, `NullablePointer`, `None`, `Iterator`, `F32`, `F64`, `Float`, `Env`, `DoNotOptimise`, `DisposableActor`, `Less`, `Equal`, `Greater`, `Compare`, `HasEq`, `Equatable`, `Comparable`, `Bool`, `AsioEventID`, `AsioEventNotify`, `AsioEvent`, `Array`, `ArrayKeys`, `ArrayValues`, `ArrayPairs`, `Any`, `AmbientAuth`), KeywordType, nil},
+			{`_?[A-Z]\w*`, NameClass, nil},
+			{`string\(\)`, NameOther, nil},
+			{`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+`, LiteralNumberFloat, nil},
+			{`0x[0-9a-fA-F]+`, LiteralNumberHex, nil},
+			{`\d+`, LiteralNumberInteger, nil},
+			{`(true|false)\b`, Keyword, nil},
+			{`_\d*`, Name, nil},
+			{`_?[a-z][\w\'_]*`, Name, nil},
+		},
+		"typename": {
+			{`(iso|trn|ref|val|box|tag)?((?:\s)*)(_?[A-Z]\w*)`, ByGroups(Keyword, Text, NameClass), Pop(1)},
+		},
+		"methodname": {
+			{`(iso|trn|ref|val|box|tag)?((?:\s)*)(_?[a-z]\w*)`, ByGroups(Keyword, Text, NameFunction), Pop(1)},
+		},
+		"nested_comment": {
+			{`[^*/]+`, CommentMultiline, nil},
+			{`/\*`, CommentMultiline, Push()},
+			{`\*/`, CommentMultiline, Pop(1)},
+			{`[*/]`, CommentMultiline, nil},
+		},
+		"string": {
+			{`"`, LiteralString, Pop(1)},
+			{`\\"`, LiteralString, nil},
+			{`[^\\"]+`, LiteralString, nil},
+		},
+	},
+))
diff --git a/lexers/t/toml.go b/lexers/t/toml.go
index 68bfc90..cc0a38e 100644
--- a/lexers/t/toml.go
+++ b/lexers/t/toml.go
@@ -22,7 +22,7 @@ var TOML = internal.Register(MustNewLexer(
 			{`[+-]?[0-9](_?\d)*`, LiteralNumberInteger, nil},
 			{`"(\\\\|\\"|[^"])*"`, StringDouble, nil},
 			{`'(\\\\|\\'|[^'])*'`, StringSingle, nil},
-			{`[.,=\[\]]`, Punctuation, nil},
+			{`[.,=\[\]{}]`, Punctuation, nil},
 			{`[^\W\d]\w*`, NameOther, nil},
 		},
 	},
diff --git a/lexers/t/typescript.go b/lexers/t/typescript.go
index 3b425ed..5205e01 100644
--- a/lexers/t/typescript.go
+++ b/lexers/t/typescript.go
@@ -38,14 +38,14 @@ var TypeScript = internal.Register(MustNewLexer(
 			{`\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?`, Operator, Push("slashstartsregex")},
 			{`[{(\[;,]`, Punctuation, Push("slashstartsregex")},
 			{`[})\].]`, Punctuation, nil},
-			{`(for|in|while|do|break|return|continue|switch|case|default|if|else|throw|try|catch|finally|new|delete|typeof|instanceof|void|this)\b`, Keyword, Push("slashstartsregex")},
+			{`(for|in|of|while|do|break|return|yield|continue|switch|case|default|if|else|throw|try|catch|finally|new|delete|typeof|instanceof|keyof|asserts|is|infer|await|void|this)\b`, Keyword, Push("slashstartsregex")},
 			{`(var|let|with|function)\b`, KeywordDeclaration, Push("slashstartsregex")},
-			{`(abstract|boolean|byte|char|class|const|debugger|double|enum|export|extends|final|float|goto|implements|import|int|interface|long|native|package|private|protected|public|short|static|super|synchronized|throws|transient|volatile)\b`, KeywordReserved, nil},
+			{`(abstract|async|boolean|class|const|debugger|enum|export|extends|from|get|global|goto|implements|import|interface|namespace|package|private|protected|public|readonly|require|set|static|super|type)\b`, KeywordReserved, nil},
 			{`(true|false|null|NaN|Infinity|undefined)\b`, KeywordConstant, nil},
-			{`(Array|Boolean|Date|Error|Function|Math|netscape|Number|Object|Packages|RegExp|String|sun|decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|window)\b`, NameBuiltin, nil},
+			{`(Array|Boolean|Date|Error|Function|Math|Number|Object|Packages|RegExp|String|decodeURI|decodeURIComponent|encodeURI|encodeURIComponent|eval|isFinite|isNaN|parseFloat|parseInt|document|this|window)\b`, NameBuiltin, nil},
 			{`\b(module)(\s*)(\s*[\w?.$][\w?.$]*)(\s*)`, ByGroups(KeywordReserved, Text, NameOther, Text), Push("slashstartsregex")},
-			{`\b(string|bool|number)\b`, KeywordType, nil},
-			{`\b(constructor|declare|interface|as|AS)\b`, KeywordReserved, nil},
+			{`\b(string|bool|number|any|never|object|symbol|unique|unknown|bigint)\b`, KeywordType, nil},
+			{`\b(constructor|declare|interface|as)\b`, KeywordReserved, nil},
 			{`(super)(\s*)(\([\w,?.$\s]+\s*\))`, ByGroups(KeywordReserved, Text), Push("slashstartsregex")},
 			{`([a-zA-Z_?.$][\w?.$]*)\(\) \{`, NameOther, Push("slashstartsregex")},
 			{`([\w?.$][\w?.$]*)(\s*:\s*)([\w?.$][\w?.$]*)`, ByGroups(NameOther, Text, KeywordType), nil},
diff --git a/regexp.go b/regexp.go
index 96bcce6..590bcab 100644
--- a/regexp.go
+++ b/regexp.go
@@ -160,6 +160,14 @@ func Tokenise(lexer Lexer, options *TokeniseOptions, text string) ([]Token, erro
 // Rules maps from state to a sequence of Rules.
 type Rules map[string][]Rule
 
+// Rename clones rules then a rule.
+func (r Rules) Rename(old, new string) Rules {
+	r = r.Clone()
+	r[new] = r[old]
+	delete(r, old)
+	return r
+}
+
 // Clone returns a clone of the Rules.
 func (r Rules) Clone() Rules {
 	out := map[string][]Rule{}


Test patch:
diff --git a/lexers/lexers_test.go b/lexers/lexers_test.go
index a54db51..93d5189 100644
--- a/lexers/lexers_test.go
+++ b/lexers/lexers_test.go
@@ -73,7 +73,7 @@ func TestLexers(t *testing.T) {
 			if os.Getenv("RECORD") == "true" {
 				// Update the expected file with the generated output of this lexer
 				f, err := os.Create(expectedFilename)
-				defer f.Close()
+				defer f.Close() // nolint: gosec
 				assert.NoError(t, err)
 				assert.NoError(t, formatters.JSON.Format(f, nil, chroma.Literator(actual...)))
 			} else {
diff --git a/lexers/testdata/php.actual b/lexers/testdata/php.actual
index 764de58..936827e 100644
--- a/lexers/testdata/php.actual
+++ b/lexers/testdata/php.actual
@@ -1,10 +1,3 @@
-<!DOCTYPE html>
-<html>
-<body>
-
-<h1>My first PHP page</h1>
-<?php
-
 $docs = $modx->getIterator('modResource', ["parent" => 84]);
 
 foreach($docs as $doc){
@@ -12,8 +5,5 @@ foreach($docs as $doc){
     $doc->set("content", preg_replace("/Some value/i", "Replacement", $q));
     print_r($doc->content);
     // $doc->save();
-} 
+}
 // some comment
-?>
-</body>
-</html>
diff --git a/lexers/testdata/php.expected b/lexers/testdata/php.expected
index 108a23b..0e0ed98 100644
--- a/lexers/testdata/php.expected
+++ b/lexers/testdata/php.expected
@@ -1,24 +1,4 @@
 [
-  {"type":"CommentPreproc","value":"\u003c!DOCTYPE html\u003e"},
-  {"type":"Text","value":"\n"},
-  {"type":"Punctuation","value":"\u003c"},
-  {"type":"NameTag","value":"html"},
-  {"type":"Punctuation","value":"\u003e"},
-  {"type":"Text","value":"\n"},
-  {"type":"Punctuation","value":"\u003c"},
-  {"type":"NameTag","value":"body"},
-  {"type":"Punctuation","value":"\u003e"},
-  {"type":"Text","value":"\n\n"},
-  {"type":"Punctuation","value":"\u003c"},
-  {"type":"NameTag","value":"h1"},
-  {"type":"Punctuation","value":"\u003e"},
-  {"type":"Text","value":"My first PHP page"},
-  {"type":"Punctuation","value":"\u003c/"},
-  {"type":"NameTag","value":"h1"},
-  {"type":"Punctuation","value":"\u003e"},
-  {"type":"Text","value":"\n"},
-  {"type":"CommentPreproc","value":"\u003c?php"},
-  {"type":"Text","value":"\n\n"},
   {"type":"NameVariable","value":"$docs"},
   {"type":"Text","value":" "},
   {"type":"Operator","value":"="},
@@ -81,16 +61,6 @@
   {"type":"Text","value":"\n    "},
   {"type":"CommentSingle","value":"// $doc-\u003esave();\n"},
   {"type":"Punctuation","value":"}"},
-  {"type":"Text","value":" \n"},
-  {"type":"CommentSingle","value":"// some comment\n"},
-  {"type":"CommentPreproc","value":"?\u003e"},
-  {"type":"Text","value":"\n"},
-  {"type":"Punctuation","value":"\u003c/"},
-  {"type":"NameTag","value":"body"},
-  {"type":"Punctuation","value":"\u003e"},
   {"type":"Text","value":"\n"},
-  {"type":"Punctuation","value":"\u003c/"},
-  {"type":"NameTag","value":"html"},
-  {"type":"Punctuation","value":"\u003e"},
-  {"type":"Text","value":"\n"}
+  {"type":"CommentSingle","value":"// some comment\n"}
 ]
diff --git a/lexers/testdata/phtml.actual b/lexers/testdata/phtml.actual
new file mode 100644
index 0000000..764de58
--- /dev/null
+++ b/lexers/testdata/phtml.actual
@@ -0,0 +1,19 @@
+<!DOCTYPE html>
+<html>
+<body>
+
+<h1>My first PHP page</h1>
+<?php
+
+$docs = $modx->getIterator('modResource', ["parent" => 84]);
+
+foreach($docs as $doc){
+    $q=$doc->content;
+    $doc->set("content", preg_replace("/Some value/i", "Replacement", $q));
+    print_r($doc->content);
+    // $doc->save();
+} 
+// some comment
+?>
+</body>
+</html>
diff --git a/lexers/testdata/phtml.expected b/lexers/testdata/phtml.expected
new file mode 100644
index 0000000..108a23b
--- /dev/null
+++ b/lexers/testdata/phtml.expected
@@ -0,0 +1,96 @@
+[
+  {"type":"CommentPreproc","value":"\u003c!DOCTYPE html\u003e"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"\u003c"},
+  {"type":"NameTag","value":"html"},
+  {"type":"Punctuation","value":"\u003e"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"\u003c"},
+  {"type":"NameTag","value":"body"},
+  {"type":"Punctuation","value":"\u003e"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Punctuation","value":"\u003c"},
+  {"type":"NameTag","value":"h1"},
+  {"type":"Punctuation","value":"\u003e"},
+  {"type":"Text","value":"My first PHP page"},
+  {"type":"Punctuation","value":"\u003c/"},
+  {"type":"NameTag","value":"h1"},
+  {"type":"Punctuation","value":"\u003e"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentPreproc","value":"\u003c?php"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"NameVariable","value":"$docs"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$modx"},
+  {"type":"Operator","value":"-\u003e"},
+  {"type":"NameAttribute","value":"getIterator"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralStringSingle","value":"'modResource'"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"parent\""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"=\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"84"},
+  {"type":"Punctuation","value":"]);"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"foreach"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"$docs"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"as"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$doc"},
+  {"type":"Punctuation","value":"){"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameVariable","value":"$q"},
+  {"type":"Operator","value":"="},
+  {"type":"NameVariable","value":"$doc"},
+  {"type":"Operator","value":"-\u003e"},
+  {"type":"NameAttribute","value":"content"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameVariable","value":"$doc"},
+  {"type":"Operator","value":"-\u003e"},
+  {"type":"NameAttribute","value":"set"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralStringDouble","value":"\"content\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"preg_replace"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralStringDouble","value":"\"/Some value/i\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"Replacement\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"$q"},
+  {"type":"Punctuation","value":"));"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameOther","value":"print_r"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"$doc"},
+  {"type":"Operator","value":"-\u003e"},
+  {"type":"NameAttribute","value":"content"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// $doc-\u003esave();\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":" \n"},
+  {"type":"CommentSingle","value":"// some comment\n"},
+  {"type":"CommentPreproc","value":"?\u003e"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"\u003c/"},
+  {"type":"NameTag","value":"body"},
+  {"type":"Punctuation","value":"\u003e"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"\u003c/"},
+  {"type":"NameTag","value":"html"},
+  {"type":"Punctuation","value":"\u003e"},
+  {"type":"Text","value":"\n"}
+]
diff --git a/lexers/testdata/pony.actual b/lexers/testdata/pony.actual
new file mode 100644
index 0000000..b4f272d
--- /dev/null
+++ b/lexers/testdata/pony.actual
@@ -0,0 +1,82 @@
+use "net"
+use "files"
+
+class ClientSide is TCPConnectionNotify
+  let _env: Env
+
+  new iso create(env: Env) =>
+    _env = env
+
+  fun ref connecting(conn: TCPConnection ref, count: U32) =>
+    _env.out.print("connecting: " + count.string())
+
+  fun ref connected(conn: TCPConnection ref) =>
+    try
+      (let host, let service) = conn.remote_address().name()?
+      _env.out.print("connected to " + host + ":" + service)
+      conn.set_nodelay(true)
+      conn.set_keepalive(10)
+      conn.write("client says hi")
+    end
+
+class Listener is TCPListenNotify
+  let _env: Env
+  let _limit: USize
+  var _host: String = ""
+  var _count: USize = 0
+
+  new create(env: Env, limit: USize) =>
+    _env = env
+    _limit = limit
+
+  fun ref connected(listen: TCPListener ref): TCPConnectionNotify iso^ =>
+    let env = _env
+
+    _env.out.print("Server starting")
+
+    let server = ServerSide(env)
+
+    _spawn(listen)
+    server
+
+  fun ref _spawn(listen: TCPListener ref) =>
+    if (_limit > 0) and (_count >= _limit) then
+      listen.dispose()
+      return
+    end
+
+    _count = _count + 1
+    _env.out.print("spawn " + _count.string())
+
+    try
+      let env = _env
+
+      _env.out.print("Client starting")
+      TCPConnection(
+        _env.root as AmbientAuth,
+        ClientSide(env),
+        _host,
+        _service)
+    else
+      _env.out.print("couldn't create client side")
+      listen.close()
+    end
+
+actor Main
+  new create(env: Env) =>
+    let limit = try
+      env.args(1)?.usize()?
+    else
+      1
+    end
+
+    try
+      let auth = env.root as AmbientAuth
+      TCPListener(auth, recover Listener(env, limit) end)
+      UDPSocket(auth, recover Pong(env) end)
+    else
+      env.out.print("unable to use the network")
+    end
+  be test() =>
+    nonsensical.stuff.here()
+
diff --git a/lexers/testdata/pony.expected b/lexers/testdata/pony.expected
new file mode 100644
index 0000000..be46eb5
--- /dev/null
+++ b/lexers/testdata/pony.expected
@@ -0,0 +1,541 @@
+[
+  {"type":"Keyword","value":"use"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"net\""},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"use"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"files\""},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"ClientSide"},
+  {"type":"Text","value":" "},
+  {"type":"OperatorWord","value":"is"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TCPConnectionNotify"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"Env"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Keyword","value":"new"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"iso"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"create"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"Env"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"_env"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"env"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Keyword","value":"fun"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"connecting"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"conn"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TCPConnection"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"count"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"U32"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"print"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"connecting: \""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"count"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"string()"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Keyword","value":"fun"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"connected"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"conn"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TCPConnection"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"try"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"host"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"service"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"conn"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"remote_address"},
+  {"type":"Punctuation","value":"()."},
+  {"type":"Name","value":"name"},
+  {"type":"Punctuation","value":"()?"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"print"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"connected to \""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"host"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\":\""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"service"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"conn"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"set_nodelay"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Keyword","value":"true"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"conn"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"set_keepalive"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"conn"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"write"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"client says hi\""},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Listener"},
+  {"type":"Text","value":" "},
+  {"type":"OperatorWord","value":"is"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TCPListenNotify"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"Env"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_limit"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"USize"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"var"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_host"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"String"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"\""},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"var"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_count"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"USize"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Keyword","value":"new"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"create"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"Env"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"limit"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"USize"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"_env"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"env"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"_limit"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"limit"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Keyword","value":"fun"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"connected"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"listen"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TCPListener"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Punctuation","value":"):"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TCPConnectionNotify"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"iso"},
+  {"type":"Punctuation","value":"^"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"env"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_env"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"print"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"Server starting\""},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"server"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"ServerSide"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Name","value":"_spawn"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"listen"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"server"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Keyword","value":"fun"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"_spawn"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"listen"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TCPListener"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"ref"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"if"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"_limit"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"OperatorWord","value":"and"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"_count"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003e="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_limit"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"then"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"listen"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"dispose"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Name","value":"_count"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_count"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"print"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"spawn \""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_count"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"string()"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"try"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"env"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"_env"},
+  {"type":"Text","value":"\n\n      "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"print"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"Client starting\""},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n      "},
+  {"type":"NameClass","value":"TCPConnection"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Text","value":"\n        "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"root"},
+  {"type":"Text","value":" "},
+  {"type":"OperatorWord","value":"as"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"AmbientAuth"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n        "},
+  {"type":"NameClass","value":"ClientSide"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":"),"},
+  {"type":"Text","value":"\n        "},
+  {"type":"Name","value":"_host"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n        "},
+  {"type":"Name","value":"_service"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"else"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"_env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"print"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"couldn't create client side\""},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"listen"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"close"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"actor"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Main"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"new"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"create"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"Env"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"limit"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"try"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"args"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Punctuation","value":")?."},
+  {"type":"Name","value":"usize"},
+  {"type":"Punctuation","value":"()?"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"else"},
+  {"type":"Text","value":"\n      "},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"try"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"auth"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"root"},
+  {"type":"Text","value":" "},
+  {"type":"OperatorWord","value":"as"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"AmbientAuth"},
+  {"type":"Text","value":"\n      "},
+  {"type":"NameClass","value":"TCPListener"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"auth"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"recover"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Listener"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"limit"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n      "},
+  {"type":"NameClass","value":"UDPSocket"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"auth"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"recover"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Pong"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"else"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"env"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"print"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralString","value":"\"unable to use the network\""},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"be"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"test"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"=\u003e"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"nonsensical"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"stuff"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"here"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":"\n\n"}
+]
diff --git a/lexers/testdata/tsx.expected b/lexers/testdata/tsx.expected
index 69acfc4..c498ff3 100644
--- a/lexers/testdata/tsx.expected
+++ b/lexers/testdata/tsx.expected
@@ -3,7 +3,7 @@
   {"type":"Text","value":" "},
   {"type":"NameOther","value":"React"},
   {"type":"Text","value":" "},
-  {"type":"NameOther","value":"from"},
+  {"type":"KeywordReserved","value":"from"},
   {"type":"Text","value":" "},
   {"type":"LiteralStringSingle","value":"'react'"},
   {"type":"Text","value":"\n"},
@@ -11,7 +11,7 @@
   {"type":"Text","value":" "},
   {"type":"NameOther","value":"ReactDOM"},
   {"type":"Text","value":" "},
-  {"type":"NameOther","value":"from"},
+  {"type":"KeywordReserved","value":"from"},
   {"type":"Text","value":" "},
   {"type":"LiteralStringSingle","value":"'react-dom'"},
   {"type":"Text","value":"\n"},
@@ -27,7 +27,7 @@
   {"type":"Text","value":" "},
   {"type":"Punctuation","value":"}"},
   {"type":"Text","value":" "},
-  {"type":"NameOther","value":"from"},
+  {"type":"KeywordReserved","value":"from"},
   {"type":"Text","value":" "},
   {"type":"LiteralStringSingle","value":"'./App'"},
   {"type":"Text","value":"\n\n"},