Repository: alecthomas/chroma
Pull request #255 (2019-05-27 17:43:54)

Update Java lexer to use Punctuation where applicable (#254)
PR for https://github.com/alecthomas/chroma/issues/254

alecthomas: Thanks for the PR, but the tests are failing.
dpromanko: Yeah, I was a little confused at the tests failing at first. Didn't realize there was a concept of sublexers. I'll fix the tests when I get a chance.
dpromanko: Sorry for not running tests locally the first time. This is the first time I've cloned and contributed to a Go project and was a little confused at how to run stuff locally. Newer to git as well, so out of curiosity is there something I should do to squash these commits, or is that something you do on merge?
alecthomas: No worries, thanks for contributing!

Nope I will squash them on merge. Thanks again!

Patch:
diff --git a/lexers/j/java.go b/lexers/j/java.go
index c6b9a76..d9beb46 100644
--- a/lexers/j/java.go
+++ b/lexers/j/java.go
@@ -20,7 +20,7 @@ var Java = internal.Register(MustNewLexer(
 			{`//.*?\n`, CommentSingle, nil},
 			{`/\*.*?\*/`, CommentMultiline, nil},
 			{`(assert|break|case|catch|continue|default|do|else|finally|for|if|goto|instanceof|new|return|switch|this|throw|try|while)\b`, Keyword, nil},
-			{`((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)((?:[^\W\d]|\$)[\w$]*)(\s*)(\()`, ByGroups(UsingSelf("root"), NameFunction, Text, Operator), nil},
+			{`((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)((?:[^\W\d]|\$)[\w$]*)(\s*)`, ByGroups(UsingSelf("root"), NameFunction, Text, Operator), nil},
 			{`@[^\W\d][\w.]*`, NameDecorator, nil},
 			{`(abstract|const|enum|extends|final|implements|native|private|protected|public|static|strictfp|super|synchronized|throws|transient|volatile)\b`, KeywordDeclaration, nil},
 			{`(boolean|byte|char|double|float|int|long|short|void)\b`, KeywordType, nil},
@@ -30,7 +30,7 @@ var Java = internal.Register(MustNewLexer(
 			{`(import(?:\s+static)?)(\s+)`, ByGroups(KeywordNamespace, Text), Push("import")},
 			{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
 			{`'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'`, LiteralStringChar, nil},
-			{`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Operator, NameAttribute), nil},
+			{`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Punctuation, NameAttribute), nil},
 			{`^\s*([^\W\d]|\$)[\w$]*:`, NameLabel, nil},
 			{`([^\W\d]|\$)[\w$]*`, Name, nil},
 			{`([0-9][0-9_]*\.([0-9][0-9_]*)?|\.[0-9][0-9_]*)([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)[pP][+\-]?[0-9][0-9_]*[fFdD]?`, LiteralNumberFloat, nil},
@@ -38,7 +38,8 @@ var Java = internal.Register(MustNewLexer(
 			{`0[bB][01][01_]*[lL]?`, LiteralNumberBin, nil},
 			{`0[0-7_]+[lL]?`, LiteralNumberOct, nil},
 			{`0|[1-9][0-9_]*[lL]?`, LiteralNumberInteger, nil},
-			{`[~^*!%&\[\](){}<>|+=:;,./?-]`, Operator, nil},
+			{`[~^*!%&<>|+=:/?-]`, Operator, nil},
+			{`[\[\](){};,.]`, Punctuation, nil},
 			{`\n`, Text, nil},
 		},
 		"class": {


Test patch:
diff --git a/lexers/testdata/cql.expected b/lexers/testdata/cql.expected
index bc5f849..3ad730d 100644
--- a/lexers/testdata/cql.expected
+++ b/lexers/testdata/cql.expected
@@ -1068,17 +1068,17 @@
   {"type":"Keyword","value":"return"},
   {"type":"Text","value":" "},
   {"type":"Name","value":"Double"},
-  {"type":"Operator","value":"."},
+  {"type":"Punctuation","value":"."},
   {"type":"NameAttribute","value":"valueOf"},
-  {"type":"Operator","value":"("},
+  {"type":"Punctuation","value":"("},
   {"type":"Name","value":"Math"},
-  {"type":"Operator","value":"."},
+  {"type":"Punctuation","value":"."},
   {"type":"NameAttribute","value":"log"},
-  {"type":"Operator","value":"("},
+  {"type":"Punctuation","value":"("},
   {"type":"Name","value":"input"},
-  {"type":"Operator","value":"."},
+  {"type":"Punctuation","value":"."},
   {"type":"NameAttribute","value":"doubleValue"},
-  {"type":"Operator","value":"()));"},
+  {"type":"Punctuation","value":"()));"},
   {"type":"LiteralStringHeredoc","value":"'"},
   {"type":"Punctuation","value":";"},
   {"type":"TextWhitespace","value":"\n"}
diff --git a/lexers/testdata/markdown.expected b/lexers/testdata/markdown.expected
index 1d39de5..9c62d3c 100644
--- a/lexers/testdata/markdown.expected
+++ b/lexers/testdata/markdown.expected
@@ -119,17 +119,17 @@
   {"type":"Keyword","value":"return"},
   {"type":"Text","value":" "},
   {"type":"Name","value":"Double"},
-  {"type":"Operator","value":"."},
+  {"type":"Punctuation","value":"."},
   {"type":"NameAttribute","value":"valueOf"},
-  {"type":"Operator","value":"("},
+  {"type":"Punctuation","value":"("},
   {"type":"Name","value":"Math"},
-  {"type":"Operator","value":"."},
+  {"type":"Punctuation","value":"."},
   {"type":"NameAttribute","value":"log"},
-  {"type":"Operator","value":"("},
+  {"type":"Punctuation","value":"("},
   {"type":"Name","value":"input"},
-  {"type":"Operator","value":"."},
+  {"type":"Punctuation","value":"."},
   {"type":"NameAttribute","value":"doubleValue"},
-  {"type":"Operator","value":"()));"},
+  {"type":"Punctuation","value":"()));"},
   {"type":"LiteralStringHeredoc","value":"'"},
   {"type":"Punctuation","value":";"},
   {"type":"TextWhitespace","value":"\n"},