Repository: alecthomas/chroma
Pull request #444 (2021-01-25 22:17:55)

Change Text token to TextWhitespace token in SQL lexers
For consistency with other lexers, convert the Text token on whitespace
to TextWhitespace.


Patch:
diff --git a/lexers/m/mysql.go b/lexers/m/mysql.go
index 9f47e32..a102b34 100644
--- a/lexers/m/mysql.go
+++ b/lexers/m/mysql.go
@@ -17,7 +17,7 @@ var MySQL = internal.Register(MustNewLexer(
 	},
 	Rules{
 		"root": {
-			{`\s+`, Text, nil},
+			{`\s+`, TextWhitespace, nil},
 			{`(#|--\s+).*\n?`, CommentSingle, nil},
 			{`/\*`, CommentMultiline, Push("multiline-comments")},
 			{`[0-9]+`, LiteralNumberInteger, nil},
@@ -25,11 +25,11 @@ var MySQL = internal.Register(MustNewLexer(
 			{`((?:_[a-z0-9]+)?)(')`, ByGroups(LiteralStringAffix, LiteralStringSingle), Push("string")},
 			{`((?:_[a-z0-9]+)?)(")`, ByGroups(LiteralStringAffix, LiteralStringDouble), Push("double-string")},
 			{"[+*/<>=~!@#%^&|`?-]", Operator, nil},
-			{`\b(tinyint|smallint|mediumint|int|integer|bigint|date|datetime|time|bit|bool|tinytext|mediumtext|longtext|text|tinyblob|mediumblob|longblob|blob|float|double|double\s+precision|real|numeric|dec|decimal|timestamp|year|char|varchar|varbinary|varcharacter|enum|set)(\b\s*)(\()?`, ByGroups(KeywordType, Text, Punctuation), nil},
+			{`\b(tinyint|smallint|mediumint|int|integer|bigint|date|datetime|time|bit|bool|tinytext|mediumtext|longtext|text|tinyblob|mediumblob|longblob|blob|float|double|double\s+precision|real|numeric|dec|decimal|timestamp|year|char|varchar|varbinary|varcharacter|enum|set)(\b\s*)(\()?`, ByGroups(KeywordType, TextWhitespace, Punctuation), nil},
 			{`\b(add|all|alter|analyze|and|as|asc|asensitive|before|between|bigint|binary|blob|both|by|call|cascade|case|change|char|character|check|collate|column|condition|constraint|continue|convert|create|cross|current_date|current_time|current_timestamp|current_user|cursor|database|databases|day_hour|day_microsecond|day_minute|day_second|dec|decimal|declare|default|delayed|delete|desc|describe|deterministic|distinct|distinctrow|div|double|drop|dual|each|else|elseif|enclosed|escaped|exists|exit|explain|fetch|flush|float|float4|float8|for|force|foreign|from|fulltext|grant|group|having|high_priority|hour_microsecond|hour_minute|hour_second|identified|if|ignore|in|index|infile|inner|inout|insensitive|insert|int|int1|int2|int3|int4|int8|integer|interval|into|is|iterate|join|key|keys|kill|leading|leave|left|like|limit|lines|load|localtime|localtimestamp|lock|long|loop|low_priority|match|minute_microsecond|minute_second|mod|modifies|natural|no_write_to_binlog|not|numeric|on|optimize|option|optionally|or|order|out|outer|outfile|precision|primary|privileges|procedure|purge|raid0|read|reads|real|references|regexp|release|rename|repeat|replace|require|restrict|return|revoke|right|rlike|schema|schemas|second_microsecond|select|sensitive|separator|set|show|smallint|soname|spatial|specific|sql|sql_big_result|sql_calc_found_rows|sql_small_result|sqlexception|sqlstate|sqlwarning|ssl|starting|straight_join|table|terminated|then|to|trailing|trigger|undo|union|unique|unlock|unsigned|update|usage|use|user|using|utc_date|utc_time|utc_timestamp|values|varying|when|where|while|with|write|x509|xor|year_month|zerofill)\b`, Keyword, nil},
 			{`\b(auto_increment|engine|charset|tables)\b`, KeywordPseudo, nil},
 			{`(true|false|null)`, NameConstant, nil},
-			{`([a-z_]\w*)(\s*)(\()`, ByGroups(NameFunction, Text, Punctuation), nil},
+			{`([a-z_]\w*)(\s*)(\()`, ByGroups(NameFunction, TextWhitespace, Punctuation), nil},
 			{`[a-z_]\w*`, Name, nil},
 			{`@[a-z0-9]*[._]*[a-z0-9]*`, NameVariable, nil},
 			{`[;:()\[\],.]`, Punctuation, nil},
diff --git a/lexers/p/plsql.go b/lexers/p/plsql.go
index a6068fd..8b034ab 100644
--- a/lexers/p/plsql.go
+++ b/lexers/p/plsql.go
@@ -21,7 +21,7 @@ var PLpgSQL = internal.Register(MustNewLexer(
 			{`:=`, Operator, nil},
 			{`\<\<[a-z]\w*\>\>`, NameLabel, nil},
 			{`\#[a-z]\w*\b`, KeywordPseudo, nil},
-			{`\s+`, Text, nil},
+			{`\s+`, TextWhitespace, nil},
 			{`--.*\n?`, CommentSingle, nil},
 			{`/\*`, CommentMultiline, Push("multiline-comments")},
 			{`(bigint|bigserial|bit|bit\s+varying|bool|boolean|box|bytea|char|character|character\s+varying|cidr|circle|date|decimal|double\s+precision|float4|float8|inet|int|int2|int4|int8|integer|interval|json|jsonb|line|lseg|macaddr|money|numeric|path|pg_lsn|point|polygon|real|serial|serial2|serial4|serial8|smallint|smallserial|text|time|timestamp|timestamptz|timetz|tsquery|tsvector|txid_snapshot|uuid|varbit|varchar|with\s+time\s+zone|without\s+time\s+zone|xml|anyarray|anyelement|anyenum|anynonarray|anyrange|cstring|fdw_handler|internal|language_handler|opaque|record|void)\b`, NameBuiltin, nil},
diff --git a/lexers/s/sql.go b/lexers/s/sql.go
index 6f4e3e0..b22eb42 100644
--- a/lexers/s/sql.go
+++ b/lexers/s/sql.go
@@ -17,7 +17,7 @@ var SQL = internal.Register(MustNewLexer(
 	},
 	Rules{
 		"root": {
-			{`\s+`, Text, nil},
+			{`\s+`, TextWhitespace, nil},
 			{`--.*\n?`, CommentSingle, nil},
 			{`/\*`, CommentMultiline, Push("multiline-comments")},
 			{`'`, LiteralStringSingle, Push("string")},


Test patch:
diff --git a/lexers/testdata/markdown.expected b/lexers/testdata/markdown.expected
index 0c50084..e4506d6 100644
--- a/lexers/testdata/markdown.expected
+++ b/lexers/testdata/markdown.expected
@@ -218,123 +218,123 @@
   {"type":"Keyword","value":"AS"},
   {"type":"Text","value":" "},
   {"type":"LiteralStringHeredoc","value":"$emp_stamp$"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Keyword","value":"BEGIN"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"CommentSingle","value":"-- Check that empname and salary are given\n"},
-  {"type":"Text","value":"  "},
+  {"type":"TextWhitespace","value":"  "},
   {"type":"Keyword","value":"IF"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"NEW"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"empname"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"IS"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"NULL"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"THEN"},
-  {"type":"Text","value":"\n    "},
+  {"type":"TextWhitespace","value":"\n    "},
   {"type":"Keyword","value":"RAISE"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"EXCEPTION"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"LiteralStringSingle","value":"'empname cannot be null'"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Keyword","value":"END"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"IF"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Keyword","value":"IF"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"NEW"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"salary"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"IS"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"NULL"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"THEN"},
-  {"type":"Text","value":"\n    "},
+  {"type":"TextWhitespace","value":"\n    "},
   {"type":"Keyword","value":"RAISE"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"EXCEPTION"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"LiteralStringSingle","value":"'% cannot have null salary'"},
   {"type":"Punctuation","value":","},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"NEW"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"empname"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Keyword","value":"END"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"IF"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n\n  "},
+  {"type":"TextWhitespace","value":"\n\n  "},
   {"type":"CommentSingle","value":"-- Who works for us when she must pay for it?\n"},
-  {"type":"Text","value":"  "},
+  {"type":"TextWhitespace","value":"  "},
   {"type":"Keyword","value":"IF"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"NEW"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"salary"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":"\u003c"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"LiteralNumberFloat","value":"0"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"THEN"},
-  {"type":"Text","value":"\n    "},
+  {"type":"TextWhitespace","value":"\n    "},
   {"type":"Keyword","value":"RAISE"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"EXCEPTION"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"LiteralStringSingle","value":"'% cannot have a negative salary'"},
   {"type":"Punctuation","value":","},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"NEW"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"empname"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Keyword","value":"END"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"IF"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n\n  "},
+  {"type":"TextWhitespace","value":"\n\n  "},
   {"type":"CommentSingle","value":"-- Remember who changed the payroll when\n"},
-  {"type":"Text","value":"  "},
+  {"type":"TextWhitespace","value":"  "},
   {"type":"Name","value":"NEW"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"last_date"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":":="},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"current_timestamp"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Name","value":"NEW"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"last_user"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":":="},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"current_user"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Keyword","value":"RETURN"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"NEW"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Keyword","value":"END"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"LiteralStringHeredoc","value":"$emp_stamp$"},
   {"type":"Text","value":" "},
   {"type":"Keyword","value":"LANGUAGE"},
@@ -380,54 +380,54 @@
   {"type":"LiteralStringSingle","value":"plpgsql"},
   {"type":"Text","value":" "},
   {"type":"LiteralStringHeredoc","value":"$$"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Keyword","value":"declare"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"r"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"NameBuiltin","value":"record"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Keyword","value":"begin"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Keyword","value":"for"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"r"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"in"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"select"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":"*"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"from"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"books"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Keyword","value":"loop"},
-  {"type":"Text","value":"\n  "},
+  {"type":"TextWhitespace","value":"\n  "},
   {"type":"Keyword","value":"execute"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"LiteralStringSingle","value":"'select '''"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":"||"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Name","value":"r"},
   {"type":"LiteralNumberFloat","value":"."},
   {"type":"Name","value":"title"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Operator","value":"||"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"LiteralStringSingle","value":"''''"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Keyword","value":"end"},
-  {"type":"Text","value":" "},
+  {"type":"TextWhitespace","value":" "},
   {"type":"Keyword","value":"loop"},
   {"type":"Punctuation","value":";"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"Keyword","value":"end"},
-  {"type":"Text","value":"\n"},
+  {"type":"TextWhitespace","value":"\n"},
   {"type":"LiteralStringHeredoc","value":"$$"},
   {"type":"Punctuation","value":";"},
   {"type":"Text","value":"\n\n"},