Repository: alecthomas/chroma
Pull request #509 (2021-05-16 19:04:14)

pygments2chroma updates


Patch:
diff --git a/README.md b/README.md
index 5f0bce7..95d54e6 100644
--- a/README.md
+++ b/README.md
@@ -210,10 +210,10 @@ using the included Python 3 script `pygments2chroma.py`. I use something like
 the following:
 
 ```sh
-python3 ~/Projects/chroma/_tools/pygments2chroma.py \
+python3 _tools/pygments2chroma.py \
   pygments.lexers.jvm.KotlinLexer \
-  > ~/Projects/chroma/lexers/kotlin.go \
-  && gofmt -s -w ~/Projects/chroma/lexers/*.go
+  > lexers/k/kotlin.go \
+  && gofmt -s -w lexers/k/kotlin.go
 ```
 
 See notes in [pygments-lexers.txt](https://github.com/alecthomas/chroma/blob/master/pygments-lexers.txt)
diff --git a/_tools/pygments2chroma.py b/_tools/pygments2chroma.py
index 5e18e02..006ea31 100644
--- a/_tools/pygments2chroma.py
+++ b/_tools/pygments2chroma.py
@@ -12,41 +12,43 @@ from pygments.token import _TokenType
 
 
 TEMPLATE = r'''
-package lexers
+package {{package}}
 
 import (
-    . "github.com/alecthomas/chroma" // nolint
-    "github.com/alecthomas/chroma/lexers/internal"
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
 )
 
 // {{upper_name}} lexer.
 var {{upper_name}} = internal.Register(MustNewLazyLexer(
-    &Config{
-        Name:      "{{name}}",
-        Aliases:   []string{ {{#aliases}}"{{.}}", {{/aliases}} },
-        Filenames: []string{ {{#filenames}}"{{.}}", {{/filenames}} },
-        MimeTypes: []string{ {{#mimetypes}}"{{.}}", {{/mimetypes}} },
+	&Config{
+		Name:      "{{name}}",
+		{{=<% %>=}}
+		Aliases:   []string{<%#aliases%>"<%.%>", <%/aliases%>},
+		Filenames: []string{<%#filenames%>"<%.%>", <%/filenames%>},
+		MimeTypes: []string{<%#mimetypes%>"<%.%>", <%/mimetypes%>},
+		<%={{ }}=%>
 {{#re_not_multiline}}
-        NotMultiline: true,
+		NotMultiline: true,
 {{/re_not_multiline}}
 {{#re_dotall}}
-        DotAll: true,
+		DotAll: true,
 {{/re_dotall}}
 {{#re_ignorecase}}
-        CaseInsensitive: true,
+		CaseInsensitive: true,
 {{/re_ignorecase}}
-    },
-    func() Rules {
-        return Rules{
+	},
+	func() Rules {
+		return Rules{
 {{#tokens}}
-            "{{state}}": {
-                {{#rules}}
-                {{{.}}},
-                {{/rules}}
-            },
+			"{{state}}": {
+				{{#rules}}
+				{{{.}}},
+				{{/rules}}
+			},
 {{/tokens}}
-        }
-    },
+		}
+	},
 ))
 '''
 
@@ -150,7 +152,7 @@ def translate_rules(rules):
                 modifier = 'Push("%s")' % '", "'.join(rule[2])
             else:
                 raise ValueError('unsupported modifier %r' % (rule[2],))
-            out.append('{{ {}, {}, {} }}'.format(regex, emitter, modifier))
+            out.append('{{{}, {}, {}}}'.format(regex, emitter, modifier))
         elif isinstance(rule, pygments_lexer.include):
             out.append('Include("{}")'.format(rule))
         elif isinstance(rule, pygments_lexer.default):
@@ -185,9 +187,10 @@ def main():
     assert issubclass(lexer_cls, pygments_lexer.RegexLexer), 'can only translate from RegexLexer'
 
     print(pystache.render(TEMPLATE, TemplateView(
+        package=lexer_cls.name.lower()[0],
         name=lexer_cls.name,
         regex_flags=lexer_cls.flags,
-        upper_name=to_camel_case(lexer_cls.name),
+        upper_name=to_camel_case(re.sub(r'\W', '_', lexer_cls.name)),
         aliases=lexer_cls.aliases,
         filenames=lexer_cls.filenames,
         mimetypes=lexer_cls.mimetypes,
diff --git a/lexers/m/metal.go b/lexers/m/metal.go
new file mode 100644
index 0000000..4a9ba2b
--- /dev/null
+++ b/lexers/m/metal.go
@@ -0,0 +1,101 @@
+package m
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Metal lexer.
+var Metal = internal.Register(MustNewLazyLexer(
+	&Config{
+		Name:      "Metal",
+		Aliases:   []string{"metal"},
+		Filenames: []string{"*.metal"},
+		MimeTypes: []string{"text/x-metal"},
+		EnsureNL:  true,
+	},
+	metalRules,
+))
+
+func metalRules() Rules {
+	return Rules{
+		"statements": {
+			{Words(``, `\b`, `namespace`, `operator`, `template`, `this`, `using`, `constexpr`), Keyword, nil},
+			{`(enum)\b(\s+)(class)\b(\s*)`, ByGroups(Keyword, Text, Keyword, Text), Push("classname")},
+			{`(class|struct|enum|union)\b(\s*)`, ByGroups(Keyword, Text), Push("classname")},
+			{`\[\[.+\]\]`, NameAttribute, nil},
+			{`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*`, LiteralNumberFloat, nil},
+			{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
+			{`0[xX]([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)[LlUu]*`, LiteralNumberHex, nil},
+			{`0('?[0-7]+)+[LlUu]*`, LiteralNumberOct, nil},
+			{`0[Bb][01]('?[01]+)*[LlUu]*`, LiteralNumberBin, nil},
+			{`[0-9]('?[0-9]+)*[LlUu]*`, LiteralNumberInteger, nil},
+			{`\*/`, Error, nil},
+			{`[~!%^&*+=|?:<>/-]`, Operator, nil},
+			{`[()\[\],.]`, Punctuation, nil},
+			{Words(``, `\b`, `break`, `case`, `const`, `continue`, `do`, `else`, `enum`, `extern`, `for`, `if`, `return`, `sizeof`, `static`, `struct`, `switch`, `typedef`, `union`, `while`), Keyword, nil},
+			{`(bool|float|half|long|ptrdiff_t|size_t|unsigned|u?char|u?int((8|16|32|64)_t)?|u?short)\b`, KeywordType, nil},
+			{`(bool|float|half|u?(char|int|long|short))(2|3|4)\b`, KeywordType, nil},
+			{`packed_(float|half|long|u?(char|int|short))(2|3|4)\b`, KeywordType, nil},
+			{`(float|half)(2|3|4)x(2|3|4)\b`, KeywordType, nil},
+			{`atomic_u?int\b`, KeywordType, nil},
+			{`(rg?(8|16)(u|s)norm|rgba(8|16)(u|s)norm|srgba8unorm|rgb10a2|rg11b10f|rgb9e5)\b`, KeywordType, nil},
+			{`(array|depth(2d|cube)(_array)?|depth2d_ms(_array)?|sampler|texture_buffer|texture(1|2)d(_array)?|texture2d_ms(_array)?|texture3d|texturecube(_array)?|uniform|visible_function_table)\b`, KeywordType, nil},
+			{`(true|false|NULL)\b`, NameBuiltin, nil},
+			{Words(``, `\b`, `device`, `constant`, `ray_data`, `thread`, `threadgroup`, `threadgroup_imageblock`), Keyword, nil},
+			{`([a-zA-Z_]\w*)(\s*)(:)(?!:)`, ByGroups(NameLabel, Text, Punctuation), nil},
+			{`[a-zA-Z_]\w*`, Name, nil},
+		},
+		"root": {
+			Include("whitespace"),
+			{`(fragment|kernel|vertex)?((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)`, ByGroups(Keyword, UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), Push("function")},
+			{`(fragment|kernel|vertex)?((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)`, ByGroups(Keyword, UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), nil},
+			Default(Push("statement")),
+		},
+		"classname": {
+			{`(\[\[.+\]\])(\s*)`, ByGroups(NameAttribute, Text), nil},
+			{`[a-zA-Z_]\w*`, NameClass, Pop(1)},
+			{`\s*(?=[>{])`, Text, Pop(1)},
+		},
+		"whitespace": {
+			{`^#if\s+0`, CommentPreproc, Push("if0")},
+			{`^#`, CommentPreproc, Push("macro")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#if\s+0)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("if0")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("macro")},
+			{`\n`, Text, nil},
+			{`\s+`, Text, nil},
+			{`\\\n`, Text, nil},
+			{`//(\n|[\w\W]*?[^\\]\n)`, CommentSingle, nil},
+			{`/(\\\n)?[*][\w\W]*?[*](\\\n)?/`, CommentMultiline, nil},
+			{`/(\\\n)?[*][\w\W]*`, CommentMultiline, nil},
+		},
+		"statement": {
+			Include("whitespace"),
+			Include("statements"),
+			{`[{]`, Punctuation, Push("root")},
+			{`[;}]`, Punctuation, Pop(1)},
+		},
+		"function": {
+			Include("whitespace"),
+			Include("statements"),
+			{`;`, Punctuation, nil},
+			{`\{`, Punctuation, Push()},
+			{`\}`, Punctuation, Pop(1)},
+		},
+		"macro": {
+			{`(include)(\s*(?:/[*].*?[*]/\s*)?)([^\n]+)`, ByGroups(CommentPreproc, Text, CommentPreprocFile), nil},
+			{`[^/\n]+`, CommentPreproc, nil},
+			{`/[*](.|\n)*?[*]/`, CommentMultiline, nil},
+			{`//.*?\n`, CommentSingle, Pop(1)},
+			{`/`, CommentPreproc, nil},
+			{`(?<=\\)\n`, CommentPreproc, nil},
+			{`\n`, CommentPreproc, Pop(1)},
+		},
+		"if0": {
+			{`^\s*#if.*?(?<!\\)\n`, CommentPreproc, Push()},
+			{`^\s*#el(?:se|if).*\n`, CommentPreproc, Pop(1)},
+			{`^\s*#endif.*?(?<!\\)\n`, CommentPreproc, Pop(1)},
+			{`.*?\n`, Comment, nil},
+		},
+	}
+}
diff --git a/regexp.go b/regexp.go
index 2e3219d..129e6f7 100644
--- a/regexp.go
+++ b/regexp.go
@@ -404,7 +404,7 @@ func (r *RegexLexer) maybeCompile() (err error) {
 					pattern = "(?" + rule.flags + ")" + pattern
 				}
 				pattern = `\G` + pattern
-				rule.Regexp, err = regexp2.Compile(pattern, 0)
+				rule.Regexp, err = regexp2.Compile(pattern, regexp2.RE2)
 				if err != nil {
 					return fmt.Errorf("failed to compile rule %s.%d: %s", state, i, err)
 				}


Test patch:
diff --git a/lexers/testdata/metal.actual b/lexers/testdata/metal.actual
new file mode 100644
index 0000000..39d79e1
--- /dev/null
+++ b/lexers/testdata/metal.actual
@@ -0,0 +1,43 @@
+#include <metal_stdlib>
+
+using namespace metal;
+
+// Include header shared between C code and .metal files.
+#include "AAPLShaderTypes.h"
+
+// Include header shared between all .metal files.
+#include "AAPLShaderCommon.h"
+
+struct VertexOutput
+{
+    float4 position [[position]];
+};
+
+// A depth pre-pass is necessary in forward plus rendering to produce
+// minimum and maximum depth bounds for light culling.
+vertex VertexOutput depth_pre_pass_vertex(Vertex in [[ stage_in ]],
+                                          constant AAPLFrameData & frameData [[ buffer(AAPLBufferIndexFrameData) ]])
+{
+    // Make the position a float4 to perform 4x4 matrix math on it.
+    VertexOutput out;
+    float4 position = float4(in.position, 1.0);
+
+    // Calculate the position in clip space.
+    out.position = frameData.projectionMatrix * frameData.modelViewMatrix * position;
+
+    return out;
+}
+
+fragment ColorData depth_pre_pass_fragment(VertexOutput in [[ stage_in ]])
+{
+    // Populate on-tile geometry buffer data.
+    ColorData f;
+
+    // Setting color in the depth pre-pass is unnecessary, but may make debugging easier.
+    // f.lighting = half4(0, 0, 0, 1);
+
+    // Set the depth in clip space, which you use in `AAPLCulling` to perform per-tile light culling.
+    f.depth = in.position.z;
+
+    return f;
+}
diff --git a/lexers/testdata/metal.expected b/lexers/testdata/metal.expected
new file mode 100644
index 0000000..f3a6597
--- /dev/null
+++ b/lexers/testdata/metal.expected
@@ -0,0 +1,175 @@
+[
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\u003cmetal_stdlib\u003e"},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"using"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"namespace"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"metal"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"// Include header shared between C code and .metal files.\n"},
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\"AAPLShaderTypes.h\""},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSingle","value":"// Include header shared between all .metal files.\n"},
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\"AAPLShaderCommon.h\""},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"struct"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"VertexOutput"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[position]]"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"// A depth pre-pass is necessary in forward plus rendering to produce\n// minimum and maximum depth bounds for light culling.\n"},
+  {"type":"Keyword","value":"vertex"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"depth_pre_pass_vertex"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"Vertex"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[ stage_in ]]"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n                                          "},
+  {"type":"Keyword","value":"constant"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"AAPLFrameData"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u0026"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"[["},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"buffer"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"AAPLBufferIndexFrameData"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"]])"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Make the position a float4 to perform 4x4 matrix math on it.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"in"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1.0"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"CommentSingle","value":"// Calculate the position in clip space.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"projectionMatrix"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"modelViewMatrix"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"fragment"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"ColorData"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"depth_pre_pass_fragment"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[ stage_in ]]"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Populate on-tile geometry buffer data.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"ColorData"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"CommentSingle","value":"// Setting color in the depth pre-pass is unnecessary, but may make debugging easier.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"CommentSingle","value":"// f.lighting = half4(0, 0, 0, 1);\n"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Set the depth in clip space, which you use in `AAPLCulling` to perform per-tile light culling.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"depth"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"z"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"}
+]