Repository: alecthomas/chroma
Pull request #511 (2021-05-18 16:14:24)

Add Metal lexer
Since Metal shading language is a subset of C++14, the lexer is a reduced version of the C++ one. Check the language specification [here](https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf).


Patch:
diff --git a/lexers/m/metal.go b/lexers/m/metal.go
new file mode 100644
index 0000000..4a9ba2b
--- /dev/null
+++ b/lexers/m/metal.go
@@ -0,0 +1,101 @@
+package m
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Metal lexer.
+var Metal = internal.Register(MustNewLazyLexer(
+	&Config{
+		Name:      "Metal",
+		Aliases:   []string{"metal"},
+		Filenames: []string{"*.metal"},
+		MimeTypes: []string{"text/x-metal"},
+		EnsureNL:  true,
+	},
+	metalRules,
+))
+
+func metalRules() Rules {
+	return Rules{
+		"statements": {
+			{Words(``, `\b`, `namespace`, `operator`, `template`, `this`, `using`, `constexpr`), Keyword, nil},
+			{`(enum)\b(\s+)(class)\b(\s*)`, ByGroups(Keyword, Text, Keyword, Text), Push("classname")},
+			{`(class|struct|enum|union)\b(\s*)`, ByGroups(Keyword, Text), Push("classname")},
+			{`\[\[.+\]\]`, NameAttribute, nil},
+			{`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*`, LiteralNumberFloat, nil},
+			{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
+			{`0[xX]([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)[LlUu]*`, LiteralNumberHex, nil},
+			{`0('?[0-7]+)+[LlUu]*`, LiteralNumberOct, nil},
+			{`0[Bb][01]('?[01]+)*[LlUu]*`, LiteralNumberBin, nil},
+			{`[0-9]('?[0-9]+)*[LlUu]*`, LiteralNumberInteger, nil},
+			{`\*/`, Error, nil},
+			{`[~!%^&*+=|?:<>/-]`, Operator, nil},
+			{`[()\[\],.]`, Punctuation, nil},
+			{Words(``, `\b`, `break`, `case`, `const`, `continue`, `do`, `else`, `enum`, `extern`, `for`, `if`, `return`, `sizeof`, `static`, `struct`, `switch`, `typedef`, `union`, `while`), Keyword, nil},
+			{`(bool|float|half|long|ptrdiff_t|size_t|unsigned|u?char|u?int((8|16|32|64)_t)?|u?short)\b`, KeywordType, nil},
+			{`(bool|float|half|u?(char|int|long|short))(2|3|4)\b`, KeywordType, nil},
+			{`packed_(float|half|long|u?(char|int|short))(2|3|4)\b`, KeywordType, nil},
+			{`(float|half)(2|3|4)x(2|3|4)\b`, KeywordType, nil},
+			{`atomic_u?int\b`, KeywordType, nil},
+			{`(rg?(8|16)(u|s)norm|rgba(8|16)(u|s)norm|srgba8unorm|rgb10a2|rg11b10f|rgb9e5)\b`, KeywordType, nil},
+			{`(array|depth(2d|cube)(_array)?|depth2d_ms(_array)?|sampler|texture_buffer|texture(1|2)d(_array)?|texture2d_ms(_array)?|texture3d|texturecube(_array)?|uniform|visible_function_table)\b`, KeywordType, nil},
+			{`(true|false|NULL)\b`, NameBuiltin, nil},
+			{Words(``, `\b`, `device`, `constant`, `ray_data`, `thread`, `threadgroup`, `threadgroup_imageblock`), Keyword, nil},
+			{`([a-zA-Z_]\w*)(\s*)(:)(?!:)`, ByGroups(NameLabel, Text, Punctuation), nil},
+			{`[a-zA-Z_]\w*`, Name, nil},
+		},
+		"root": {
+			Include("whitespace"),
+			{`(fragment|kernel|vertex)?((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)`, ByGroups(Keyword, UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), Push("function")},
+			{`(fragment|kernel|vertex)?((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)`, ByGroups(Keyword, UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), nil},
+			Default(Push("statement")),
+		},
+		"classname": {
+			{`(\[\[.+\]\])(\s*)`, ByGroups(NameAttribute, Text), nil},
+			{`[a-zA-Z_]\w*`, NameClass, Pop(1)},
+			{`\s*(?=[>{])`, Text, Pop(1)},
+		},
+		"whitespace": {
+			{`^#if\s+0`, CommentPreproc, Push("if0")},
+			{`^#`, CommentPreproc, Push("macro")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#if\s+0)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("if0")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("macro")},
+			{`\n`, Text, nil},
+			{`\s+`, Text, nil},
+			{`\\\n`, Text, nil},
+			{`//(\n|[\w\W]*?[^\\]\n)`, CommentSingle, nil},
+			{`/(\\\n)?[*][\w\W]*?[*](\\\n)?/`, CommentMultiline, nil},
+			{`/(\\\n)?[*][\w\W]*`, CommentMultiline, nil},
+		},
+		"statement": {
+			Include("whitespace"),
+			Include("statements"),
+			{`[{]`, Punctuation, Push("root")},
+			{`[;}]`, Punctuation, Pop(1)},
+		},
+		"function": {
+			Include("whitespace"),
+			Include("statements"),
+			{`;`, Punctuation, nil},
+			{`\{`, Punctuation, Push()},
+			{`\}`, Punctuation, Pop(1)},
+		},
+		"macro": {
+			{`(include)(\s*(?:/[*].*?[*]/\s*)?)([^\n]+)`, ByGroups(CommentPreproc, Text, CommentPreprocFile), nil},
+			{`[^/\n]+`, CommentPreproc, nil},
+			{`/[*](.|\n)*?[*]/`, CommentMultiline, nil},
+			{`//.*?\n`, CommentSingle, Pop(1)},
+			{`/`, CommentPreproc, nil},
+			{`(?<=\\)\n`, CommentPreproc, nil},
+			{`\n`, CommentPreproc, Pop(1)},
+		},
+		"if0": {
+			{`^\s*#if.*?(?<!\\)\n`, CommentPreproc, Push()},
+			{`^\s*#el(?:se|if).*\n`, CommentPreproc, Pop(1)},
+			{`^\s*#endif.*?(?<!\\)\n`, CommentPreproc, Pop(1)},
+			{`.*?\n`, Comment, nil},
+		},
+	}
+}


Test patch:
diff --git a/lexers/testdata/metal.actual b/lexers/testdata/metal.actual
new file mode 100644
index 0000000..39d79e1
--- /dev/null
+++ b/lexers/testdata/metal.actual
@@ -0,0 +1,43 @@
+#include <metal_stdlib>
+
+using namespace metal;
+
+// Include header shared between C code and .metal files.
+#include "AAPLShaderTypes.h"
+
+// Include header shared between all .metal files.
+#include "AAPLShaderCommon.h"
+
+struct VertexOutput
+{
+    float4 position [[position]];
+};
+
+// A depth pre-pass is necessary in forward plus rendering to produce
+// minimum and maximum depth bounds for light culling.
+vertex VertexOutput depth_pre_pass_vertex(Vertex in [[ stage_in ]],
+                                          constant AAPLFrameData & frameData [[ buffer(AAPLBufferIndexFrameData) ]])
+{
+    // Make the position a float4 to perform 4x4 matrix math on it.
+    VertexOutput out;
+    float4 position = float4(in.position, 1.0);
+
+    // Calculate the position in clip space.
+    out.position = frameData.projectionMatrix * frameData.modelViewMatrix * position;
+
+    return out;
+}
+
+fragment ColorData depth_pre_pass_fragment(VertexOutput in [[ stage_in ]])
+{
+    // Populate on-tile geometry buffer data.
+    ColorData f;
+
+    // Setting color in the depth pre-pass is unnecessary, but may make debugging easier.
+    // f.lighting = half4(0, 0, 0, 1);
+
+    // Set the depth in clip space, which you use in `AAPLCulling` to perform per-tile light culling.
+    f.depth = in.position.z;
+
+    return f;
+}
diff --git a/lexers/testdata/metal.expected b/lexers/testdata/metal.expected
new file mode 100644
index 0000000..f3a6597
--- /dev/null
+++ b/lexers/testdata/metal.expected
@@ -0,0 +1,175 @@
+[
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\u003cmetal_stdlib\u003e"},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"using"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"namespace"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"metal"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"// Include header shared between C code and .metal files.\n"},
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\"AAPLShaderTypes.h\""},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSingle","value":"// Include header shared between all .metal files.\n"},
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\"AAPLShaderCommon.h\""},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"struct"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"VertexOutput"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[position]]"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"// A depth pre-pass is necessary in forward plus rendering to produce\n// minimum and maximum depth bounds for light culling.\n"},
+  {"type":"Keyword","value":"vertex"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"depth_pre_pass_vertex"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"Vertex"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[ stage_in ]]"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n                                          "},
+  {"type":"Keyword","value":"constant"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"AAPLFrameData"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u0026"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"[["},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"buffer"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"AAPLBufferIndexFrameData"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"]])"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Make the position a float4 to perform 4x4 matrix math on it.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"in"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1.0"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"CommentSingle","value":"// Calculate the position in clip space.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"projectionMatrix"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"modelViewMatrix"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"fragment"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"ColorData"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"depth_pre_pass_fragment"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[ stage_in ]]"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Populate on-tile geometry buffer data.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"ColorData"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"CommentSingle","value":"// Setting color in the depth pre-pass is unnecessary, but may make debugging easier.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"CommentSingle","value":"// f.lighting = half4(0, 0, 0, 1);\n"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Set the depth in clip space, which you use in `AAPLCulling` to perform per-tile light culling.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"depth"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"z"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"}
+]