Repository: alecthomas/chroma
Pull request #508 (2021-05-15 22:25:41)

Add set text analyser for MySQL lexer
This PR adds text analysis to MySQL lexer. This is an exact port of pygments text analysis implementation (https://github.com/pygments/pygments/blob/master/pygments/lexers/sql.py#L747).

Testing follows the standard chroma approach for testing lexer tokenization. 

------------------

Hey Alec, since we talked last time in October 2020, we ported text analysis for various lexers from pygments to chroma in our fork. This was a working solution for wakatime, rewriting our python cli in golang.

Your comment from back then (https://github.com/alecthomas/chroma/pull/404#issuecomment-706428990) slipped through, but we would be ready to contribute the changes on text analysis from our fork upstream.

Let me know what you think :+1: 




dron22: @alecthomas Thanks for the feedback :+1: I did adjust the code accordingly and also adjusted some naming (e.g. `filename` -> `filepath`, to reflect the variable content better.

Does this fit, what you had in mind?
alecthomas: Perfect, thanks. CI is failing but once that's fixed I'll merge.
dron22: @alecthomas Alright, here we go. I fixed the CI issues, including linter and also rebased on the latest state of master :+1: 

Patch:
diff --git a/README.md b/README.md
index 5f0bce7..95d54e6 100644
--- a/README.md
+++ b/README.md
@@ -210,10 +210,10 @@ using the included Python 3 script `pygments2chroma.py`. I use something like
 the following:
 
 ```sh
-python3 ~/Projects/chroma/_tools/pygments2chroma.py \
+python3 _tools/pygments2chroma.py \
   pygments.lexers.jvm.KotlinLexer \
-  > ~/Projects/chroma/lexers/kotlin.go \
-  && gofmt -s -w ~/Projects/chroma/lexers/*.go
+  > lexers/k/kotlin.go \
+  && gofmt -s -w lexers/k/kotlin.go
 ```
 
 See notes in [pygments-lexers.txt](https://github.com/alecthomas/chroma/blob/master/pygments-lexers.txt)
diff --git a/_tools/pygments2chroma.py b/_tools/pygments2chroma.py
index 5e18e02..006ea31 100644
--- a/_tools/pygments2chroma.py
+++ b/_tools/pygments2chroma.py
@@ -12,41 +12,43 @@ from pygments.token import _TokenType
 
 
 TEMPLATE = r'''
-package lexers
+package {{package}}
 
 import (
-    . "github.com/alecthomas/chroma" // nolint
-    "github.com/alecthomas/chroma/lexers/internal"
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
 )
 
 // {{upper_name}} lexer.
 var {{upper_name}} = internal.Register(MustNewLazyLexer(
-    &Config{
-        Name:      "{{name}}",
-        Aliases:   []string{ {{#aliases}}"{{.}}", {{/aliases}} },
-        Filenames: []string{ {{#filenames}}"{{.}}", {{/filenames}} },
-        MimeTypes: []string{ {{#mimetypes}}"{{.}}", {{/mimetypes}} },
+	&Config{
+		Name:      "{{name}}",
+		{{=<% %>=}}
+		Aliases:   []string{<%#aliases%>"<%.%>", <%/aliases%>},
+		Filenames: []string{<%#filenames%>"<%.%>", <%/filenames%>},
+		MimeTypes: []string{<%#mimetypes%>"<%.%>", <%/mimetypes%>},
+		<%={{ }}=%>
 {{#re_not_multiline}}
-        NotMultiline: true,
+		NotMultiline: true,
 {{/re_not_multiline}}
 {{#re_dotall}}
-        DotAll: true,
+		DotAll: true,
 {{/re_dotall}}
 {{#re_ignorecase}}
-        CaseInsensitive: true,
+		CaseInsensitive: true,
 {{/re_ignorecase}}
-    },
-    func() Rules {
-        return Rules{
+	},
+	func() Rules {
+		return Rules{
 {{#tokens}}
-            "{{state}}": {
-                {{#rules}}
-                {{{.}}},
-                {{/rules}}
-            },
+			"{{state}}": {
+				{{#rules}}
+				{{{.}}},
+				{{/rules}}
+			},
 {{/tokens}}
-        }
-    },
+		}
+	},
 ))
 '''
 
@@ -150,7 +152,7 @@ def translate_rules(rules):
                 modifier = 'Push("%s")' % '", "'.join(rule[2])
             else:
                 raise ValueError('unsupported modifier %r' % (rule[2],))
-            out.append('{{ {}, {}, {} }}'.format(regex, emitter, modifier))
+            out.append('{{{}, {}, {}}}'.format(regex, emitter, modifier))
         elif isinstance(rule, pygments_lexer.include):
             out.append('Include("{}")'.format(rule))
         elif isinstance(rule, pygments_lexer.default):
@@ -185,9 +187,10 @@ def main():
     assert issubclass(lexer_cls, pygments_lexer.RegexLexer), 'can only translate from RegexLexer'
 
     print(pystache.render(TEMPLATE, TemplateView(
+        package=lexer_cls.name.lower()[0],
         name=lexer_cls.name,
         regex_flags=lexer_cls.flags,
-        upper_name=to_camel_case(lexer_cls.name),
+        upper_name=to_camel_case(re.sub(r'\W', '_', lexer_cls.name)),
         aliases=lexer_cls.aliases,
         filenames=lexer_cls.filenames,
         mimetypes=lexer_cls.mimetypes,
diff --git a/lexers/m/metal.go b/lexers/m/metal.go
new file mode 100644
index 0000000..4a9ba2b
--- /dev/null
+++ b/lexers/m/metal.go
@@ -0,0 +1,101 @@
+package m
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Metal lexer.
+var Metal = internal.Register(MustNewLazyLexer(
+	&Config{
+		Name:      "Metal",
+		Aliases:   []string{"metal"},
+		Filenames: []string{"*.metal"},
+		MimeTypes: []string{"text/x-metal"},
+		EnsureNL:  true,
+	},
+	metalRules,
+))
+
+func metalRules() Rules {
+	return Rules{
+		"statements": {
+			{Words(``, `\b`, `namespace`, `operator`, `template`, `this`, `using`, `constexpr`), Keyword, nil},
+			{`(enum)\b(\s+)(class)\b(\s*)`, ByGroups(Keyword, Text, Keyword, Text), Push("classname")},
+			{`(class|struct|enum|union)\b(\s*)`, ByGroups(Keyword, Text), Push("classname")},
+			{`\[\[.+\]\]`, NameAttribute, nil},
+			{`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*`, LiteralNumberFloat, nil},
+			{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
+			{`0[xX]([0-9A-Fa-f]('?[0-9A-Fa-f]+)*)[LlUu]*`, LiteralNumberHex, nil},
+			{`0('?[0-7]+)+[LlUu]*`, LiteralNumberOct, nil},
+			{`0[Bb][01]('?[01]+)*[LlUu]*`, LiteralNumberBin, nil},
+			{`[0-9]('?[0-9]+)*[LlUu]*`, LiteralNumberInteger, nil},
+			{`\*/`, Error, nil},
+			{`[~!%^&*+=|?:<>/-]`, Operator, nil},
+			{`[()\[\],.]`, Punctuation, nil},
+			{Words(``, `\b`, `break`, `case`, `const`, `continue`, `do`, `else`, `enum`, `extern`, `for`, `if`, `return`, `sizeof`, `static`, `struct`, `switch`, `typedef`, `union`, `while`), Keyword, nil},
+			{`(bool|float|half|long|ptrdiff_t|size_t|unsigned|u?char|u?int((8|16|32|64)_t)?|u?short)\b`, KeywordType, nil},
+			{`(bool|float|half|u?(char|int|long|short))(2|3|4)\b`, KeywordType, nil},
+			{`packed_(float|half|long|u?(char|int|short))(2|3|4)\b`, KeywordType, nil},
+			{`(float|half)(2|3|4)x(2|3|4)\b`, KeywordType, nil},
+			{`atomic_u?int\b`, KeywordType, nil},
+			{`(rg?(8|16)(u|s)norm|rgba(8|16)(u|s)norm|srgba8unorm|rgb10a2|rg11b10f|rgb9e5)\b`, KeywordType, nil},
+			{`(array|depth(2d|cube)(_array)?|depth2d_ms(_array)?|sampler|texture_buffer|texture(1|2)d(_array)?|texture2d_ms(_array)?|texture3d|texturecube(_array)?|uniform|visible_function_table)\b`, KeywordType, nil},
+			{`(true|false|NULL)\b`, NameBuiltin, nil},
+			{Words(``, `\b`, `device`, `constant`, `ray_data`, `thread`, `threadgroup`, `threadgroup_imageblock`), Keyword, nil},
+			{`([a-zA-Z_]\w*)(\s*)(:)(?!:)`, ByGroups(NameLabel, Text, Punctuation), nil},
+			{`[a-zA-Z_]\w*`, Name, nil},
+		},
+		"root": {
+			Include("whitespace"),
+			{`(fragment|kernel|vertex)?((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)`, ByGroups(Keyword, UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), Push("function")},
+			{`(fragment|kernel|vertex)?((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)`, ByGroups(Keyword, UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), nil},
+			Default(Push("statement")),
+		},
+		"classname": {
+			{`(\[\[.+\]\])(\s*)`, ByGroups(NameAttribute, Text), nil},
+			{`[a-zA-Z_]\w*`, NameClass, Pop(1)},
+			{`\s*(?=[>{])`, Text, Pop(1)},
+		},
+		"whitespace": {
+			{`^#if\s+0`, CommentPreproc, Push("if0")},
+			{`^#`, CommentPreproc, Push("macro")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#if\s+0)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("if0")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("macro")},
+			{`\n`, Text, nil},
+			{`\s+`, Text, nil},
+			{`\\\n`, Text, nil},
+			{`//(\n|[\w\W]*?[^\\]\n)`, CommentSingle, nil},
+			{`/(\\\n)?[*][\w\W]*?[*](\\\n)?/`, CommentMultiline, nil},
+			{`/(\\\n)?[*][\w\W]*`, CommentMultiline, nil},
+		},
+		"statement": {
+			Include("whitespace"),
+			Include("statements"),
+			{`[{]`, Punctuation, Push("root")},
+			{`[;}]`, Punctuation, Pop(1)},
+		},
+		"function": {
+			Include("whitespace"),
+			Include("statements"),
+			{`;`, Punctuation, nil},
+			{`\{`, Punctuation, Push()},
+			{`\}`, Punctuation, Pop(1)},
+		},
+		"macro": {
+			{`(include)(\s*(?:/[*].*?[*]/\s*)?)([^\n]+)`, ByGroups(CommentPreproc, Text, CommentPreprocFile), nil},
+			{`[^/\n]+`, CommentPreproc, nil},
+			{`/[*](.|\n)*?[*]/`, CommentMultiline, nil},
+			{`//.*?\n`, CommentSingle, Pop(1)},
+			{`/`, CommentPreproc, nil},
+			{`(?<=\\)\n`, CommentPreproc, nil},
+			{`\n`, CommentPreproc, Pop(1)},
+		},
+		"if0": {
+			{`^\s*#if.*?(?<!\\)\n`, CommentPreproc, Push()},
+			{`^\s*#el(?:se|if).*\n`, CommentPreproc, Pop(1)},
+			{`^\s*#endif.*?(?<!\\)\n`, CommentPreproc, Pop(1)},
+			{`.*?\n`, Comment, nil},
+		},
+	}
+}
diff --git a/lexers/m/mysql.go b/lexers/m/mysql.go
index 7e08e24..2738628 100644
--- a/lexers/m/mysql.go
+++ b/lexers/m/mysql.go
@@ -1,10 +1,17 @@
 package m
 
 import (
+	"regexp"
+
 	. "github.com/alecthomas/chroma" // nolint
 	"github.com/alecthomas/chroma/lexers/internal"
 )
 
+var (
+	mysqlAnalyserNameBetweenBacktickRe = regexp.MustCompile("`[a-zA-Z_]\\w*`")
+	mysqlAnalyserNameBetweenBracketRe  = regexp.MustCompile(`\[[a-zA-Z_]\w*\]`)
+)
+
 // MySQL lexer.
 var MySQL = internal.Register(MustNewLazyLexer(
 	&Config{
@@ -16,7 +23,25 @@ var MySQL = internal.Register(MustNewLazyLexer(
 		CaseInsensitive: true,
 	},
 	mySQLRules,
-))
+).SetAnalyser(func(text string) float32 {
+	nameBetweenBacktickCount := len(mysqlAnalyserNameBetweenBacktickRe.FindAllString(text, -1))
+	nameBetweenBracketCount := len(mysqlAnalyserNameBetweenBracketRe.FindAllString(text, -1))
+
+	var result float32
+
+	// Same logic as above in the TSQL analysis.
+	dialectNameCount := nameBetweenBacktickCount + nameBetweenBracketCount
+	if dialectNameCount >= 1 && nameBetweenBacktickCount >= (2*nameBetweenBracketCount) {
+		// Found at least twice as many `name` as [name].
+		result += 0.5
+	} else if nameBetweenBacktickCount > nameBetweenBracketCount {
+		result += 0.2
+	} else if nameBetweenBacktickCount > 0 {
+		result += 0.1
+	}
+
+	return result
+}))
 
 func mySQLRules() Rules {
 	return Rules{


Test patch:
diff --git a/lexers/lexers_test.go b/lexers/lexers_test.go
index 6fbb900..0c5d840 100644
--- a/lexers/lexers_test.go
+++ b/lexers/lexers_test.go
@@ -5,6 +5,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -79,6 +80,11 @@ func TestLexers(t *testing.T) {
 	assert.NoError(t, err)
 
 	for _, file := range files {
+		// skip text analysis test files
+		if file.Name() == "analysis" {
+			continue
+		}
+
 		if file.IsDir() {
 			dirname := filepath.Join("testdata", file.Name())
 			lexer := lexers.Get(file.Name())
@@ -117,3 +123,52 @@ func TestLexers(t *testing.T) {
 		}
 	}
 }
+
+func FileTestAnalysis(t *testing.T, lexer chroma.Lexer, actualFilepath, expectedFilepath string) {
+	t.Helper()
+	t.Run(lexer.Config().Name+"/"+actualFilepath, func(t *testing.T) {
+		expectedData, err := ioutil.ReadFile(expectedFilepath)
+		assert.NoError(t, err)
+
+		analyser, ok := lexer.(chroma.Analyser)
+		assert.True(t, ok, "lexer %q does not set analyser", lexer.Config().Name)
+
+		data, err := ioutil.ReadFile(actualFilepath)
+		assert.NoError(t, err)
+
+		actual := analyser.AnalyseText(string(data))
+
+		if os.Getenv("RECORD") == "true" {
+			// Update the expected file with the generated output of this lexer
+			f, err := os.Create(expectedFilepath)
+			defer f.Close() // nolint: gosec
+			assert.NoError(t, err)
+
+			_, err = f.WriteString(strconv.FormatFloat(float64(actual), 'f', -1, 32))
+			assert.NoError(t, err)
+		} else {
+			expected, err := strconv.ParseFloat(strings.TrimSpace(string(expectedData)), 32)
+			assert.NoError(t, err)
+
+			assert.Equal(t, float32(expected), actual)
+		}
+	})
+}
+
+func TestLexersTextAnalyser(t *testing.T) {
+	files, err := filepath.Glob("testdata/analysis/*.actual")
+	assert.NoError(t, err)
+
+	for _, actualFilepath := range files {
+		filename := filepath.Base(actualFilepath)
+		baseFilename := strings.TrimSuffix(filename, filepath.Ext(filename))
+		lexerName := strings.Split(baseFilename, ".")[0]
+
+		lexer := lexers.Get(lexerName)
+		assert.NotNil(t, lexer, "no lexer found for name %q", lexerName)
+
+		expectedFilepath := "testdata/analysis/" + baseFilename + ".expected"
+
+		FileTestAnalysis(t, lexer, actualFilepath, expectedFilepath)
+	}
+}
diff --git a/lexers/testdata/analysis/mysql.backtick.actual b/lexers/testdata/analysis/mysql.backtick.actual
new file mode 100644
index 0000000..d892734
--- /dev/null
+++ b/lexers/testdata/analysis/mysql.backtick.actual
@@ -0,0 +1,1 @@
+CREATE TABLE `my_table` (id INT);
diff --git a/lexers/testdata/analysis/mysql.backtick.expected b/lexers/testdata/analysis/mysql.backtick.expected
new file mode 100644
index 0000000..ea2303b
--- /dev/null
+++ b/lexers/testdata/analysis/mysql.backtick.expected
@@ -0,0 +1,1 @@
+0.5
\ No newline at end of file
diff --git a/lexers/testdata/metal.actual b/lexers/testdata/metal.actual
new file mode 100644
index 0000000..39d79e1
--- /dev/null
+++ b/lexers/testdata/metal.actual
@@ -0,0 +1,43 @@
+#include <metal_stdlib>
+
+using namespace metal;
+
+// Include header shared between C code and .metal files.
+#include "AAPLShaderTypes.h"
+
+// Include header shared between all .metal files.
+#include "AAPLShaderCommon.h"
+
+struct VertexOutput
+{
+    float4 position [[position]];
+};
+
+// A depth pre-pass is necessary in forward plus rendering to produce
+// minimum and maximum depth bounds for light culling.
+vertex VertexOutput depth_pre_pass_vertex(Vertex in [[ stage_in ]],
+                                          constant AAPLFrameData & frameData [[ buffer(AAPLBufferIndexFrameData) ]])
+{
+    // Make the position a float4 to perform 4x4 matrix math on it.
+    VertexOutput out;
+    float4 position = float4(in.position, 1.0);
+
+    // Calculate the position in clip space.
+    out.position = frameData.projectionMatrix * frameData.modelViewMatrix * position;
+
+    return out;
+}
+
+fragment ColorData depth_pre_pass_fragment(VertexOutput in [[ stage_in ]])
+{
+    // Populate on-tile geometry buffer data.
+    ColorData f;
+
+    // Setting color in the depth pre-pass is unnecessary, but may make debugging easier.
+    // f.lighting = half4(0, 0, 0, 1);
+
+    // Set the depth in clip space, which you use in `AAPLCulling` to perform per-tile light culling.
+    f.depth = in.position.z;
+
+    return f;
+}
diff --git a/lexers/testdata/metal.expected b/lexers/testdata/metal.expected
new file mode 100644
index 0000000..f3a6597
--- /dev/null
+++ b/lexers/testdata/metal.expected
@@ -0,0 +1,175 @@
+[
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\u003cmetal_stdlib\u003e"},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"using"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"namespace"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"metal"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"// Include header shared between C code and .metal files.\n"},
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\"AAPLShaderTypes.h\""},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSingle","value":"// Include header shared between all .metal files.\n"},
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\"AAPLShaderCommon.h\""},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"struct"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"VertexOutput"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[position]]"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"};"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentSingle","value":"// A depth pre-pass is necessary in forward plus rendering to produce\n// minimum and maximum depth bounds for light culling.\n"},
+  {"type":"Keyword","value":"vertex"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"depth_pre_pass_vertex"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"Vertex"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[ stage_in ]]"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":"\n                                          "},
+  {"type":"Keyword","value":"constant"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"AAPLFrameData"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u0026"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"[["},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"buffer"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"AAPLBufferIndexFrameData"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"]])"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Make the position a float4 to perform 4x4 matrix math on it.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"float4"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"in"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberFloat","value":"1.0"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"CommentSingle","value":"// Calculate the position in clip space.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"projectionMatrix"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"frameData"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"modelViewMatrix"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"out"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"fragment"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"ColorData"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"depth_pre_pass_fragment"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"VertexOutput"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"NameAttribute","value":"[[ stage_in ]]"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Populate on-tile geometry buffer data.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"ColorData"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"CommentSingle","value":"// Setting color in the depth pre-pass is unnecessary, but may make debugging easier.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"CommentSingle","value":"// f.lighting = half4(0, 0, 0, 1);\n"},
+  {"type":"Text","value":"\n    "},
+  {"type":"CommentSingle","value":"// Set the depth in clip space, which you use in `AAPLCulling` to perform per-tile light culling.\n"},
+  {"type":"Text","value":"    "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"depth"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"in"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"position"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"z"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n    "},
+  {"type":"Keyword","value":"return"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"f"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"}
+]