Repository: alecthomas/chroma
Pull request #311 (2019-12-02 03:44:47)

Add lexer for LLVM TableGen
Add a basic lexer for LLVM's TableGen (https://llvm.org/docs/TableGen/LangRef.html). Used an example from a tutorial (modulo verifying an additional template highlighting) for testdata and I just chose the mime type arbitrarily.


Patch:
diff --git a/lexers/t/tablegen.go b/lexers/t/tablegen.go
new file mode 100644
index 0000000..18c6978
--- /dev/null
+++ b/lexers/t/tablegen.go
@@ -0,0 +1,42 @@
+package t
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// TableGen lexer.
+var Tablegen = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "TableGen",
+		Aliases:   []string{"tablegen"},
+		Filenames: []string{"*.td"},
+		MimeTypes: []string{"text/x-tablegen"},
+	},
+	Rules{
+		"root": {
+			Include("macro"),
+			Include("whitespace"),
+			{`c?"[^"]*?"`, LiteralString, nil},
+			Include("keyword"),
+			{`\$[_a-zA-Z][_\w]*`, NameVariable, nil},
+			{`\d*[_a-zA-Z][_\w]*`, NameVariable, nil},
+			{`\[\{[\w\W]*?\}\]`, LiteralString, nil},
+			{`[+-]?\d+|0x[\da-fA-F]+|0b[01]+`, LiteralNumber, nil},
+			{`[=<>{}\[\]()*.,!:;]`, Punctuation, nil},
+		},
+		"macro": {
+			{`(#include\s+)("[^"]*")`, ByGroups(CommentPreproc, LiteralString), nil},
+			{`^\s*#(ifdef|ifndef)\s+[_\w][_\w\d]*`, CommentPreproc, nil},
+			{`^\s*#define\s+[_\w][_\w\d]*`, CommentPreproc, nil},
+			{`^\s*#endif`, CommentPreproc, nil},
+		},
+		"whitespace": {
+			{`(\n|\s)+`, Text, nil},
+			{`//.*?\n`, Comment, nil},
+		},
+		"keyword": {
+			{Words(``, `\b`, `bit`, `bits`, `class`, `code`, `dag`, `def`, `defm`, `field`, `foreach`, `in`, `int`, `let`, `list`, `multiclass`, `string`), Keyword, nil},
+		},
+	},
+))


Test patch:
diff --git a/lexers/testdata/tablegen.actual b/lexers/testdata/tablegen.actual
new file mode 100644
index 0000000..bf563df
--- /dev/null
+++ b/lexers/testdata/tablegen.actual
@@ -0,0 +1,29 @@
+#include "main"
+
+#ifndef SHOULD_DEF
+#define SHOULD_DEF
+def ConstantOp : Toy_Op<"constant", [NoSideEffect]> {
+  // Provide a summary and description for this operation. This can be used to
+  // auto-generate documenatation of the operations within our dialect.
+  let summary = "constant operation";
+  let description = [{
+    Constant operation turns a literal into an SSA value. The data is attached
+    to the operation as an attribute. For example:
+
+      %0 = "toy.constant"()
+         { value = dense<[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]> : tensor<2x3xf64> }
+        : () -> tensor<2x3xf64>
+  }];
+
+  // The constant operation takes an attribute as the only input.
+  // `F64ElementsAttr` corresponds to a 64-bit floating-point ElementsAttr.
+  let arguments = (ins F64ElementsAttr:$value);
+
+  // The generic call operation returns a single value of TensorType.
+  // F64Tensor corresponds to a 64-bit floating-point TensorType.
+  let results = (outs F64Tensor<10>);
+
+  // Add additional verification logic to the constant operation.
+  let verifier = [{ return ::verify(*this); }];
+}
+#endif  // SHOULD_DEF
diff --git a/lexers/testdata/tablegen.expected b/lexers/testdata/tablegen.expected
new file mode 100644
index 0000000..1b6e797
--- /dev/null
+++ b/lexers/testdata/tablegen.expected
@@ -0,0 +1,100 @@
+[
+  {"type":"CommentPreproc","value":"#include "},
+  {"type":"LiteralString","value":"\"main\""},
+  {"type":"Text","value":"\n\n"},
+  {"type":"CommentPreproc","value":"#ifndef SHOULD_DEF"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentPreproc","value":"#define SHOULD_DEF"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"def"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"ConstantOp"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"Toy_Op"},
+  {"type":"Punctuation","value":"\u003c"},
+  {"type":"LiteralString","value":"\"constant\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"NameVariable","value":"NoSideEffect"},
+  {"type":"Punctuation","value":"]\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Comment","value":"// Provide a summary and description for this operation. This can be used to\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Comment","value":"// auto-generate documenatation of the operations within our dialect.\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"summary"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"constant operation\""},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"description"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"[{\n    Constant operation turns a literal into an SSA value. The data is attached\n    to the operation as an attribute. For example:\n\n      %0 = \"toy.constant\"()\n         { value = dense\u003c[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]\u003e : tensor\u003c2x3xf64\u003e }\n        : () -\u003e tensor\u003c2x3xf64\u003e\n  }]"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Comment","value":"// The constant operation takes an attribute as the only input.\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Comment","value":"// `F64ElementsAttr` corresponds to a 64-bit floating-point ElementsAttr.\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"arguments"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"ins"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"F64ElementsAttr"},
+  {"type":"Punctuation","value":":"},
+  {"type":"NameVariable","value":"$value"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Comment","value":"// The generic call operation returns a single value of TensorType.\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Comment","value":"// F64Tensor corresponds to a 64-bit floating-point TensorType.\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"results"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameVariable","value":"outs"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"F64Tensor"},
+  {"type":"Punctuation","value":"\u003c"},
+  {"type":"LiteralNumber","value":"10"},
+  {"type":"Punctuation","value":"\u003e);"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Comment","value":"// Add additional verification logic to the constant operation.\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Keyword","value":"let"},
+  {"type":"Text","value":" "},
+  {"type":"NameVariable","value":"verifier"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"[{ return ::verify(*this); }]"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentPreproc","value":"#endif"},
+  {"type":"Text","value":"  "},
+  {"type":"Comment","value":"// SHOULD_DEF\n"}
+]