Repository: alecthomas/chroma
Pull request #335 (2020-02-21 13:52:06)

Terraform 0.12 syntax
Solves: https://github.com/alecthomas/chroma/issues/330

Did my best to add support for the new Terraform syntax, based on their documentation: https://www.terraform.io/docs/configuration/index.html

This is my first time working with these kinds of lexers, so please let me know if there is anything I should change.


Patch:
diff --git a/lexers/t/terraform.go b/lexers/t/terraform.go
index 05125f7..093ee30 100644
--- a/lexers/t/terraform.go
+++ b/lexers/t/terraform.go
@@ -15,55 +15,46 @@ var Terraform = internal.Register(MustNewLexer(
 	},
 	Rules{
 		"root": {
-			Include("string"),
-			Include("punctuation"),
-			Include("curly"),
-			Include("basic"),
-			Include("whitespace"),
-			{`[0-9]+`, LiteralNumber, nil},
-		},
-		"basic": {
-			{Words(`\b`, `\b`, `true`, `false`), KeywordType, nil},
-			{`\s*/\*`, CommentMultiline, Push("comment")},
-			{`\s*#.*\n`, CommentSingle, nil},
-			{`(.*?)(\s*)(=)`, ByGroups(NameAttribute, Text, Operator), nil},
-			{Words(`\b`, `\b`, `variable`, `resource`, `provider`, `provisioner`, `module`), KeywordReserved, Push("function")},
-			{Words(`\b`, `\b`, `ingress`, `egress`, `listener`, `default`, `connection`, `alias`), KeywordDeclaration, nil},
-			{`\$\{`, LiteralStringInterpol, Push("var_builtin")},
-		},
-		"function": {
-			{`(\s+)(".*")(\s+)`, ByGroups(Text, LiteralString, Text), nil},
-			Include("punctuation"),
-			Include("curly"),
-		},
-		"var_builtin": {
-			{`\$\{`, LiteralStringInterpol, Push()},
-			{Words(`\b`, `\b`, `concat`, `file`, `join`, `lookup`, `element`), NameBuiltin, nil},
-			Include("string"),
-			Include("punctuation"),
-			{`\s+`, Text, nil},
-			{`\}`, LiteralStringInterpol, Pop(1)},
+			{`[\[\](),.{}]`, Punctuation, nil},
+			{`-?[0-9]+`, LiteralNumber, nil},
+			{`=>`, Punctuation, nil},
+			{Words(``, `\b`, `true`, `false`), KeywordConstant, nil},
+			{`/(?s)\*(((?!\*/).)*)\*/`, CommentMultiline, nil},
+			{`\s*(#|//).*\n`, CommentSingle, nil},
+			{`([a-zA-Z]\w*)(\s*)(=(?!>))`, ByGroups(NameAttribute, Text, Text), nil},
+			{Words(`^\s*`, `\b`, `variable`, `data`, `resource`, `provider`, `provisioner`, `module`, `output`), KeywordReserved, nil},
+			{Words(``, `\b`, `for`, `in`), Keyword, nil},
+			{Words(``, ``, `count`, `data`, `var`, `module`, `each`), NameBuiltin, nil},
+			{Words(``, `\b`, `abs`, `ceil`, `floor`, `log`, `max`, `min`, `parseint`, `pow`, `signum`), NameBuiltin, nil},
+			{Words(``, `\b`, `chomp`, `format`, `formatlist`, `indent`, `join`, `lower`, `regex`, `regexall`, `replace`, `split`, `strrev`, `substr`, `title`, `trim`, `trimprefix`, `trimsuffix`, `trimspace`, `upper`), NameBuiltin, nil},
+			{Words(`[^.]`, `\b`, `chunklist`, `coalesce`, `coalescelist`, `compact`, `concat`, `contains`, `distinct`, `element`, `flatten`, `index`, `keys`, `length`, `list`, `lookup`, `map`, `matchkeys`, `merge`, `range`, `reverse`, `setintersection`, `setproduct`, `setsubtract`, `setunion`, `slice`, `sort`, `transpose`, `values`, `zipmap`), NameBuiltin, nil},
+			{Words(`[^.]`, `\b`, `base64decode`, `base64encode`, `base64gzip`, `csvdecode`, `jsondecode`, `jsonencode`, `urlencode`, `yamldecode`, `yamlencode`), NameBuiltin, nil},
+			{Words(``, `\b`, `abspath`, `dirname`, `pathexpand`, `basename`, `file`, `fileexists`, `fileset`, `filebase64`, `templatefile`), NameBuiltin, nil},
+			{Words(``, `\b`, `formatdate`, `timeadd`, `timestamp`), NameBuiltin, nil},
+			{Words(``, `\b`, `base64sha256`, `base64sha512`, `bcrypt`, `filebase64sha256`, `filebase64sha512`, `filemd5`, `filesha1`, `filesha256`, `filesha512`, `md5`, `rsadecrypt`, `sha1`, `sha256`, `sha512`, `uuid`, `uuidv5`), NameBuiltin, nil},
+			{Words(``, `\b`, `cidrhost`, `cidrnetmask`, `cidrsubnet`), NameBuiltin, nil},
+			{Words(``, `\b`, `can`, `tobool`, `tolist`, `tomap`, `tonumber`, `toset`, `tostring`, `try`), NameBuiltin, nil},
+			{`=(?!>)|\+|-|\*|\/|:|!|%|>|<(?!<)|>=|<=|==|!=|&&|\||\?`, Operator, nil},
+			{`\n|\s+|\\\n`, Text, nil},
+			{`[a-zA-Z]\w*`, NameOther, nil},
+			{`"`, LiteralStringDouble, Push("string")},
+			{`(?s)(<<-?)(\w+)(\n\s*(?:(?!\2).)*\s*\n\s*)(\2)`, ByGroups(Operator, Operator, String, Operator), nil},
+		},
+		"declaration": {
+			{`(\s*)("(?:\\\\|\\"|[^"])*")(\s*)`, ByGroups(Text, NameVariable, Text), nil},
+			{`\{`, Punctuation, Pop(1)},
 		},
 		"string": {
-			{`(".*")`, ByGroups(LiteralStringDouble), nil},
-		},
-		"punctuation": {
-			{`[\[\](),.]`, Punctuation, nil},
-		},
-		"curly": {
-			{`\{`, TextPunctuation, nil},
-			{`\}`, TextPunctuation, nil},
-		},
-		"comment": {
-			{`[^*/]`, CommentMultiline, nil},
-			{`/\*`, CommentMultiline, Push()},
-			{`\*/`, CommentMultiline, Pop(1)},
-			{`[*/]`, CommentMultiline, nil},
-		},
-		"whitespace": {
-			{`\n`, Text, nil},
-			{`\s+`, Text, nil},
-			{`\\\n`, Text, nil},
+			{`"`, LiteralStringDouble, Pop(1)},
+			{`\\\\`, LiteralStringDouble, nil},
+			{`\\\\"`, LiteralStringDouble, nil},
+			{`\$\{`, LiteralStringInterpol, Push("interp-inside")},
+			{`\$`, LiteralStringDouble, nil},
+			{`[^"\\\\$]+`, LiteralStringDouble, nil},
+		},
+		"interp-inside": {
+			{`\}`, LiteralStringInterpol, Pop(1)},
+			Include("root"),
 		},
 	},
 ))


Test patch:
diff --git a/lexers/testdata/terraform.actual b/lexers/testdata/terraform.actual
new file mode 100644
index 0000000..1eb6a27
--- /dev/null
+++ b/lexers/testdata/terraform.actual
@@ -0,0 +1,39 @@
+variable "some_var" {
+    default = 12
+}
+
+resource "something" "nice" {
+  name        = "hello"
+  value       = var.some_var
+  x           = var.some_var > 5 ? var.some_var : 10
+
+  cidr_blocks = [
+    for num in [1,2,3]:
+    cidrhost("10.0.0.0/24", num)
+  ]
+  str = <<-EOT
+    hello
+    world
+  EOT
+
+  /*
+    Multiline comment
+  */
+  # Single comment
+
+  dynamic "setting" {
+    for_each = var.settings
+    content {
+      namespace = setting.value["namespace"]
+      name = setting.value["name"]
+      value = setting.value["value"]
+    }
+  }
+}
+
+
+resource "other" "resource" {
+  count = 3
+  name = "resource${count.index+1}"
+}
+
diff --git a/lexers/testdata/terraform.expected b/lexers/testdata/terraform.expected
new file mode 100644
index 0000000..dcf983b
--- /dev/null
+++ b/lexers/testdata/terraform.expected
@@ -0,0 +1,165 @@
+[
+  {"type":"KeywordReserved","value":"variable"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"some_var\""},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameAttribute","value":"default"},
+  {"type":"Text","value":" = "},
+  {"type":"LiteralNumber","value":"12"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordReserved","value":"\nresource"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"something\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"nice\""},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameAttribute","value":"name"},
+  {"type":"Text","value":"        = "},
+  {"type":"LiteralStringDouble","value":"\"hello\""},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameAttribute","value":"value"},
+  {"type":"Text","value":"       = "},
+  {"type":"NameBuiltin","value":"var"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"some_var"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameAttribute","value":"x"},
+  {"type":"Text","value":"           = "},
+  {"type":"NameBuiltin","value":"var"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"some_var"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumber","value":"5"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"?"},
+  {"type":"Text","value":" "},
+  {"type":"NameBuiltin","value":"var"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"some_var"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumber","value":"10"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"NameAttribute","value":"cidr_blocks"},
+  {"type":"Text","value":" = "},
+  {"type":"Punctuation","value":"["},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"for"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"num"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"in"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralNumber","value":"1"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumber","value":"2"},
+  {"type":"Punctuation","value":","},
+  {"type":"LiteralNumber","value":"3"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameBuiltin","value":"cidrhost"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralStringDouble","value":"\"10.0.0.0/24\""},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"num"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"]"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameAttribute","value":"str"},
+  {"type":"Text","value":" = "},
+  {"type":"Operator","value":"\u003c\u003c-EOT"},
+  {"type":"LiteralString","value":"\n    hello\n    world\n  "},
+  {"type":"Operator","value":"EOT"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"CommentMultiline","value":"/*\n    Multiline comment\n  */"},
+  {"type":"CommentSingle","value":"\n  # Single comment\n"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameOther","value":"dynamic"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"setting\""},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameAttribute","value":"for_each"},
+  {"type":"Text","value":" = "},
+  {"type":"NameBuiltin","value":"var"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"settings"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameOther","value":"content"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n      "},
+  {"type":"NameAttribute","value":"namespace"},
+  {"type":"Text","value":" = "},
+  {"type":"NameOther","value":"setting"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"value"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"namespace\""},
+  {"type":"Punctuation","value":"]"},
+  {"type":"Text","value":"\n      "},
+  {"type":"NameAttribute","value":"name"},
+  {"type":"Text","value":" = "},
+  {"type":"NameOther","value":"setting"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"value"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"name\""},
+  {"type":"Punctuation","value":"]"},
+  {"type":"Text","value":"\n      "},
+  {"type":"NameAttribute","value":"value"},
+  {"type":"Text","value":" = "},
+  {"type":"NameOther","value":"setting"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"value"},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringDouble","value":"\"value\""},
+  {"type":"Punctuation","value":"]"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordReserved","value":"\n\nresource"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"other\""},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"resource\""},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameAttribute","value":"count"},
+  {"type":"Text","value":" = "},
+  {"type":"LiteralNumber","value":"3"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameAttribute","value":"name"},
+  {"type":"Text","value":" = "},
+  {"type":"LiteralStringDouble","value":"\"resource"},
+  {"type":"LiteralStringInterpol","value":"${"},
+  {"type":"NameBuiltin","value":"count"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameOther","value":"index"},
+  {"type":"Operator","value":"+"},
+  {"type":"LiteralNumber","value":"1"},
+  {"type":"LiteralStringInterpol","value":"}"},
+  {"type":"LiteralStringDouble","value":"\""},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n\n"}
+]