Repository: alecthomas/chroma
Pull request #395 (2020-09-03 08:54:46)

Improve Elixir lexer based on Pygments implementation
**Description**
It appears that not all the rules from the Pygments Elixir lexer were correctly ported over to Chroma. It was missing mostly the builtin keywords and declarations that are processed in the [`get_tokens_unprocessed`](https://github.com/pygments/pygments/blob/master/pygments/lexers/erlang.py#L269) function of the Pygments Elixir lexer.

This commit adds the missing lines to the Elixir lexer and updates the tests accordingly. Below I've added some screenshots of before and after this patch and a comparison to the formatted output of Pygments itself.

**Examples**

<table>
<tr>
<td><img alt="pygments" src="https://user-images.githubusercontent.com/639858/92093246-6b29b800-edd3-11ea-9acf-0d300a95689a.png" /></td>
<td><img alt="chroma-unpatched" src="https://user-images.githubusercontent.com/639858/92093319-809ee200-edd3-11ea-9d4f-28ccc5bb5c1c.png" /></td>
<td><img alt="chroma-patched" src="https://user-images.githubusercontent.com/639858/92093367-901e2b00-edd3-11ea-8210-3ff0451d5297.png" /></td>
</tr>
<tr>
<td>Pygments</td>
<td>Chroma / Unpatched</td>
<td>Chroma / Patched</td>
</tr>
</table>

**Issues**
This should address https://github.com/alecthomas/chroma/issues/379

alecthomas: I appreciate it, thanks for the PR.

Patch:
diff --git a/lexers/e/elixir.go b/lexers/e/elixir.go
index 4073029..11dc844 100644
--- a/lexers/e/elixir.go
+++ b/lexers/e/elixir.go
@@ -28,6 +28,13 @@ var Elixir = internal.Register(MustNewLexer(
 			{`:"`, LiteralStringSymbol, Push("string_double_atom")},
 			{`:'`, LiteralStringSymbol, Push("string_single_atom")},
 			{`((?:\.\.\.|<<>>|%\{\}|%|\{\})|(?:(?:\.\.\.|[a-z_]\w*[!?]?)|[A-Z]\w*(?:\.[A-Z]\w*)*|(?:\<\<\<|\>\>\>|\|\|\||\&\&\&|\^\^\^|\~\~\~|\=\=\=|\!\=\=|\~\>\>|\<\~\>|\|\~\>|\<\|\>|\=\=|\!\=|\<\=|\>\=|\&\&|\|\||\<\>|\+\+|\-\-|\|\>|\=\~|\-\>|\<\-|\||\.|\=|\~\>|\<\~|\<|\>|\+|\-|\*|\/|\!|\^|\&)))(:)(?=\s|\n)`, ByGroups(LiteralStringSymbol, Punctuation), nil},
+			{`(fn|do|end|after|else|rescue|catch)\b`, Keyword, nil},
+			{`(not|and|or|when|in)\b`, OperatorWord, nil},
+			{`(case|cond|for|if|unless|try|receive|raise|quote|unquote|unquote_splicing|throw|super|while)\b`, Keyword, nil},
+			{`(def|defp|defmodule|defprotocol|defmacro|defmacrop|defdelegate|defexception|defstruct|defimpl|defcallback)\b`, KeywordDeclaration, nil},
+			{`(import|require|use|alias)\b`, KeywordNamespace, nil},
+			{`(nil|true|false)\b`, NameConstant, nil},
+			{`(_|__MODULE__|__DIR__|__ENV__|__CALLER__)\b`, NamePseudo, nil},
 			{`@(?:\.\.\.|[a-z_]\w*[!?]?)`, NameAttribute, nil},
 			{`(?:\.\.\.|[a-z_]\w*[!?]?)`, Name, nil},
 			{`(%?)([A-Z]\w*(?:\.[A-Z]\w*)*)`, ByGroups(Punctuation, NameClass), nil},


Test patch:
diff --git a/lexers/testdata/elixir.actual b/lexers/testdata/elixir.actual
new file mode 100644
index 0000000..11302c7
--- /dev/null
+++ b/lexers/testdata/elixir.actual
@@ -0,0 +1,12 @@
+defmodule Test do
+  defmodule Formatter do
+    defstruct [:name]
+  end
+
+  def hello(name \\ "chroma") do
+    case name do
+      "chroma" -> IO.puts("Hello, CHROMA!!")
+      other -> raise "no others please"
+    end
+  end
+end
diff --git a/lexers/testdata/elixir.expected b/lexers/testdata/elixir.expected
new file mode 100644
index 0000000..6ae9c03
--- /dev/null
+++ b/lexers/testdata/elixir.expected
@@ -0,0 +1,66 @@
+[
+  {"type":"KeywordDeclaration","value":"defmodule"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Test"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"do"},
+  {"type":"Text","value":"\n  "},
+  {"type":"KeywordDeclaration","value":"defmodule"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"Formatter"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"do"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordDeclaration","value":"defstruct"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"["},
+  {"type":"LiteralStringSymbol","value":":name"},
+  {"type":"Punctuation","value":"]"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"KeywordDeclaration","value":"def"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"hello"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"name"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"\\\\"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"chroma\""},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"do"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"case"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"name"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"do"},
+  {"type":"Text","value":"\n      "},
+  {"type":"LiteralStringDouble","value":"\"chroma\""},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"IO"},
+  {"type":"Operator","value":"."},
+  {"type":"Name","value":"puts"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralStringDouble","value":"\"Hello, CHROMA!!\""},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"other"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"Keyword","value":"raise"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralStringDouble","value":"\"no others please\""},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"end"},
+  {"type":"Text","value":"\n"}
+]