Repository: alecthomas/chroma
Pull request #230 (2019-02-02 20:49:58)

Add Turing lexer
Turing source files usually end with .t, but that clashes with Perl so I left it out.


Patch:
diff --git a/README.md b/README.md
index 5c7d256..419b1bf 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ translators for Pygments lexers and styles.
 
 Prefix | Language
 :----: | --------
-A | ABNF, ActionScript, ActionScript 3, Ada, Angular2, ANTLR, ApacheConf, APL, AppleScript, Awk
+A | ABNF, ActionScript, ActionScript 3, Ada, Angular2, ANTLR, ApacheConf, APL, AppleScript, Arduino, Awk
 B | Ballerina, Base Makefile, Bash, Batchfile, BlitzBasic, BNF, Brainfuck
 C | C, C#, C++, Cassandra CQL, CFEngine3, cfstatement/ColdFusion, CMake, COBOL, CSS, Cap'n Proto, Ceylon, ChaiScript, Cheetah, Clojure, CoffeeScript, Common Lisp, Coq, Crystal, Cython
 D | Dart, Diff, Django/Jinja, Docker, DTD
diff --git a/lexers/a/arduino.go b/lexers/a/arduino.go
new file mode 100644
index 0000000..b0cd8c9
--- /dev/null
+++ b/lexers/a/arduino.go
@@ -0,0 +1,110 @@
+package a
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Arduino lexer.
+var Arduino = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "Arduino",
+		Aliases:   []string{"arduino"},
+		Filenames: []string{"*.ino"},
+		MimeTypes: []string{"text/x-arduino"},
+		EnsureNL:  true,
+	},
+	Rules{
+		"statements": {
+			{Words(``, `\b`, `catch`, `const_cast`, `delete`, `dynamic_cast`, `explicit`, `export`, `friend`, `mutable`, `namespace`, `new`, `operator`, `private`, `protected`, `public`, `reinterpret_cast`, `restrict`, `static_cast`, `template`, `this`, `throw`, `throws`, `try`, `typeid`, `typename`, `using`, `virtual`, `constexpr`, `nullptr`, `decltype`, `thread_local`, `alignas`, `alignof`, `static_assert`, `noexcept`, `override`, `final`), Keyword, nil},
+			{`char(16_t|32_t)\b`, KeywordType, nil},
+			{`(class)\b`, ByGroups(Keyword, Text), Push("classname")},
+			{`(R)(")([^\\()\s]{,16})(\()((?:.|\n)*?)(\)\3)(")`, ByGroups(LiteralStringAffix, LiteralString, LiteralStringDelimiter, LiteralStringDelimiter, LiteralString, LiteralStringDelimiter, LiteralString), nil},
+			{`(u8|u|U)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")},
+			{`(L?)(")`, ByGroups(LiteralStringAffix, LiteralString), Push("string")},
+			{`(L?)(')(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])(')`, ByGroups(LiteralStringAffix, LiteralStringChar, LiteralStringChar, LiteralStringChar), nil},
+			{`(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*`, LiteralNumberFloat, nil},
+			{`(\d+\.\d*|\.\d+|\d+[fF])[fF]?`, LiteralNumberFloat, nil},
+			{`0x[0-9a-fA-F]+[LlUu]*`, LiteralNumberHex, nil},
+			{`0[0-7]+[LlUu]*`, LiteralNumberOct, nil},
+			{`\d+[LlUu]*`, LiteralNumberInteger, nil},
+			{`\*/`, Error, nil},
+			{`[~!%^&*+=|?:<>/-]`, Operator, nil},
+			{`[()\[\],.]`, Punctuation, nil},
+			{Words(``, `\b`, `asm`, `auto`, `break`, `case`, `const`, `continue`, `default`, `do`, `else`, `enum`, `extern`, `for`, `goto`, `if`, `register`, `restricted`, `return`, `sizeof`, `static`, `struct`, `switch`, `typedef`, `union`, `volatile`, `while`), Keyword, nil},
+			{`(_Bool|_Complex|_Imaginary|array|atomic_bool|atomic_char|atomic_int|atomic_llong|atomic_long|atomic_schar|atomic_short|atomic_uchar|atomic_uint|atomic_ullong|atomic_ulong|atomic_ushort|auto|bool|boolean|BooleanVariables|Byte|byte|Char|char|char16_t|char32_t|class|complex|Const|const|const_cast|delete|double|dynamic_cast|enum|explicit|extern|Float|float|friend|inline|Int|int|int16_t|int32_t|int64_t|int8_t|Long|long|new|NULL|null|operator|private|PROGMEM|protected|public|register|reinterpret_cast|short|signed|sizeof|Static|static|static_cast|String|struct|typedef|uint16_t|uint32_t|uint64_t|uint8_t|union|unsigned|virtual|Void|void|Volatile|volatile|word)\b`, KeywordType, nil},
+			// Start of: Arduino-specific syntax
+			{`(and|final|If|Loop|loop|not|or|override|setup|Setup|throw|try|xor)\b`, Keyword, nil}, // Addition to keywords already defined by C++
+			{`(ANALOG_MESSAGE|BIN|CHANGE|DEC|DEFAULT|DIGITAL_MESSAGE|EXTERNAL|FALLING|FIRMATA_STRING|HALF_PI|HEX|HIGH|INPUT|INPUT_PULLUP|INTERNAL|INTERNAL1V1|INTERNAL1V1|INTERNAL2V56|INTERNAL2V56|LED_BUILTIN|LED_BUILTIN_RX|LED_BUILTIN_TX|LOW|LSBFIRST|MSBFIRST|OCT|OUTPUT|PI|REPORT_ANALOG|REPORT_DIGITAL|RISING|SET_PIN_MODE|SYSEX_START|SYSTEM_RESET|TWO_PI)\b`, KeywordConstant, nil},
+			{`(boolean|const|byte|word|string|String|array)\b`, NameVariable, nil},
+			{`(Keyboard|KeyboardController|MouseController|SoftwareSerial|EthernetServer|EthernetClient|LiquidCrystal|RobotControl|GSMVoiceCall|EthernetUDP|EsploraTFT|HttpClient|RobotMotor|WiFiClient|GSMScanner|FileSystem|Scheduler|GSMServer|YunClient|YunServer|IPAddress|GSMClient|GSMModem|Keyboard|Ethernet|Console|GSMBand|Esplora|Stepper|Process|WiFiUDP|GSM_SMS|Mailbox|USBHost|Firmata|PImage|Client|Server|GSMPIN|FileIO|Bridge|Serial|EEPROM|Stream|Mouse|Audio|Servo|File|Task|GPRS|WiFi|Wire|TFT|GSM|SPI|SD)\b`, NameClass, nil},
+			{`(abs|Abs|accept|ACos|acos|acosf|addParameter|analogRead|AnalogRead|analogReadResolution|AnalogReadResolution|analogReference|AnalogReference|analogWrite|AnalogWrite|analogWriteResolution|AnalogWriteResolution|answerCall|asin|ASin|asinf|atan|ATan|atan2|ATan2|atan2f|atanf|attach|attached|attachGPRS|attachInterrupt|AttachInterrupt|autoscroll|available|availableForWrite|background|beep|begin|beginPacket|beginSD|beginSMS|beginSpeaker|beginTFT|beginTransmission|beginWrite|bit|Bit|BitClear|bitClear|bitRead|BitRead|bitSet|BitSet|BitWrite|bitWrite|blink|blinkVersion|BSSID|buffer|byte|cbrt|cbrtf|Ceil|ceil|ceilf|changePIN|char|charAt|checkPIN|checkPUK|checkReg|circle|cityNameRead|cityNameWrite|clear|clearScreen|click|close|compareTo|compassRead|concat|config|connect|connected|constrain|Constrain|copysign|copysignf|cos|Cos|cosf|cosh|coshf|countryNameRead|countryNameWrite|createChar|cursor|debugPrint|degrees|Delay|delay|DelayMicroseconds|delayMicroseconds|detach|DetachInterrupt|detachInterrupt|DigitalPinToInterrupt|digitalPinToInterrupt|DigitalRead|digitalRead|DigitalWrite|digitalWrite|disconnect|display|displayLogos|drawBMP|drawCompass|encryptionType|end|endPacket|endSMS|endsWith|endTransmission|endWrite|equals|equalsIgnoreCase|exists|exitValue|Exp|exp|expf|fabs|fabsf|fdim|fdimf|fill|find|findUntil|float|floor|Floor|floorf|flush|fma|fmaf|fmax|fmaxf|fmin|fminf|fmod|fmodf|gatewayIP|get|getAsynchronously|getBand|getButton|getBytes|getCurrentCarrier|getIMEI|getKey|getModifiers|getOemKey|getPINUsed|getResult|getSignalStrength|getSocket|getVoiceCallStatus|getXChange|getYChange|hangCall|height|highByte|HighByte|home|hypot|hypotf|image|indexOf|int|interrupts|IPAddress|IRread|isActionDone|isAlpha|isAlphaNumeric|isAscii|isControl|isDigit|isDirectory|isfinite|isGraph|isHexadecimalDigit|isinf|isListening|isLowerCase|isnan|isPIN|isPressed|isPrintable|isPunct|isSpace|isUpperCase|isValid|isWhitespace|keyboardRead|keyPressed|keyReleased|knobRead|lastIndexOf|ldexp|ldexpf|leftToRight|length|line|lineFollowConfig|listen|listenOnLocalhost|loadImage|localIP|log|Log|log10|log10f|logf|long|lowByte|LowByte|lrint|lrintf|lround|lroundf|macAddress|maintain|map|Map|Max|max|messageAvailable|Micros|micros|millis|Millis|Min|min|mkdir|motorsStop|motorsWrite|mouseDragged|mouseMoved|mousePressed|mouseReleased|move|noAutoscroll|noBlink|noBuffer|noCursor|noDisplay|noFill|noInterrupts|NoInterrupts|noListenOnLocalhost|noStroke|noTone|NoTone|onReceive|onRequest|open|openNextFile|overflow|parseCommand|parseFloat|parseInt|parsePacket|pauseMode|peek|PinMode|pinMode|playFile|playMelody|point|pointTo|position|Pow|pow|powf|prepare|press|print|printFirmwareVersion|println|printVersion|process|processInput|PulseIn|pulseIn|pulseInLong|PulseInLong|put|radians|random|Random|randomSeed|RandomSeed|read|readAccelerometer|readBlue|readButton|readBytes|readBytesUntil|readGreen|readJoystickButton|readJoystickSwitch|readJoystickX|readJoystickY|readLightSensor|readMessage|readMicrophone|readNetworks|readRed|readSlider|readString|readStringUntil|readTemperature|ready|rect|release|releaseAll|remoteIP|remoteNumber|remotePort|remove|replace|requestFrom|retrieveCallingNumber|rewindDirectory|rightToLeft|rmdir|robotNameRead|robotNameWrite|round|roundf|RSSI|run|runAsynchronously|running|runShellCommand|runShellCommandAsynchronously|scanNetworks|scrollDisplayLeft|scrollDisplayRight|seek|sendAnalog|sendDigitalPortPair|sendDigitalPorts|sendString|sendSysex|Serial_Available|Serial_Begin|Serial_End|Serial_Flush|Serial_Peek|Serial_Print|Serial_Println|Serial_Read|serialEvent|setBand|setBitOrder|setCharAt|setClockDivider|setCursor|setDataMode|setDNS|setFirmwareVersion|setMode|setPINUsed|setSpeed|setTextSize|setTimeout|ShiftIn|shiftIn|ShiftOut|shiftOut|shutdown|signbit|sin|Sin|sinf|sinh|sinhf|size|sizeof|Sq|sq|Sqrt|sqrt|sqrtf|SSID|startLoop|startsWith|step|stop|stroke|subnetMask|substring|switchPIN|tan|Tan|tanf|tanh|tanhf|tempoWrite|text|toCharArray|toInt|toLowerCase|tone|Tone|toUpperCase|transfer|trim|trunc|truncf|tuneWrite|turn|updateIR|userNameRead|userNameWrite|voiceCall|waitContinue|width|WiFiServer|word|write|writeBlue|writeGreen|writeJSON|writeMessage|writeMicroseconds|writeRed|writeRGB|yield|Yield)\b`, NameFunction, nil},
+			// End of: Arduino-specific syntax
+			{Words(``, `\b`, `inline`, `_inline`, `__inline`, `naked`, `restrict`, `thread`, `typename`), KeywordReserved, nil},
+			{`(__m(128i|128d|128|64))\b`, KeywordReserved, nil},
+			{Words(`__`, `\b`, `asm`, `int8`, `based`, `except`, `int16`, `stdcall`, `cdecl`, `fastcall`, `int32`, `declspec`, `finally`, `int64`, `try`, `leave`, `wchar_t`, `w64`, `unaligned`, `raise`, `noop`, `identifier`, `forceinline`, `assume`), KeywordReserved, nil},
+			{`(true|false|NULL)\b`, NameBuiltin, nil},
+			{`([a-zA-Z_]\w*)(\s*)(:)(?!:)`, ByGroups(NameLabel, Text, Punctuation), nil},
+			{`[a-zA-Z_]\w*`, Name, nil},
+		},
+		"root": {
+			Include("whitespace"),
+			{`((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;{]*)(\{)`, ByGroups(UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), Push("function")},
+			{`((?:[\w*\s])+?(?:\s|[*]))([a-zA-Z_]\w*)(\s*\([^;]*?\))([^;]*)(;)`, ByGroups(UsingSelf("root"), NameFunction, UsingSelf("root"), UsingSelf("root"), Punctuation), nil},
+			Default(Push("statement")),
+			{Words(`__`, `\b`, `virtual_inheritance`, `uuidof`, `super`, `single_inheritance`, `multiple_inheritance`, `interface`, `event`), KeywordReserved, nil},
+			{`__(offload|blockingoffload|outer)\b`, KeywordPseudo, nil},
+		},
+		"classname": {
+			{`[a-zA-Z_]\w*`, NameClass, Pop(1)},
+			{`\s*(?=>)`, Text, Pop(1)},
+		},
+		"whitespace": {
+			{`^#if\s+0`, CommentPreproc, Push("if0")},
+			{`^#`, CommentPreproc, Push("macro")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#if\s+0)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("if0")},
+			{`^(\s*(?:/[*].*?[*]/\s*)?)(#)`, ByGroups(UsingSelf("root"), CommentPreproc), Push("macro")},
+			{`\n`, Text, nil},
+			{`\s+`, Text, nil},
+			{`\\\n`, Text, nil},
+			{`//(\n|[\w\W]*?[^\\]\n)`, CommentSingle, nil},
+			{`/(\\\n)?[*][\w\W]*?[*](\\\n)?/`, CommentMultiline, nil},
+			{`/(\\\n)?[*][\w\W]*`, CommentMultiline, nil},
+		},
+		"statement": {
+			Include("whitespace"),
+			Include("statements"),
+			{`[{}]`, Punctuation, nil},
+			{`;`, Punctuation, Pop(1)},
+		},
+		"function": {
+			Include("whitespace"),
+			Include("statements"),
+			{`;`, Punctuation, nil},
+			{`\{`, Punctuation, Push()},
+			{`\}`, Punctuation, Pop(1)},
+		},
+		"string": {
+			{`"`, LiteralString, Pop(1)},
+			{`\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})`, LiteralStringEscape, nil},
+			{`[^\\"\n]+`, LiteralString, nil},
+			{`\\\n`, LiteralString, nil},
+			{`\\`, LiteralString, nil},
+		},
+		"macro": {
+			{`(include)(\s*(?:/[*].*?[*]/\s*)?)([^\n]+)`, ByGroups(CommentPreproc, Text, CommentPreprocFile), nil},
+			{`[^/\n]+`, CommentPreproc, nil},
+			{`/[*](.|\n)*?[*]/`, CommentMultiline, nil},
+			{`//.*?\n`, CommentSingle, Pop(1)},
+			{`/`, CommentPreproc, nil},
+			{`(?<=\\)\n`, CommentPreproc, nil},
+			{`\n`, CommentPreproc, Pop(1)},
+		},
+		"if0": {
+			{`^\s*#if.*?(?<!\\)\n`, CommentPreproc, Push()},
+			{`^\s*#el(?:se|if).*\n`, CommentPreproc, Pop(1)},
+			{`^\s*#endif.*?(?<!\\)\n`, CommentPreproc, Pop(1)},
+			{`.*?\n`, Comment, nil},
+		},
+	},
+))
diff --git a/lexers/j/javascript.go b/lexers/j/javascript.go
index 8301e41..6291e33 100644
--- a/lexers/j/javascript.go
+++ b/lexers/j/javascript.go
@@ -23,7 +23,6 @@ var JavascriptRules = Rules{
 	},
 	"root": {
 		{`\A#! ?/.*?\n`, CommentHashbang, nil},
-		{`^(?=\s|/|<!--)`, Text, Push("slashstartsregex")},
 		Include("commentsandwhitespace"),
 		{`(\.\d+|[0-9]+\.[0-9]*)([eE][-+]?[0-9]+)?`, LiteralNumberFloat, nil},
 		{`0[bB][01]+`, LiteralNumberBin, nil},
@@ -32,6 +31,7 @@ var JavascriptRules = Rules{
 		{`[0-9]+`, LiteralNumberInteger, nil},
 		{`\.\.\.|=>`, Punctuation, nil},
 		{`\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|(<<|>>>?|==?|!=?|[-<>+*%&|^/])=?`, Operator, Push("slashstartsregex")},
+		{`^(?=\s|/|<!--)`, Text, Push("slashstartsregex")},
 		{`[{(\[;,]`, Punctuation, Push("slashstartsregex")},
 		{`[})\].]`, Punctuation, nil},
 		{`(for|in|while|do|break|return|continue|switch|case|default|if|else|throw|try|catch|finally|new|delete|typeof|instanceof|void|yield|this|of|class|const|debugger|export|extends|import|super)\b`, Keyword, Push("slashstartsregex")},
diff --git a/lexers/r/rust.go b/lexers/r/rust.go
index 3d05cd5..5e18b8f 100644
--- a/lexers/r/rust.go
+++ b/lexers/r/rust.go
@@ -28,6 +28,8 @@ var Rust = internal.Register(MustNewLexer(
 			{`/\*\*(\n|[^/*])`, LiteralStringDoc, Push("doccomment")},
 			{`/\*!`, LiteralStringDoc, Push("doccomment")},
 			{`/\*`, CommentMultiline, Push("comment")},
+			{`r#*"(?:\\.|[^\\\r\n;])*"#*`, LiteralString, nil},
+			{`"(?:\\.|[^\\\r\n"])*"`, LiteralString, nil},
 			{`\$([a-zA-Z_]\w*|\(,?|\),?|,?)`, CommentPreproc, nil},
 			{Words(``, `\b`, `as`, `box`, `const`, `crate`, `else`, `extern`, `for`, `if`, `impl`, `in`, `loop`, `match`, `move`, `mut`, `pub`, `ref`, `return`, `static`, `super`, `trait`, `unsafe`, `use`, `where`, `while`), Keyword, nil},
 			{Words(``, `\b`, `abstract`, `alignof`, `become`, `do`, `final`, `macro`, `offsetof`, `override`, `priv`, `proc`, `pure`, `sizeof`, `typeof`, `unsized`, `virtual`, `yield`), KeywordReserved, nil},
@@ -51,7 +53,6 @@ var Rust = internal.Register(MustNewLexer(
 			{`[0-9][0-9_]*(\.[0-9_]+[eE][+\-]?[0-9_]+|\.[0-9_]*(?!\.)|[eE][+\-]?[0-9_]+)`, LiteralNumberFloat, Push("number_lit")},
 			{`[0-9][0-9_]*`, LiteralNumberInteger, Push("number_lit")},
 			{`b"`, LiteralString, Push("bytestring")},
-			{`"`, LiteralString, Push("string")},
 			{`b?r(#*)".*?"\1`, LiteralString, nil},
 			{`'static`, NameBuiltin, nil},
 			{`'[a-zA-Z_]\w*`, NameAttribute, nil},
diff --git a/lexers/t/turing.go b/lexers/t/turing.go
new file mode 100644
index 0000000..ae5671b
--- /dev/null
+++ b/lexers/t/turing.go
@@ -0,0 +1,43 @@
+package t
+
+import (
+	. "github.com/alecthomas/chroma" // nolint
+	"github.com/alecthomas/chroma/lexers/internal"
+)
+
+// Turing lexer.
+var Turing = internal.Register(MustNewLexer(
+	&Config{
+		Name:      "Turing",
+		Aliases:   []string{"turing"},
+		Filenames: []string{"*.turing", "*.tu"},
+		MimeTypes: []string{"text/x-turing"},
+	},
+	Rules{
+		"root": {
+			{`\n`, Text, nil},
+			{`\s+`, Text, nil},
+			{`\\\n`, Text, nil},
+			{`%(.*?)\n`, CommentSingle, nil},
+			{`/(\\\n)?[*](.|\n)*?[*](\\\n)?/`, CommentMultiline, nil},
+			{`(var|fcn|function|proc|procedure|process|class|end|record|type|begin|case|loop|for|const|union|monitor|module|handler)\b`, KeywordDeclaration, nil},
+			{`(all|asm|assert|bind|bits|body|break|by|cheat|checked|close|condition|decreasing|def|deferred|else|elsif|exit|export|external|flexible|fork|forward|free|get|if|implement|import|include|inherit|init|invariant|label|new|objectclass|of|opaque|open|packed|pause|pervasive|post|pre|priority|put|quit|read|register|result|seek|self|set|signal|skip|tag|tell|then|timeout|to|unchecked|unqualified|wait|when|write)\b`, Keyword, nil},
+			{`(true|false)\b`, KeywordConstant, nil},
+			{Words(``, `\b`, `addressint`, `array`, `boolean`, `char`, `int`, `int1`, `int2`, `int4`, `int8`, `nat`, `nat1`, `nat2`, `nat4`, `nat8`, `pointer`, `real`, `real4`, `real8`, `string`, `enum`), KeywordType, nil},
+			{`\d+i`, LiteralNumber, nil},
+			{`\d+\.\d*([Ee][-+]\d+)?i`, LiteralNumber, nil},
+			{`\.\d+([Ee][-+]\d+)?i`, LiteralNumber, nil},
+			{`\d+[Ee][-+]\d+i`, LiteralNumber, nil},
+			{`\d+(\.\d+[eE][+\-]?\d+|\.\d*|[eE][+\-]?\d+)`, LiteralNumberFloat, nil},
+			{`\.\d+([eE][+\-]?\d+)?`, LiteralNumberFloat, nil},
+			{`0[0-7]+`, LiteralNumberOct, nil},
+			{`0[xX][0-9a-fA-F]+`, LiteralNumberHex, nil},
+			{`(0|[1-9][0-9]*)`, LiteralNumberInteger, nil},
+			{`(div|mod|rem|\*\*|=|<|>|>=|<=|not=|not|and|or|xor|=>|in|shl|shr|->|~|~=|~in|&|:=|\.\.|[\^+\-*/&#])`, Operator, nil},
+			{`'(\\['"\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|[^\\])'`, LiteralStringChar, nil},
+			{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
+			{`[()\[\]{}.,:]`, Punctuation, nil},
+			{`[^\W\d]\w*`, NameOther, nil},
+		},
+	},
+))


Test patch:
diff --git a/lexers/testdata/arduino.actual b/lexers/testdata/arduino.actual
new file mode 100644
index 0000000..01a0bcc
--- /dev/null
+++ b/lexers/testdata/arduino.actual
@@ -0,0 +1,87 @@
+#include <Servo.h>
+#include <Adafruit_NeoPixel.h>
+
+Servo elbow_servo; 
+Servo wrist_servo;
+const int potpin = 0;  // analog pin used to connect the potentiometer
+int val;    // variable to read the value from the analog pin
+
+//// button cycles through controlling each color
+//// in the neopixel.  state 0=stop, 1=red, 2=blue, 3=green, 4=all/white
+const int button_pin = 4;
+int buttonState;             // the current reading from the input pin
+int lastButtonState = HIGH;   // the previous reading from the input pin
+// the following variables are unsigned longs because the time, measured in
+// milliseconds, will quickly become a bigger number than can be stored in an int.
+unsigned long lastDebounceTime = 0;  // the last time the output pin was toggled
+unsigned long debounceDelay = 50;    // the debounce time; increase if the output flickers
+int led_control_state = 0;
+
+const int pixel_pin = 6;
+const int num_leds = 2;
+Adafruit_NeoPixel pixel = Adafruit_NeoPixel(num_leds, pixel_pin, NEO_GRB + NEO_KHZ800);
+int red = 0;
+int green = 0;
+int blue = 0;
+
+void setup() {
+  Serial.begin(9600);
+  elbow_servo.attach(9);  // attaches the servo on pin 9 to the servo object
+  pinMode(button_pin, INPUT_PULLUP); // create a button for the neopixels
+  pixel.begin();
+}
+
+void loop() {
+  
+  //// Read signal from variable resistor or potentiometer
+  //
+  val = analogRead(potpin);            // reads the value of the potentiometer (value between 0 and 1023)
+  int v = map(val, 0, 1023, 0, 180);     // scale it to use it with the servo (value between 0 and 180)
+  elbow_servo.write(v);  
+  delay(15);                           // waits for the servo to get there
+
+
+  //// Toggle between states using a button
+  ////
+  int b = digitalRead(button_pin); // read button
+  // "debounce" the button signal
+  if (b != lastButtonState) lastDebounceTime = millis();
+  if ((millis() - lastDebounceTime) > debounceDelay) {
+    if (b != buttonState) {
+      buttonState = b;
+      if (buttonState == LOW) {
+        led_control_state++;
+        if (led_control_state > 3) led_control_state = 0;
+      }
+    }
+  }
+  lastButtonState = b;
+
+  switch(led_control_state) {
+    case 0:
+      break; 
+    case 1:
+      red = 255;
+      blue = 255;
+      green = 0;
+      break;
+    case 2:
+      red = 0;
+      blue = 255;
+      green = 255;
+      break;
+    case 3:
+      red = 255;
+      blue = 0;
+      green = 255;
+      break;
+  }
+
+  Serial.println(led_control_state);
+  // set color ofpixels
+  for(int i=0;i<num_leds;i++){
+    pixel.setPixelColor(i,red,green,blue);
+    pixel.show(); 
+    delay(15); 
+  }
+}
\ No newline at end of file
diff --git a/lexers/testdata/arduino.expected b/lexers/testdata/arduino.expected
new file mode 100644
index 0000000..c59ed2e
--- /dev/null
+++ b/lexers/testdata/arduino.expected
@@ -0,0 +1,546 @@
+[
+  {"type":"CommentPreproc","value":"#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\u003cServo.h\u003e"},
+  {"type":"CommentPreproc","value":"\n#include"},
+  {"type":"Text","value":" "},
+  {"type":"CommentPreprocFile","value":"\u003cAdafruit_NeoPixel.h\u003e"},
+  {"type":"CommentPreproc","value":"\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"NameClass","value":"Servo"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"elbow_servo"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" \n"},
+  {"type":"NameClass","value":"Servo"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"wrist_servo"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"const"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"potpin"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSingle","value":"// analog pin used to connect the potentiometer\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"val"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"    "},
+  {"type":"CommentSingle","value":"// variable to read the value from the analog pin\n"},
+  {"type":"Text","value":"\n"},
+  {"type":"CommentSingle","value":"//// button cycles through controlling each color\n//// in the neopixel.  state 0=stop, 1=red, 2=blue, 3=green, 4=all/white\n"},
+  {"type":"Keyword","value":"const"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"button_pin"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"4"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"buttonState"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"             "},
+  {"type":"CommentSingle","value":"// the current reading from the input pin\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"lastButtonState"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"KeywordConstant","value":"HIGH"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"   "},
+  {"type":"CommentSingle","value":"// the previous reading from the input pin\n// the following variables are unsigned longs because the time, measured in\n// milliseconds, will quickly become a bigger number than can be stored in an int.\n"},
+  {"type":"KeywordType","value":"unsigned"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"long"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"lastDebounceTime"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSingle","value":"// the last time the output pin was toggled\n"},
+  {"type":"KeywordType","value":"unsigned"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"long"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"debounceDelay"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"50"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"    "},
+  {"type":"CommentSingle","value":"// the debounce time; increase if the output flickers\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"led_control_state"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"const"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"pixel_pin"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"6"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Keyword","value":"const"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"num_leds"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Name","value":"Adafruit_NeoPixel"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"pixel"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Adafruit_NeoPixel"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"num_leds"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"pixel_pin"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"NEO_GRB"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"NEO_KHZ800"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"red"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"green"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"blue"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"KeywordType","value":"void"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"setup"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n  "},
+  {"type":"NameClass","value":"Serial"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameFunction","value":"begin"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"9600"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Name","value":"elbow_servo"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameFunction","value":"attach"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"9"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSingle","value":"// attaches the servo on pin 9 to the servo object\n"},
+  {"type":"Text","value":"  "},
+  {"type":"NameFunction","value":"pinMode"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"button_pin"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"KeywordConstant","value":"INPUT_PULLUP"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":" "},
+  {"type":"CommentSingle","value":"// create a button for the neopixels\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Name","value":"pixel"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameFunction","value":"begin"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"KeywordType","value":"void"},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"loop"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n  \n  "},
+  {"type":"CommentSingle","value":"//// Read signal from variable resistor or potentiometer\n"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSingle","value":"//\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Name","value":"val"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"analogRead"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"potpin"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"            "},
+  {"type":"CommentSingle","value":"// reads the value of the potentiometer (value between 0 and 1023)\n"},
+  {"type":"Text","value":"  "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"v"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"map"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"val"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1023"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"180"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"     "},
+  {"type":"CommentSingle","value":"// scale it to use it with the servo (value between 0 and 180)\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Name","value":"elbow_servo"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameFunction","value":"write"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"v"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"  \n  "},
+  {"type":"NameFunction","value":"delay"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"15"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"                           "},
+  {"type":"CommentSingle","value":"// waits for the servo to get there\n"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"CommentSingle","value":"//// Toggle between states using a button\n"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSingle","value":"////\n"},
+  {"type":"Text","value":"  "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"b"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"digitalRead"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"button_pin"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":" "},
+  {"type":"CommentSingle","value":"// read button\n"},
+  {"type":"Text","value":"  "},
+  {"type":"CommentSingle","value":"// \"debounce\" the button signal\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Keyword","value":"if"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"b"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"!="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"lastButtonState"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"lastDebounceTime"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"NameFunction","value":"millis"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Keyword","value":"if"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"(("},
+  {"type":"NameFunction","value":"millis"},
+  {"type":"Punctuation","value":"()"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"-"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"lastDebounceTime"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"debounceDelay"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"if"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"b"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"!="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"buttonState"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"buttonState"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"b"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"if"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"buttonState"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"=="},
+  {"type":"Text","value":" "},
+  {"type":"KeywordConstant","value":"LOW"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n        "},
+  {"type":"Name","value":"led_control_state"},
+  {"type":"Operator","value":"++"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n        "},
+  {"type":"Keyword","value":"if"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"led_control_state"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"\u003e"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"3"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"led_control_state"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Name","value":"lastButtonState"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"b"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"Keyword","value":"switch"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"led_control_state"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"case"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"break"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":" \n    "},
+  {"type":"Keyword","value":"case"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"1"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"red"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"255"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"blue"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"255"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"green"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"break"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"case"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"red"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"blue"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"255"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"green"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"255"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"break"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"case"},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"3"},
+  {"type":"Operator","value":":"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"red"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"255"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"blue"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Name","value":"green"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"255"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n      "},
+  {"type":"Keyword","value":"break"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Text","value":"\n  "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n\n  "},
+  {"type":"NameClass","value":"Serial"},
+  {"type":"Punctuation","value":"."},
+  {"type":"NameFunction","value":"println"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"led_control_state"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n  "},
+  {"type":"CommentSingle","value":"// set color ofpixels\n"},
+  {"type":"Text","value":"  "},
+  {"type":"Keyword","value":"for"},
+  {"type":"Punctuation","value":"("},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"i"},
+  {"type":"Operator","value":"="},
+  {"type":"LiteralNumberInteger","value":"0"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Name","value":"i"},
+  {"type":"Operator","value":"\u003c"},
+  {"type":"Name","value":"num_leds"},
+  {"type":"Punctuation","value":";"},
+  {"type":"Name","value":"i"},
+  {"type":"Operator","value":"++"},
+  {"type":"Punctuation","value":"){"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"pixel"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"setPixelColor"},
+  {"type":"Punctuation","value":"("},
+  {"type":"Name","value":"i"},
+  {"type":"Punctuation","value":","},
+  {"type":"Name","value":"red"},
+  {"type":"Punctuation","value":","},
+  {"type":"Name","value":"green"},
+  {"type":"Punctuation","value":","},
+  {"type":"Name","value":"blue"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Name","value":"pixel"},
+  {"type":"Punctuation","value":"."},
+  {"type":"Name","value":"show"},
+  {"type":"Punctuation","value":"();"},
+  {"type":"Text","value":" \n    "},
+  {"type":"NameFunction","value":"delay"},
+  {"type":"Punctuation","value":"("},
+  {"type":"LiteralNumberInteger","value":"15"},
+  {"type":"Punctuation","value":");"},
+  {"type":"Text","value":" \n  "},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"},
+  {"type":"Punctuation","value":"}"},
+  {"type":"Text","value":"\n"}
+]
diff --git a/lexers/testdata/turing.actual b/lexers/testdata/turing.actual
new file mode 100644
index 0000000..1280240
--- /dev/null
+++ b/lexers/testdata/turing.actual
@@ -0,0 +1,7 @@
+var x : int := 10
+
+fcn testFunction(a, b, c)
+    result (a + b) * c
+end testFunction
+
+put testFunction(x, 2 * x, 3)
\ No newline at end of file
diff --git a/lexers/testdata/turing.expected b/lexers/testdata/turing.expected
new file mode 100644
index 0000000..c96fc2f
--- /dev/null
+++ b/lexers/testdata/turing.expected
@@ -0,0 +1,61 @@
+[
+  {"type":"KeywordDeclaration","value":"var"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"x"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":":"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":":="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"10"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"KeywordDeclaration","value":"fcn"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"testFunction"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameOther","value":"a"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"b"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"c"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"Keyword","value":"result"},
+  {"type":"Text","value":" "},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameOther","value":"a"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"+"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"b"},
+  {"type":"Punctuation","value":")"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"c"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordDeclaration","value":"end"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"testFunction"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"Keyword","value":"put"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"testFunction"},
+  {"type":"Punctuation","value":"("},
+  {"type":"NameOther","value":"x"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"2"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"*"},
+  {"type":"Text","value":" "},
+  {"type":"NameOther","value":"x"},
+  {"type":"Punctuation","value":","},
+  {"type":"Text","value":" "},
+  {"type":"LiteralNumberInteger","value":"3"},
+  {"type":"Punctuation","value":")"}
+]