mohayonao/wamml — pull request #1 by mohayonao (2014-08-12 07:44:05+00:00)
Title: add tokenizer

```javascript
var mml = "t120 l{{len}} cdef @hello(10, 20) gab";

var tokens = Wamml.tokenize(mml);
/*
[
{ type: "char" , value: "t" },
{ type: "number" , value: "120" },
{ type: "char" , value: "l" },
{ type: "punctuator", value: "{{" },
{ type: "char" , value: "l" },
{ type: "char" , value: "e" },
{ type: "char" , value: "n" },
{ type: "punctuator", value: "}}" },
{ type: "char" , value: "c" },
{ type: "char" , value: "d" },
{ type: "char" , value: "e" },
{ type: "char" , value: "f" },
{ type: "punctuator", value: "@" },
{ type: "char" , value: "h" },
{ type: "char" , value: "e" },
{ type: "char" , value: "l" },
{ type: "char" , value: "l" },
{ type: "char" , value: "o" },
{ type: "punctuator", value: "(" },
{ type: "number" , value: "10" },
{ type: "punctuator", value: "," },
{ type: "number" , value: "20" },
{ type: "punctuator", value: ")" },
{ type: "char" , value: "g" },
{ type: "char" , value: "a" },
{ type: "char" , value: "b" },
{ type: "punctuator", value: "<" },
{ type: "char" , value: "c" },
{ type: "punctuator", value: ">" }
]
*/