Stream<Token> tokenize(String input) {
    List<Token> tokens = new ArrayList<Token>();
    int tokenPos = 0;
    while (!input.isEmpty()) {
      boolean matched = false;
      for (Token.Type tokenType : Token.Type.values()) {
        Matcher matcher = tokenType.pattern.matcher(input);
        if (matcher.find()) {
          matched = true;
          input = matcher.replaceFirst("");
          if (tokenType != Token.Type.WHITESPACE) {
            tokens.add(new Token(tokenType, matcher.group(), tokenPos));
          }
          tokenPos += matcher.end();
          break;
        }
      }
      if (!matched) {
        throw new LexerException(input);
      }
    }
    tokens.add(new Token(Token.Type.EOL, null, tokenPos));
    return new Stream<Token>(tokens.toArray(new Token[tokens.size()]));
  }