public List<Token> tokens(final InputStream input) throws TokenizeException {
        try {
            final List<Token> tokens = new ArrayList<Token>();
            final Buffer buffer = new Buffer(input);

            int b;
            while ((b = buffer.read()) != -1) {
                Token token = process(buffer, b);
                if (token != null) {
                    tokens.add(token);
                }
            }

            if (state != TokenizeState.DEFAULT) {
                Token token = process(buffer, b);
                if (token != null) {
                    tokens.add(token);
                }
            }

            if (StringUtils.isNotBlank(value)) {
                throw new TokenizeException("can't parse '" + value + "'");
            }

            return tokens;
        } catch (Exception ex) {
            if (ex instanceof TokenizeException) {
                throw (TokenizeException) ex;
            }
            throw new TokenizeException(ex);
        }
    }