@Override
    public Token read(DocumentReader scanner) throws ScannerReadException {
        String buffer = "";
        if (pattern.matcher("").matches()) {
            //we will not allow for a pattern that matches the empty string
            return null;
        }
        while (scanner.hasMore() && buffer.length() < scanner.getBufferCapacity()) {
            buffer += scanner.next();
            if (buffer.length() > 1 && !pattern.matcher(buffer).matches()) {
                try {
                    scanner.rewind(1);
                } catch (ScannerException e) {
                    throw new ScannerReadException("Could not put back the non-matching part: " + buffer.substring(buffer.length() - 1));
                }
                buffer = buffer.substring(0, buffer.length() - 1);
                if (!pattern.matcher(buffer).matches()) {
                    return null;
                }
                return getTokenFactory().getToken(buffer, scanner.getCursor() - buffer.length());
            }
        }
        if (pattern.matcher(buffer).matches()) {
            return getTokenFactory().getToken(buffer, scanner.getCursor() - buffer.length());
        }
        return null;
    }