dump_tokens(PointerHolder<InputSource> is, std::string const& label,
            size_t max_len, bool include_ignorable,
            bool skip_streams, bool skip_inline_images, bool old_ei)
{
    Finder f1(is, "endstream");
    std::cout << "--- BEGIN " << label << " ---" << std::endl;
    bool done = false;
    QPDFTokenizer tokenizer;
    tokenizer.allowEOF();
    if (include_ignorable)
    {
        tokenizer.includeIgnorable();
    }
    qpdf_offset_t inline_image_offset = 0;
    while (! done)
    {
        QPDFTokenizer::Token token =
            tokenizer.readToken(is, "test", true,
                                inline_image_offset ? 0 : max_len);
        if (inline_image_offset && (token.getType() == QPDFTokenizer::tt_bad))
        {
            std::cout << "EI not found; resuming normal scanning" << std::endl;
            is->seek(inline_image_offset, SEEK_SET);
            inline_image_offset = 0;
            continue;
        }
        inline_image_offset = 0;

        qpdf_offset_t offset = is->getLastOffset();
        std::cout << offset << ": "
                  << tokenTypeName(token.getType());
        if (token.getType() != QPDFTokenizer::tt_eof)
        {
            std::cout << ": "
                      << sanitize(token.getValue());
            if (token.getValue() != token.getRawValue())
            {
                std::cout << " (raw: " << sanitize(token.getRawValue()) << ")";
            }
        }
        if (token.getType() == QPDFTokenizer::tt_bad)
        {
            std::cout << " (" << token.getErrorMessage() << ")";
        }
        std::cout << std::endl;
        if (skip_streams &&
            (token == QPDFTokenizer::Token(QPDFTokenizer::tt_word, "stream")))
        {
            try_skipping(tokenizer, is, max_len, "endstream", f1);
        }
        else if (skip_inline_images &&
                 (token == QPDFTokenizer::Token(QPDFTokenizer::tt_word, "ID")))
        {
            char ch;
            is->read(&ch, 1);
            if (old_ei)
            {
                tokenizer.expectInlineImage();
            }
            else
            {
                tokenizer.expectInlineImage(is);
            }
            inline_image_offset = is->tell();
        }
        else if (token.getType() == QPDFTokenizer::tt_eof)
        {
            done = true;
        }
    }
    std::cout << "--- END " << label << " ---" << std::endl;
}
dump_tokens(PointerHolder<InputSource> is, std::string const& label,
            size_t max_len, bool include_ignorable,
            bool skip_streams, bool skip_inline_images, bool old_ei)
{
    Finder f1(is, "endstream");
    std::cout << "--- BEGIN " << label << " ---" << std::endl;
    bool done = false;
    QPDFTokenizer tokenizer;
    tokenizer.allowEOF();
    if (include_ignorable)
    {
        tokenizer.includeIgnorable();
    }
    qpdf_offset_t inline_image_offset = 0;
    while (! done)
    {
        QPDFTokenizer::Token token =
            tokenizer.readToken(is, "test", true,
                                inline_image_offset ? 0 : max_len);
        if (inline_image_offset && (token.getType() == QPDFTokenizer::tt_bad))
        {
            std::cout << "EI not found; resuming normal scanning" << std::endl;
            is->seek(inline_image_offset, SEEK_SET);
            inline_image_offset = 0;
            continue;
        }
        inline_image_offset = 0;

        qpdf_offset_t offset = is->getLastOffset();
        std::cout << offset << ": "
                  << tokenTypeName(token.getType());
        if (token.getType() != QPDFTokenizer::tt_eof)
        {
            std::cout << ": "
                      << sanitize(token.getValue());
            if (token.getValue() != token.getRawValue())
            {
                std::cout << " (raw: " << sanitize(token.getRawValue()) << ")";
            }
        }
        if (token.getType() == QPDFTokenizer::tt_bad)
        {
            std::cout << " (" << token.getErrorMessage() << ")";
        }
        std::cout << std::endl;
        if (skip_streams &&
            (token == QPDFTokenizer::Token(QPDFTokenizer::tt_word, "stream")))
        {
            try_skipping(tokenizer, is, max_len, "endstream", f1);
        }
        else if (skip_inline_images &&
                 (token == QPDFTokenizer::Token(QPDFTokenizer::tt_word, "ID")))
        {
            char ch;
            is->read(&ch, 1);
            if (old_ei)
            {
                tokenizer.expectInlineImage();
            }
            else
            {
                tokenizer.expectInlineImage(is);
            }
            inline_image_offset = is->tell();
        }
        else if (token.getType() == QPDFTokenizer::tt_eof)
        {
            done = true;
        }
    }
    std::cout << "--- END " << label << " ---" << std::endl;
}