static void process(char const* filename, bool include_ignorable,
                    size_t max_len, bool old_ei)
{
    PointerHolder<InputSource> is;

    // Tokenize file, skipping streams
    FileInputSource* fis = new FileInputSource();
    fis->setFilename(filename);
    is = fis;
    dump_tokens(is, "FILE", max_len, include_ignorable, true, false, false);

    // Tokenize content streams, skipping inline images
    QPDF qpdf;
    qpdf.processFile(filename);
    std::vector<QPDFPageObjectHelper> pages =
        QPDFPageDocumentHelper(qpdf).getAllPages();
    int pageno = 0;
    for (std::vector<QPDFPageObjectHelper>::iterator iter = pages.begin();
         iter != pages.end(); ++iter)
    {
        ++pageno;
        Pl_Buffer plb("buffer");
        (*iter).pipePageContents(&plb);
        PointerHolder<Buffer> content_data = plb.getBuffer();
        BufferInputSource* bis = new BufferInputSource(
            "content data", content_data.getPointer());
        is = bis;
        dump_tokens(is, "PAGE " + QUtil::int_to_string(pageno),
                    max_len, include_ignorable, false, true, old_ei);
    }

    // Tokenize object streams
    std::vector<QPDFObjectHandle> all = qpdf.getAllObjects();
    for (std::vector<QPDFObjectHandle>::iterator iter = all.begin();
         iter != all.end(); ++iter)
    {
        if ((*iter).isStream() &&
            (*iter).getDict().getKey("/Type").isName() &&
            (*iter).getDict().getKey("/Type").getName() == "/ObjStm")
        {
            PointerHolder<Buffer> b =
                (*iter).getStreamData(qpdf_dl_specialized);
            BufferInputSource* bis = new BufferInputSource(
                "object stream data", b.getPointer());
            is = bis;
            dump_tokens(is, "OBJECT STREAM " +
                        QUtil::int_to_string((*iter).getObjectID()),
                        max_len, include_ignorable, false, false, false);
        }
    }
}
static void process(char const* filename, bool include_ignorable,
                    size_t max_len, bool old_ei)
{
    PointerHolder<InputSource> is;

     
    FileInputSource* fis = new FileInputSource();
    fis->setFilename(filename);
    is = fis;
    dump_tokens(is, "FILE", max_len, include_ignorable, true, false, false);

     
    QPDF qpdf;
    qpdf.processFile(filename);
    std::vector<QPDFPageObjectHelper> pages =
        QPDFPageDocumentHelper(qpdf).getAllPages();
    int pageno = 0;
    for (std::vector<QPDFPageObjectHelper>::iterator iter = pages.begin();
         iter != pages.end(); ++iter)
    {
        ++pageno;
        Pl_Buffer plb("buffer");
        (*iter).pipePageContents(&plb);
        PointerHolder<Buffer> content_data = plb.getBuffer();
        BufferInputSource* bis = new BufferInputSource(
            "content data", content_data.getPointer());
        is = bis;
        dump_tokens(is, "PAGE " + QUtil::int_to_string(pageno),
                    max_len, include_ignorable, false, true, old_ei);
    }

     
    std::vector<QPDFObjectHandle> all = qpdf.getAllObjects();
    for (std::vector<QPDFObjectHandle>::iterator iter = all.begin();
         iter != all.end(); ++iter)
    {
        if ((*iter).isStream() &&
            (*iter).getDict().getKey("/Type").isName() &&
            (*iter).getDict().getKey("/Type").getName() == "/ObjStm")
        {
            PointerHolder<Buffer> b =
                (*iter).getStreamData(qpdf_dl_specialized);
            BufferInputSource* bis = new BufferInputSource(
                "object stream data", b.getPointer());
            is = bis;
            dump_tokens(is, "OBJECT STREAM " +
                        QUtil::int_to_string((*iter).getObjectID()),
                        max_len, include_ignorable, false, false, false);
        }
    }
}