QPDFTokenizer::readToken(PointerHolder<InputSource> input,
                         std::string const& context,
                         bool allow_bad,
                         size_t max_len)
{
    qpdf_offset_t offset = input->tell();
    Token token;
    bool unread_char;
    char char_to_unread;
    bool presented_eof = false;
    while (! getToken(token, unread_char, char_to_unread))
    {
	char ch;
	if (input->read(&ch, 1) == 0)
	{
            if (! presented_eof)
            {
                presentEOF();
                presented_eof = true;
                if ((this->m->type == tt_eof) && (! this->m->allow_eof))
                {
                    // Nothing in the qpdf library calls readToken
                    // without allowEOF anymore, so this case is not
                    // exercised.
                    this->m->type = tt_bad;
                    this->m->error_message = "unexpected EOF";
                    offset = input->getLastOffset();
                }
            }
            else
            {
                throw std::logic_error(
                    "getToken returned false after presenting EOF");
            }
	}
	else
	{
	    presentCharacter(ch);
	    if (betweenTokens() && (input->getLastOffset() == offset))
	    {
		++offset;
	    }
            if (max_len && (this->m->raw_val.length() >= max_len) &&
                (this->m->state != st_token_ready))
            {
                // terminate this token now
                QTC::TC("qpdf", "QPDFTokenizer block long token");
                this->m->type = tt_bad;
                this->m->state = st_token_ready;
                this->m->error_message =
                    "exceeded allowable length while reading token";
            }
	}
    }

    if (unread_char)
    {
	input->unreadCh(char_to_unread);
    }

    if (token.getType() != tt_eof)
    {
        input->setLastOffset(offset);
    }

    if (token.getType() == tt_bad)
    {
        if (allow_bad)
        {
            QTC::TC("qpdf", "QPDFTokenizer allowing bad token");
        }
        else
        {
            throw QPDFExc(qpdf_e_damaged_pdf, input->getName(),
                          context, offset, token.getErrorMessage());
        }
    }

    return token;
}
QPDFTokenizer::readToken(PointerHolder<InputSource> input,
                         std::string const& context,
                         bool allow_bad,
                         size_t max_len)
{
    qpdf_offset_t offset = input->tell();
    Token token;
    bool unread_char;
    char char_to_unread;
    bool presented_eof = false;
    while (! getToken(token, unread_char, char_to_unread))
    {
	char ch;
	if (input->read(&ch, 1) == 0)
	{
            if (! presented_eof)
            {
                presentEOF();
                presented_eof = true;
                if ((this->m->type == tt_eof) && (! this->m->allow_eof))
                {
                     
                     
                     
                    this->m->type = tt_bad;
                    this->m->error_message = "unexpected EOF";
                    offset = input->getLastOffset();
                }
            }
            else
            {
                throw std::logic_error(
                    "getToken returned false after presenting EOF");
            }
	}
	else
	{
	    presentCharacter(ch);
	    if (betweenTokens() && (input->getLastOffset() == offset))
	    {
		++offset;
	    }
            if (max_len && (this->m->raw_val.length() >= max_len) &&
                (this->m->state != st_token_ready))
            {
                 
                QTC::TC("qpdf", "QPDFTokenizer block long token");
                this->m->type = tt_bad;
                this->m->state = st_token_ready;
                this->m->error_message =
                    "exceeded allowable length while reading token";
            }
	}
    }

    if (unread_char)
    {
	input->unreadCh(char_to_unread);
    }

    if (token.getType() != tt_eof)
    {
        input->setLastOffset(offset);
    }

    if (token.getType() == tt_bad)
    {
        if (allow_bad)
        {
            QTC::TC("qpdf", "QPDFTokenizer allowing bad token");
        }
        else
        {
            throw QPDFExc(qpdf_e_damaged_pdf, input->getName(),
                          context, offset, token.getErrorMessage());
        }
    }

    return token;
}