Repository: nomic-ai/gpt4all
File: gpt4all-chat/src/database.cpp (C++)

Code before:
    const QString &file, const QString &title, const QString &author, const QString &subject, const QString &keywords,
    int page, int maxChunks)
{
    // TODO: implement line_from/line_to
    constexpr int line_from = -1;
    constexpr int line_to = -1;

    int chunks = 0;
    int addedWords = 0;

    QString buffer;
    size_t bufferSz = m_chunkSize * 2;
    buffer.reserve(bufferSz);

    QList<QString> chunkOutputs;
    while (!stream.atEnd() || !buffer.isEmpty()) {
        if (!stream.atEnd())
            buffer += stream.read(bufferSz - buffer.size());

        if (stream.status()) {
            Q_UNREACHABLE();
            qWarning() << "ERROR: Could not chunk stream" << stream.status();
            return -1;
        }

        QString chunk = textSplitter(buffer, m_chunkSize, false /*exceed*/);

        // If the stream has an ASCII EOT (end of transmission), then we cannot read any more data
        // even by stream.readAll() yet the stream will not be at an end. We detect this and just
        // stop the loop with an error return
        if (chunk.isEmpty() && !stream.atEnd())
            return -1;

        // Drop whitespace-only chunks on the floor
        if (chunk.trimmed().isEmpty())
            continue;

        chunkOutputs << chunk;

        // Count the words
        int words = 0;
        QTextStream wordsStream(&chunk);
        for (;;) {
            QString word;
            wordsStream >> word;
            if (!word.isEmpty())
                ++words;
            if (wordsStream.status())
                break;
        }
        addedWords += words;

Review comment (refactor):
Since `exceed` is always false, could we please simplify the called function to assume this and eliminate the variable?

Code after:
        QString word;
        stream >> word;
        if (stream.status() && !stream.atEnd())
            return -1;
        charCount += word.length();
        if (!word.isEmpty())
            words.append(word);
        if (stream.status() || charCount + words.size() - 1 >= m_chunkSize) {
            if (!words.isEmpty()) {
                const QString chunk = words.join(" ");
                QSqlQuery q(m_db);
                int chunk_id = 0;
                if (!addChunk(q,
                    document_id,
                    chunk,
                    file,
                    title,
                    author,
                    subject,
                    keywords,
                    page,
                    line_from,
                    line_to,
                    words.size(),
                    &chunk_id
                )) {
                    qWarning() << "ERROR: Could not insert chunk into db" << q.lastError();
                }

                addedWords += words.size();

                EmbeddingChunk toEmbed;
                toEmbed.model = embedding_model;
                toEmbed.folder_id = folder_id;
                toEmbed.chunk_id = chunk_id;
                toEmbed.chunk = chunk;
                appendChunk(toEmbed);
                ++chunks;

                words.clear();
                charCount = 0;
            }

            if (stream.status() || (maxChunks > 0 && chunks == maxChunks))
                break;
        }
    }

    if (chunks) {
        CollectionItem item = guiCollectionItem(folder_id);