Repository: nomic-ai/gpt4all
File: gpt4all-chat/src/database.cpp (C++)

Code before:

    int chunks = 0;
    int addedWords = 0;

    QString buffer;
    size_t bufferSz = m_chunkSize * 2;
    buffer.reserve(bufferSz);

    QList<QString> chunkOutputs;
    while (!stream.atEnd() || !buffer.isEmpty()) {
        if (!stream.atEnd())
            buffer += stream.read(bufferSz - buffer.size());

        if (stream.status()) {
            Q_UNREACHABLE();
            qWarning() << "ERROR: Could not chunk stream" << stream.status();
            return -1;
        }

        QString chunk = textSplitter(buffer, m_chunkSize, true /*exceed*/);

        // If the stream has an ASCII EOT (end of transmission), then we cannot read any more data
        // even by stream.readAll() yet the stream will not be at an end. We detect this and just
        // stop the loop with an error return
        if (chunk.isEmpty() && !stream.atEnd())
            return -1;

        // Drop whitespace-only chunks on the floor
        if (chunk.trimmed().isEmpty())
            continue;

        chunkOutputs << chunk;

        // Count the words
        int words = 0;
        QTextStream wordsStream(&chunk);
        for (;;) {
            QString word;
            wordsStream >> word;
            if (!word.isEmpty())
                ++words;
            if (wordsStream.status())
                break;
        }
        addedWords += words;

        QSqlQuery q(m_db);
        int chunk_id = 0;
        if (!addChunk(q,
            document_id,
            chunk,

Review comment (suggestion):
There is nothing special about EOT, this is just one of many ASCII control characters we can use to assume the input is binary (see BinaryDetectingFile::checkData). If we want to allow all valid UTF-8 instead of assuming some kind of extended ASCII, we could blacklist 0xC0, 0xC1, and 0xF5 - 0xFF instead - we should probably check for those either way.

The old check was `stream.status() && !stream.atEnd()` which I think is more explicit: it checks for any error other than EOF and aborts, even if we did read some data (since in the case of BinaryDetectingFile we are going to delete the file from the database afterwards).

Code after:
            words.append(word);
        if (stream.status() || charCount + words.size() - 1 >= m_chunkSize) {
            if (!words.isEmpty()) {
                const QString chunk = words.join(" ");
                QSqlQuery q(m_db);
                int chunk_id = 0;
                if (!addChunk(q,
                    document_id,
                    chunk,
                    file,
                    title,
                    author,
                    subject,
                    keywords,
                    page,
                    line_from,
                    line_to,
                    words.size(),
                    &chunk_id
                )) {
                    qWarning() << "ERROR: Could not insert chunk into db" << q.lastError();
                }

                addedWords += words.size();

                EmbeddingChunk toEmbed;
                toEmbed.model = embedding_model;
                toEmbed.folder_id = folder_id;
                toEmbed.chunk_id = chunk_id;
                toEmbed.chunk = chunk;
                appendChunk(toEmbed);
                ++chunks;

                words.clear();
                charCount = 0;
            }

            if (stream.status() || (maxChunks > 0 && chunks == maxChunks))
                break;
        }
    }

    if (chunks) {
        CollectionItem item = guiCollectionItem(folder_id);

        // Set the start update if we haven't done so already
        if (item.startUpdate <= item.lastUpdate && item.currentEmbeddingsToIndex == 0)
            setStartUpdateTime(item);

        item.currentEmbeddingsToIndex += chunks;
        item.totalEmbeddingsToIndex += chunks;