Repository: nomic-ai/gpt4all
File: gpt4all-chat/src/database.cpp (C++)

Code before:
        if (!item->embeddingModel.isNull()) {
            addFolderToWatch(path);
            scanDocuments(folder_id, path);
        }
    }
    return true;
}

void Database::removeFolder(const QString &collection, const QString &path)
{
#if defined(DEBUG)
    qDebug() << "removeFolder" << path;
#endif

    QSqlQuery q(m_db);
    int folder_id = -1;

    // See if the folder exists in the db
    if (!selectFolder(q, path, &folder_id)) {
        qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError();
        return;
    }

    // If we don't have a folder_id in the db, then something bad has happened
    Q_ASSERT(folder_id != -1);
    if (folder_id == -1) {
        qWarning() << "ERROR: Collected folder does not exist in db" << path;
        m_watcher->removePath(path);
        return;
    }

    transaction();

    if (removeFolderInternal(collection, folder_id, path)) {
        commit();
    } else {
        rollback();
    }
}

bool Database::removeFolderInternal(const QString &collection, int folder_id, const QString &path)
{
    // Remove it from the collection
    QSqlQuery q(m_db);
    int nRemaining = removeCollectionFolder(q, collection, folder_id);
    if (nRemaining == -1) {
        qWarning().nospace() << "Database ERROR: Cannot remove collection " << collection << " from folder "
                             << folder_id << ": " << q.lastError();
        return false;
    }
    removeGuiFolderById(collection, folder_id);

Review comment (suggestion):
Should add a comment explaining that `q` is expected to be the result of a SQL query returning (chunk_id, embedding) pairs. Also, I think this function can be local to database.cpp (`static`) since it no longer needs m_db.

Code after:
    qDebug() << "addFolderToWatch" << path;
#endif
    // pre-check because addPath returns false for already watched paths
    if (!m_watchedPaths.contains(path)) {
        if (!m_watcher->addPath(path))
            qWarning() << "Database::addFolderToWatch: failed to watch" << path;
        // add unconditionally to suppress repeated warnings
        m_watchedPaths << path;
    }
}

void Database::removeFolderFromWatch(const QString &path)
{
#if defined(DEBUG)
    qDebug() << "removeFolderFromWatch" << path;
#endif
    QDirIterator it(path, QDir::Readable | QDir::Dirs | QDir::NoDotAndDotDot, QDirIterator::Subdirectories);
    QStringList children { path };
    while (it.hasNext())
        children.append(it.next());

    m_watcher->removePaths(children);
    m_watchedPaths -= QSet(children.begin(), children.end());
}

QList<int> Database::searchEmbeddingsHelper(const std::vector<float> &query, QSqlQuery &q, int nNeighbors)
{
    constexpr int BATCH_SIZE = 2048;

    const int n_embd = query.size();
    const us::metric_punned_t metric(n_embd, us::metric_kind_t::ip_k); // inner product

    us::executor_default_t executor(std::thread::hardware_concurrency());
    us::exact_search_t search;

    QList<int> batchChunkIds;
    QList<float> batchEmbeddings;
    batchChunkIds.reserve(BATCH_SIZE);
    batchEmbeddings.reserve(BATCH_SIZE * n_embd);

    struct Result { int chunkId; us::distance_punned_t dist; };
    QList<Result> results;

    // The q parameter is expected to be the result of a QSqlQuery returning (chunk_id, embedding) pairs
    while (q.at() != QSql::AfterLastRow) { // batches
        batchChunkIds.clear();
        batchEmbeddings.clear();

        while (batchChunkIds.count() < BATCH_SIZE && q.next()) { // batch
            batchChunkIds << q.value(0).toInt();
            batchEmbeddings.resize(batchEmbeddings.size() + n_embd);