Repository: nomic-ai/gpt4all
File: gpt4all-chat/src/database.cpp (C++)

Code before:
    if (!missingScores.isEmpty()) {
        QList<int> scored = scoreChunks(query, missingScores);
        results << scored;
    }

    QHash<int, int> embeddingRanks;
    for (int i = 0; i < results.size(); ++i)
        embeddingRanks[results[i]] = i + 1;

    const float bmWeight = bm25Results.isEmpty() ? 0 : computeBM25Weight(bm25q);
    const int fusion_k = 60;

    std::stable_sort(
        results.begin(), results.end(),
        [&](const int &a, const int &b) {
            // Reciprocal Rank Fusion (RRF)
            const int aBm25Rank = bm25Ranks.value(a, bm25Results.size() + 1);
            const int aEmbeddingRank = embeddingRanks.value(a, embeddingResults.size() + 1);
            Q_ASSERT(embeddingRanks.contains(a));

            const int bBm25Rank = bm25Ranks.value(b, bm25Results.size() + 1);
            const int bEmbeddingRank = embeddingRanks.value(b, embeddingResults.size() + 1);
            Q_ASSERT(embeddingRanks.contains(b));

            const float aRRFScore = bmWeight * (1.0f / (fusion_k + aBm25Rank)) + (1.f - bmWeight) * (1.0f / (fusion_k + aEmbeddingRank));
            const float bRRFScore = bmWeight * (1.0f / (fusion_k + bBm25Rank)) + (1.f - bmWeight) * (1.0f / (fusion_k + bEmbeddingRank));

            // Higher RRF score means better ranking, so we use greater than for sorting
            return aRRFScore > bRRFScore;
        }
    );

    k = qMin(k, results.size());
    results.resize(k);
    return results;
}

QList<int> Database::searchDatabase(const QString &query, const QList<QString> &collections, int k)
{
    std::vector<float> queryEmbd = m_embLLM->generateQueryEmbedding(query);
    if (queryEmbd.empty()) {
        qDebug() << "ERROR: generating embeddings returned a null result";
        return { };
    }

    const QList<int> embeddingResults = searchEmbeddings(queryEmbd, collections, k);
    BM25Query bm25q;
    const QList<int> bm25Results = searchBM25(query, collections, bm25q, k);
    return reciprocalRankFusion(queryEmbd, embeddingResults, bm25Results, bm25q, k);
}


Review comment (refactor):
the weighted average should be abstracted into its own function for readability (outside of this lambda), e.g.:
```cpp
auto mix = [](float a, float b, float f) { return a * f + b * (1.0f - f); };
```

Code after:
        info.page = page;
        info.from = from;
        info.to = to;
        tempResults.insert(rowid, info);
#if defined(DEBUG)
        qDebug() << "retrieve rowid:" << rowid
                 << "chunk_text:" << chunk_text;
#endif
    }

    for (int id : searchResults)
        if (tempResults.contains(id))
            results->append(tempResults.value(id));
}

// FIXME This is very slow and non-interruptible and when we close the application and we're
// cleaning a large table this can cause the app to take forever to shut down. This would ideally be
// interruptible and we'd continue 'cleaning' when we restart
bool Database::cleanDB()
{
#if defined(DEBUG)
    qDebug() << "cleanDB";
#endif

    // Scan all folders in db to make sure they still exist
    QSqlQuery q(m_db);
    QList<CollectionItem> collections;
    if (!selectAllFromCollections(q, &collections)) {
        qWarning() << "ERROR: Cannot select collections" << q.lastError();
        return false;
    }

    transaction();

    for (const auto &i: std::as_const(collections)) {
        // Find the path for the folder
        QFileInfo info(i.folder_path);
        if (!info.exists() || !info.isReadable()) {
#if defined(DEBUG)
            qDebug() << "clean db removing folder" << i.folder_id << i.folder_path;
#endif
            if (!removeFolderInternal(i.collection, i.folder_id, i.folder_path)) {
                rollback();
                return false;
            }
        }
    }

    // Scan all documents in db to make sure they still exist
    if (!q.prepare(SELECT_ALL_DOCUMENTS_SQL)) {
        qWarning() << "ERROR: Cannot prepare sql for select all documents" << q.lastError();