Repository: nomic-ai/gpt4all
File: gpt4all-chat/src/database.cpp (C++)

Code before:
    std::partial_sort(
        results.begin(), results.begin() + k, results.end(),
        [](const SearchResult &a, const SearchResult &b) { return a.dist < b.dist; }
    );

    QList<int> chunkIds;
    chunkIds.reserve(k);
    for (int i = 0; i < k; i++)
        chunkIds << results[i].chunkId;
    return chunkIds;
}

float Database::computeBM25Weight(const Database::BM25Query &bm25q)
{
    float bmWeight = 0.0f;
    if (bm25q.type == 0 /*exact*/) {
        bmWeight = 0.9f; // the highest we give
    } else {
        // qlength is the length of the phrases in the query by number of distinct words
        // ilength is the length of the natural language query by number of distinct words
        // rlength is the number of stop words removed from the natural language query to form the query
        float queryLengthWeight = powf(bm25q.qlength, 2) / powf(float(bm25q.ilength - bm25q.rlength), 2);
        queryLengthWeight = qBound(0.0f, queryLengthWeight, 1.0f);
        bmWeight = queryLengthWeight;
        bmWeight = qBound(0.0f, bmWeight, 1.0f);
        bmWeight = (0.25f + bmWeight * (0.75f - 0.25f));
    }

#if 0
    qDebug()
        << "bm25q.type"         << bm25q.type
        << "bm25q.qlength"      << bm25q.qlength
        << "bm25q.ilength"      << bm25q.ilength
        << "bm25q.rlength"      << bm25q.rlength
        << "bmWeight"           << bmWeight;
#endif

    return bmWeight;
}

QList<int> Database::reciprocalRankFusion(const std::vector<float> &query, const QList<int> &embeddingResults,
    const QList<int> &bm25Results, const BM25Query &bm25q, int k)
{
    // We default to the embedding results and augment with bm25 if any
    QList<int> results = embeddingResults;

    QList<int> missingScores;
    QHash<int, int> bm25Ranks;
    for (int i = 0; i < bm25Results.size(); ++i) {
        if (!results.contains(bm25Results[i]))
            missingScores.append(bm25Results[i]);

Review comment (suggestion):
Should add a comment briefly describing that this is an original formula and was produced experimentally. There are some simplifications that can be made here, such as `0.75f - 0.25f` -> `0.5f`, and `powf(bm25q.qlength, 2)` -> `1` as you mentioned.

Code after:
}

QList<int> Database::searchDatabase(const QString &query, const QList<QString> &collections, int k)
{
    std::vector<float> queryEmbd = m_embLLM->generateQueryEmbedding(query);
    if (queryEmbd.empty()) {
        qDebug() << "ERROR: generating embeddings returned a null result";
        return { };
    }

    const QList<int> embeddingResults = searchEmbeddings(queryEmbd, collections, k);
    BM25Query bm25q;
    const QList<int> bm25Results = searchBM25(query, collections, bm25q, k);
    return reciprocalRankFusion(queryEmbd, embeddingResults, bm25Results, bm25q, k);
}

void Database::retrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize,
    QList<ResultInfo> *results)
{
#if defined(DEBUG)
    qDebug() << "retrieveFromDB" << collections << text << retrievalSize;
#endif

    QList<int> searchResults = searchDatabase(text, collections, retrievalSize);
    if (searchResults.isEmpty())
        return;

    QSqlQuery q(m_db);
    if (!selectChunk(q, searchResults, retrievalSize)) {
        qDebug() << "ERROR: selecting chunks:" << q.lastError();
        return;
    }

    QHash<int, ResultInfo> tempResults;
    while (q.next()) {
        const int rowid = q.value(0).toInt();
        const QString document_path = q.value(2).toString();
        const QString chunk_text = q.value(3).toString();
        const QString date = QDateTime::fromMSecsSinceEpoch(q.value(1).toLongLong()).toString("yyyy, MMMM dd");
        const QString file = q.value(4).toString();
        const QString title = q.value(5).toString();
        const QString author = q.value(6).toString();
        const int page = q.value(7).toInt();
        const int from = q.value(8).toInt();
        const int to = q.value(9).toInt();
        const QString collectionName = q.value(10).toString();
        ResultInfo info;
        info.collection = collectionName;
        info.path = document_path;
        info.file = file;
        info.title = title;