Repository: nomic-ai/gpt4all
File: gpt4all-chat/src/database.cpp (C++)

Code before:
    // Start by trying to match the entire input
    BM25Query e;
    e.type = 0; // exact
    e.input = oWords.join(" ");
    e.query = "\"" + oWords.join(" ") + "\"";
    e.qlength = oWords.size();
    e.ilength = oWords.size();
    queries << e;

    // https://github.com/igorbrigadir/stopwords?tab=readme-ov-file
    // Lucene, Solr, Elastisearch
    static const QList<QString> stopWords = {
        "a", "an", "and", "are", "as", "at", "be", "but", "by",
        "for", "if", "in", "into", "is", "it", "no", "not", "of",
        "on", "or", "such", "that", "the", "their", "then", "there",
        "these", "they", "this", "to", "was", "will", "with"
    };

    QStringList rWords;
    for (const QString &w : oWords)
        if (!stopWords.contains(w.toLower()))
            rWords << w;

    QStringList quotedWords;
    for (const QString &w : rWords)
        quotedWords << "\"" + w + "\"";

    BM25Query b;
    b.type = 1; // broad search
    b.input = oWords.join(" ");
    b.query = "(" + quotedWords.join(" OR ") + ")";
    b.qlength = 1; // length of phrase
    b.ilength = oWords.size();
    b.rlength = oWords.size() - rWords.size();
    queries << b;
    return queries;
}

QList<int> Database::searchBM25(const QString &query, const QList<QString> &collections, BM25Query &bm25q, int k)
{
    struct SearchResult { int chunkId; float dist; };
    QList<BM25Query> bm25Queries = queriesForFTS5(query);

    QList<SearchResult> results;
    for (auto bm25Query : bm25Queries) {
        QSqlQuery sqlQuery(m_db);
        sqlQuery.prepare(SELECT_CHUNKS_FTS_SQL.arg(k));
        sqlQuery.addBindValue(bm25Query.query);

        if (!sqlQuery.exec()) {
            qWarning() << "Database ERROR: Failed to execute BM25 query:" << sqlQuery.lastError();

Review comment (suggestion):
rWords is only used for its size; these two loops can be combined:
```suggestion
    QStringList quotedWords;
    for (const QString &w : oWords)
        if (!stopWords.contains(w.toLower()))
            quotedWords << "\"" + w + "\"";
```

Code after:
#if 0
    qDebug()
        << "bm25q.type"         << bm25q.type
        << "bm25q.qlength"      << bm25q.qlength
        << "bm25q.ilength"      << bm25q.ilength
        << "bm25q.rlength"      << bm25q.rlength
        << "bmWeight"           << bmWeight;
#endif

    return bmWeight;
}

QList<int> Database::reciprocalRankFusion(const std::vector<float> &query, const QList<int> &embeddingResults,
    const QList<int> &bm25Results, const BM25Query &bm25q, int k)
{
    // We default to the embedding results and augment with bm25 if any
    QList<int> results = embeddingResults;

    QList<int> missingScores;
    QHash<int, int> bm25Ranks;
    for (int i = 0; i < bm25Results.size(); ++i) {
        if (!results.contains(bm25Results[i]))
            missingScores.append(bm25Results[i]);
        bm25Ranks[bm25Results[i]] = i + 1;
    }

    if (!missingScores.isEmpty()) {
        QList<int> scored = scoreChunks(query, missingScores);
        results << scored;
    }

    QHash<int, int> embeddingRanks;
    for (int i = 0; i < results.size(); ++i)
        embeddingRanks[results[i]] = i + 1;

    const float bmWeight = bm25Results.isEmpty() ? 0 : computeBM25Weight(bm25q);

    // From the paper: "Reciprocal Rank Fusion outperforms Condorcet and individual Rank Learning Methods"
    // doi: 10.1145/1571941.157211
    const int fusion_k = 60;

    std::stable_sort(
        results.begin(), results.end(),
        [&](const int &a, const int &b) {
            // Reciprocal Rank Fusion (RRF)
            const int aBm25Rank = bm25Ranks.value(a, bm25Results.size() + 1);
            const int aEmbeddingRank = embeddingRanks.value(a, embeddingResults.size() + 1);
            Q_ASSERT(embeddingRanks.contains(a));

            const int bBm25Rank = bm25Ranks.value(b, bm25Results.size() + 1);
            const int bEmbeddingRank = embeddingRanks.value(b, embeddingResults.size() + 1);