Repository: nomic-ai/gpt4all
File: gpt4all-chat/src/database.cpp (C++)

Code before:
{
    QList<QString> chunkStrings;
    for (int id : chunks)
        chunkStrings << QString::number(id);
    QSqlQuery q(m_db);
    if (!q.exec(GET_CHUNK_EMBEDDINGS_SQL.arg(chunkStrings.join("', '")))) {
        qWarning() << "Database ERROR: Failed to exec embeddings query:" << q.lastError();
        return {};
    }
    return searchEmbeddingsHelper(query, q, chunks.size());
}

QList<Database::BM25Query> Database::queriesForFTS5(const QString &input)
{
    // Escape double quotes by adding a second double quote
    QString escapedInput = input;
    escapedInput.replace("\"", "\"\"");

    static QRegularExpression spaces("\\s+");
    QStringList oWords = escapedInput.split(spaces, Qt::SkipEmptyParts);

    QList<BM25Query> queries;

    // Start by trying to match the entire input
    BM25Query e;
    e.type = 0; // exact
    e.input = oWords.join(" ");
    e.query = "\"" + oWords.join(" ") + "\"";
    e.qlength = oWords.size();
    e.ilength = oWords.size();
    queries << e;

    // https://github.com/igorbrigadir/stopwords?tab=readme-ov-file
    // Lucene, Solr, Elastisearch
    static const QList<QString> stopWords = {
        "a", "an", "and", "are", "as", "at", "be", "but", "by",
        "for", "if", "in", "into", "is", "it", "no", "not", "of",
        "on", "or", "such", "that", "the", "their", "then", "there",
        "these", "they", "this", "to", "was", "will", "with"
    };

    QStringList rWords;
    for (const QString &w : oWords)
        if (!stopWords.contains(w.toLower()))
            rWords << w;

    QStringList quotedWords;
    for (const QString &w : rWords)
        quotedWords << "\"" + w + "\"";

    BM25Query b;

Review comment (suggestion):
Please either use a boolean (`bool isExact`) or an enumeration, so the type field can be self-documenting.

Code after:

float Database::computeBM25Weight(const Database::BM25Query &bm25q)
{
    float bmWeight = 0.0f;
    if (bm25q.isExact) {
        bmWeight = 0.9f; // the highest we give
    } else {
        // qlength is the length of the phrases in the query by number of distinct words
        // ilength is the length of the natural language query by number of distinct words
        // rlength is the number of stop words removed from the natural language query to form the query

        // calculate the query length weight based on the ratio of query terms to meaningful terms.
        // this formula adjusts the weight with the empirically determined insight that BM25's
        // effectiveness decreases as query length increases.
        float queryLengthWeight = 1 / powf(float(bm25q.ilength - bm25q.rlength), 2);
        queryLengthWeight = qBound(0.0f, queryLengthWeight, 1.0f);

        // the weighting is bound between 1/4 and 3/4 which was determined empirically to work well
        // with the beir nfcorpus, scifact, fiqa and trec-covid datasets along with our embedding
        // model
        bmWeight = 0.25f + queryLengthWeight * 0.50f;
    }

#if 0
    qDebug()
        << "bm25q.type"         << bm25q.type
        << "bm25q.qlength"      << bm25q.qlength
        << "bm25q.ilength"      << bm25q.ilength
        << "bm25q.rlength"      << bm25q.rlength
        << "bmWeight"           << bmWeight;
#endif

    return bmWeight;
}

QList<int> Database::reciprocalRankFusion(const std::vector<float> &query, const QList<int> &embeddingResults,
    const QList<int> &bm25Results, const BM25Query &bm25q, int k)
{
    // We default to the embedding results and augment with bm25 if any
    QList<int> results = embeddingResults;

    QList<int> missingScores;
    QHash<int, int> bm25Ranks;
    for (int i = 0; i < bm25Results.size(); ++i) {
        if (!results.contains(bm25Results[i]))
            missingScores.append(bm25Results[i]);
        bm25Ranks[bm25Results[i]] = i + 1;
    }

    if (!missingScores.isEmpty()) {
        QList<int> scored = scoreChunks(query, missingScores);