Repository: ggerganov/whisper.cpp
File: whisper.cpp (C++)

Code before:
        double entropy = 0.0f;

        std::map<whisper_token, int> token_counts;
        for (int i = std::max(0, sequence.result_len - n); i < sequence.result_len; ++i) {
            token_counts[sequence.tokens[i].id]++;
            cnt++;
        }

        for (const auto & kv : token_counts) {
            const auto p = kv.second/(double)cnt;
            entropy -= p*log(p);

            //WHISPER_PRINT_DEBUG("entropy: %d %f %f, count %d\n", kv.first, p, log(p), kv.second);
        }

        sequence.entropy = entropy;
    }
}

static bool whisper_fast_swap_KV(std::vector<int> & view, whisper_decoder src[], int size) {
    // (decoder->buffer->decoder or decoder->buffer + decoder->decoder)
    std::unordered_set<int> two_copy; // decoder indices requires two copies to safely modify KV caches
    two_copy.reserve(size);

    // (buffer->decoder or decoder->decoder)
    std::unordered_set<int> one_copy; // decoder indices requires one copies to safely modify KV caches
    one_copy.reserve(size);

    for (int i = 0; i < size; i++) {
        // zero-copy (no modification)
        if (i == view[i] || view[i] < 0) {continue;}

        bool is_one_copy = true;
        // since we modify data sequentially, we only consider decoder indices after current index
        for (int j = i + 1; j < size; j++) {
            if (i == view[j]) {
                two_copy.insert(i);
                is_one_copy = false;
                break;
            }
        }
        if (is_one_copy) {
            one_copy.insert(i);
        }
    }

    // beam-search helpers
    struct kv_buf {
        std::vector<uint8_t> k;
        std::vector<uint8_t> v;
    };

Review comment (performance):
```suggestion
    std::unordered_set<int> one_copy; // decoder indices require one copy to safely modify KV caches
```

Code after:

    if (params.length_penalty > 0.0f) {
        penalty = pow((5.0 + penalty)/6.0, params.length_penalty);
    }

    sequence.score = result/penalty;

    // compute the entropy of the sequence of the last 32 tokens
    {
        const int n = 32;

        int cnt = 0;
        double entropy = 0.0f;

        std::map<whisper_token, int> token_counts;
        for (int i = std::max(0, sequence.result_len - n); i < sequence.result_len; ++i) {
            token_counts[sequence.tokens[i].id]++;
            cnt++;
        }

        for (const auto & kv : token_counts) {
            const auto p = kv.second/(double)cnt;
            entropy -= p*log(p);

            //WHISPER_PRINT_DEBUG("entropy: %d %f %f, count %d\n", kv.first, p, log(p), kv.second);
        }

        sequence.entropy = entropy;
    }
}

static bool whisper_kv_swap_fast(
                   std::vector<int> & view,
                    whisper_decoder   src[],
                std::vector<kv_buf> & kv_swap_bufs,
                          const int & n_decoders) {
    WHISPER_PRINT_DEBUG("%s: n_decoders %d\n", __func__, n_decoders);

    // (decoder->buffer->decoder or decoder->buffer + decoder->decoder)
    std::set<int> two_copy; // decoder indices require two copies to safely modify KV caches

    // (buffer->decoder or decoder->decoder)
    std::set<int> one_copy; // decoder indices require one copy to safely modify KV caches

    // (decoder<->decoder)
    std::set<int> p_swap_set; // decoder indices able to swap KV-cache pointers
    std::vector<whisper_pair<int, int>> p_swap_vec;
    p_swap_vec.reserve(n_decoders);

    // see https://github.com/ggerganov/whisper.cpp/wiki
    for (int i = 0; i < n_decoders; i++) {