Repository: ggerganov/whisper.cpp
File: whisper.cpp (C++)

Code before:
    auto pos = path_bin.rfind('.');
    if (pos != std::string::npos) {
        path_bin = path_bin.substr(0, pos);
    }

    path_bin += "-encoder-openvino-cache";

    return path_bin;
}
#endif

struct whisper_state * whisper_init_state(whisper_context * ctx) {
    whisper_state * state = new whisper_state;

    state->backends = whisper_backend_init(ctx->params);
    if (state->backends.empty()) {
        WHISPER_LOG_ERROR("%s: whisper_backend_init() failed\n", __func__);
        whisper_free_state(state);
        return nullptr;
    }

    state->backends_used.push_back(ctx->backend);
    state->backends_used.insert(state->backends_used.end(), state->backends.begin(), state->backends.end());
    for (int i = 0; i < (int) state->backends_used.size(); ++i) {
        printf("state->backends_used[%d] = %p, name = %s\n", i, state->backends_used[i], ggml_backend_name(state->backends_used[i]));
    }

    state->mel_calc = whisper_mel_calc_create(state->backends[0], ctx->model.filters);

    // at this point, we don't know yet how many decoders will be used, so we overallocate 3x ctx
    // in theory, there can be a case where this is not enough, but in practice it should always be enough
    const int factor = 3;

    if (!whisper_kv_cache_init(state->kv_self, state->backends[0], ctx->itype,
                ctx->model.hparams.n_text_state,
                ctx->model.hparams.n_text_layer,
                GGML_PAD(ctx->model.hparams.n_text_ctx, 256)*factor)) {
        WHISPER_LOG_ERROR("%s: whisper_kv_cache_init() failed for self-attention cache\n", __func__);
        whisper_free_state(state);
        return nullptr;
    }

    {
        const size_t memory_size = ggml_nbytes(state->kv_self.k) + ggml_nbytes(state->kv_self.v);
        WHISPER_LOG_INFO("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1e6);
    }

    if (!whisper_kv_cache_init(state->kv_cross, state->backends[0], ctx->itype,
                ctx->model.hparams.n_text_state,
                ctx->model.hparams.n_text_layer,
                GGML_PAD(ctx->model.hparams.n_audio_ctx, 256))) {

Review comment (suggestion):
The backend instance in `whisper_context` is used to create a read-only buffer that contains all tensors of the model. Do I need to pass this backend together with the `whisper_state` backends when initiating the schedulers? Currently it works both ways

Code after:
        path_bin = path_bin.substr(0, pos);
    }

    path_bin += "-encoder-openvino-cache";

    return path_bin;
}
#endif

struct whisper_state * whisper_init_state(whisper_context * ctx) {
    whisper_state * state = new whisper_state;

    state->backends = whisper_backend_init(ctx->params);
    if (state->backends.empty()) {
        WHISPER_LOG_ERROR("%s: whisper_backend_init() failed\n", __func__);
        whisper_free_state(state);
        return nullptr;
    }

    state->mel_calc = whisper_mel_calc_create(state->backends[0], ctx->model.filters);

    // init 60s of random mel data
    {
        const int n_len = 2*100*WHISPER_CHUNK_SIZE;
        const int n_mel = ctx->model.filters.n_mel;

        whisper_mel_free(state->mel);
        whisper_mel_init(state->mel, state->backends[0], n_len, n_len, n_mel);
    }

    // at this point, we don't know yet how many decoders will be used, so we overallocate 3x ctx
    // in theory, there can be a case where this is not enough, but in practice it should always be enough
    const int factor = 3;

    if (!whisper_kv_cache_init(state->kv_self, state->backends[0], ctx->itype,
                ctx->model.hparams.n_text_state,
                ctx->model.hparams.n_text_layer,
                GGML_PAD(ctx->model.hparams.n_text_ctx, 256)*factor)) {
        WHISPER_LOG_ERROR("%s: whisper_kv_cache_init() failed for self-attention cache\n", __func__);
        whisper_free_state(state);
        return nullptr;
    }

    {
        const size_t memory_size = ggml_nbytes(state->kv_self.k) + ggml_nbytes(state->kv_self.v);
        WHISPER_LOG_INFO("%s: kv self size  = %7.2f MB\n", __func__, memory_size / 1e6);
    }

    if (!whisper_kv_cache_init(state->kv_cross, state->backends[0], ctx->itype,
                ctx->model.hparams.n_text_state,
                ctx->model.hparams.n_text_layer,