Repository: ggerganov/whisper.cpp
File: whisper.cpp (C++)

Code before:

    //WHISPER_LOG_DEBUG("%s: n_past = %d, n_tokens = %d, n_audio_ctx = %d, n_ctx = %d\n", __func__, n_past, n_tokens, n_audio_ctx, n_ctx);

    struct ggml_init_params params = {
        /*.mem_size   =*/ wstate.alloc_decode.meta.size(),
        /*.mem_buffer =*/ wstate.alloc_decode.meta.data(),
        /*.no_alloc   =*/ true,
    };

    struct ggml_context * ctx0 = ggml_init(params);

    ggml_cgraph * gf = ggml_new_graph_custom(ctx0, WHISPER_MAX_NODES, false);

    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
    ggml_set_name(embd, "embd");
    //ggml_set_input(embd);

    struct ggml_tensor * position = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
    ggml_set_name(position, "position");
    //ggml_set_input(position);

    const float KQscale = pow(float(n_state_head), -0.25);

    struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD), 1);
    ggml_set_name(KQ_mask, "KQ_mask");
    //ggml_set_input(KQ_mask);

    struct ggml_tensor * KQ_mask_f16 = ggml_cast(ctx0, KQ_mask, GGML_TYPE_F16);

    // token encoding + position encoding
    struct ggml_tensor * cur =
        ggml_add(ctx0,
                ggml_get_rows(ctx0, model.d_te, embd),
                ggml_get_rows(ctx0, model.d_pe, position));

    struct ggml_tensor * inpL = cur;

    // [EXPERIMENTAL] Token-level timestamps with DTW
    struct ggml_tensor * aheads_cross_QKs = nullptr;

    for (int il = 0; il < n_layer; ++il) {
        const auto & layer = model.layers_decoder[il];

        // norm
        {
            cur = ggml_norm(ctx0, inpL, hparams.eps);

            // cur = ln_0_w*cur + ln_0_b
            cur = ggml_add(ctx0,
                    ggml_mul(ctx0,
                        cur,

Review comment (bug):
When running with the Metal backend, having these `ggml_set_input()` calls in the decoder graph causes the assigned backends to be NULL (nodes 0, 1, 27):

```
## SPLIT #0: Metal # 0 inputs: 
node #  0 (  GET_ROWS):               node_0 (   2K) [Metal   1.wgt0]:               leaf_1 (  50M) [Metal    1.dst]         Metal#embd#0 (   1K) [ NULL    4.cpy]
node #  1 (  GET_ROWS):               node_1 (   2K) [Metal   1.wgt0]:               leaf_3 ( 896K) [Metal    1.dst]     Metal#position#0 (   1K) [ NULL    4.cpy]
node #  2 (       ADD):               node_2 (   2K) [Metal    2.sup]:               node_0 (   2K) [Metal   1.wgt0]               node_1 (   2K) [Metal   1.wgt0]
node #  3 (      NORM):               node_3 (   2K) [Metal    2.sup]:               node_2 (   2K) [Metal    2.sup]
node #  4 (       MUL):               node_4 (   2K) [Metal   1.wgt1]:               node_3 (   2K) [Metal    2.sup]               leaf_5 (   2K) [Metal    1.dst]
node #  5 (       ADD):               node_5 (   2K) [Metal   1.wgt1]:               node_4 (   2K) [Metal   1.wgt1]               leaf_6 (   2K) [Metal    1.dst]
node #  6 (   MUL_MAT):               node_6 (   2K) [Metal   1.wgt0]:               leaf_0 ( 512K) [Metal    1.dst]               node_5 (   2K) [Metal   1.wgt1]
node #  7 (     SCALE):               node_7 (   2K) [Metal    2.sup]:               node_6 (   2K) [Metal   1.wgt0]
node #  9 (       CPY): leaf_7 (view) (copy  (   1K) [Metal   1.vsrc]:               node_7 (   2K) [Metal    2.sup]        leaf_7 (view) (   1K) [Metal   1.vsrc]
node # 10 (   MUL_MAT):              node_10 (   2K) [Metal   1.wgt0]:               leaf_8 ( 512K) [Metal    1.dst]               node_5 (   2K) [Metal   1.wgt1]
node # 11 (       ADD):              node_11 (   2K) [Metal   1.wgt1]:              node_10 (   2K) [Metal   1.wgt0]               leaf_9 (   2K) [Metal    1.dst]
node # 15 (       CPY): leaf_10 (view) (copy (   1M) [Metal   1.vsrc]:  (reshaped) (transpo (   2K) [Metal   4.vsrc]       leaf_10 (view) (   1M) [Metal   1.vsrc]
node # 21 (   MUL_MAT):              node_21 (   2K) [Metal   1.wgt0]:              leaf_19 ( 512K) [Metal    1.dst]               node_5 (   2K) [Metal   1.wgt1]
node # 22 (       ADD):              node_22 (   2K) [Metal   1.wgt1]:              node_21 (   2K) [Metal   1.wgt0]              leaf_20 (   2K) [Metal    1.dst]
node # 23 (     SCALE):              node_23 (   2K) [Metal    2.sup]:              node_22 (   2K) [Metal   1.wgt1]
node # 26 (   MUL_MAT):              node_26 (   0K) [Metal    2.sup]:        leaf_7 (view) (   1K) [Metal   1.vsrc]  (reshaped) (permute (   2K) [Metal   4.vsrc]
node # 27 (  SOFT_MAX):              node_27 (   0K) [Metal    2.sup]:              node_26 (   0K) [Metal    2.sup]      Metal#KQ_mask#0 (   2M) [ NULL    4.cpy]
node # 28 (   MUL_MAT):              node_28 (   2K) [Metal    2.sup]:       leaf_10 (view) (   1M) [Metal   1.vsrc]              node_27 (   0K) [Metal    2.sup]
```

Is this expected? Without Metal it works as expected. Removing the calls also seems to work, but seems incorrect since the `KQ_mask` might get overwritten?

Code after:

    struct ggml_init_params params = {
        /*.mem_size   =*/ wstate.sched_decode.meta.size(),
        /*.mem_buffer =*/ wstate.sched_decode.meta.data(),
        /*.no_alloc   =*/ true,
    };

    struct ggml_context * ctx0 = ggml_init(params);

    ggml_cgraph * gf = ggml_new_graph_custom(ctx0, WHISPER_MAX_NODES, false);

    struct ggml_tensor * embd = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
    ggml_set_name(embd, "embd");
    ggml_set_input(embd);

    struct ggml_tensor * position = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
    ggml_set_name(position, "position");
    ggml_set_input(position);

    const float KQscale = pow(float(n_state_head), -0.25);

    struct ggml_tensor * KQ_mask = ggml_new_tensor_3d(ctx0, GGML_TYPE_F32, n_kv, GGML_PAD(n_tokens, GGML_KQ_MASK_PAD), 1);
    ggml_set_name(KQ_mask, "KQ_mask");
    ggml_set_input(KQ_mask);

    struct ggml_tensor * KQ_mask_f16 = ggml_cast(ctx0, KQ_mask, GGML_TYPE_F16);

    // token encoding + position encoding
    struct ggml_tensor * cur =
        ggml_add(ctx0,
                ggml_get_rows(ctx0, model.d_te, embd),
                ggml_get_rows(ctx0, model.d_pe, position));

    struct ggml_tensor * inpL = cur;

    // [EXPERIMENTAL] Token-level timestamps with DTW
    struct ggml_tensor * aheads_cross_QKs = nullptr;

    for (int il = 0; il < n_layer; ++il) {
        const auto & layer = model.layers_decoder[il];

        // norm
        {
            cur = ggml_norm(ctx0, inpL, hparams.eps);

            // cur = ln_0_w*cur + ln_0_b
            cur = ggml_add(ctx0,
                    ggml_mul(ctx0,
                        cur,
                        layer.attn_ln_0_w),
                    layer.attn_ln_0_b);