Repository: ggerganov/whisper.cpp
File: whisper.cpp (C++)

Code before:
        const int i1 = std::min(mel_offset + out_s, n_len);
        const int mel_s = i1 - i0;

        assert(mel_inp->type == GGML_TYPE_F32);
        assert(mel_inp->ne[1] == n_mels);

        ggml_tensor * cur = ggml_view_2d(ctx0, mel_inp, out_s, n_mels, mel_inp->nb[1], ggml_row_size(mel_inp->type, i0));

        if (mel_s < out_s) {
            mel = ggml_pad(ctx0, cur, out_s - mel_s, 0, 0, 0);
        }
        else {
            mel = ggml_cont(ctx0, cur);
        }
    }
    else {
        // just create some tensor so that the graph/buffer size estimation is correct
        mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 2 * n_ctx, n_mels);
    }

    if (mel_only) {
        ggml_set_output(mel);
        ggml_build_forward_expand(gf, mel);
        ggml_free(ctx0);
        return gf;
    }

    struct ggml_tensor * cur = nullptr;

    if (!whisper_encode_external(wstate)) {
        // convolution + gelu
        {
            cur = ggml_conv_1d_ph(ctx0, model.e_conv_1_w, mel, 1, 1);
            cur = ggml_add(ctx0, cur, model.e_conv_1_b);

            cur = ggml_gelu(ctx0, cur);

            cur = ggml_conv_1d_ph(ctx0, model.e_conv_2_w, cur, 2, 1);
            cur = ggml_add(ctx0, cur, model.e_conv_2_b);

            cur = ggml_gelu(ctx0, cur);
        }

        ggml_set_name(cur, "embd_conv");
        wstate.embd_conv = cur;
    } else {
        ggml_build_forward_expand(gf, mel);

        cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_state, n_ctx);

        ggml_set_name(cur, "embd_enc");

Review comment (bug):
This breaks the external encoder logic because the `wstate.embd_enc` tensor is not initialized.

I think `mel_only` is not needed at all:

```diff
diff --git a/whisper.cpp b/whisper.cpp
index b8fa77d..e0e6719 100644
--- a/whisper.cpp
+++ b/whisper.cpp
@@ -1815,8 +1815,7 @@ static bool whisper_encode_external(const whisper_state & wstate) {
 static struct ggml_cgraph * whisper_build_graph_conv(
         whisper_context & wctx,
           whisper_state & wstate,
-              const int   mel_offset,
-              bool        mel_only) {
+              const int   mel_offset) {
     const auto & model   = wctx.model;
     const auto & hparams = model.hparams;
 
@@ -1861,13 +1860,6 @@ static struct ggml_cgraph * whisper_build_graph_conv(
         mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 2 * n_ctx, n_mels);
     }
 
-    if (mel_only) {
-        ggml_set_output(mel);
-        ggml_build_forward_expand(gf, mel);
-        ggml_free(ctx0);
-        return gf;
-    }
-
     struct ggml_tensor * cur = nullptr;
 
     if (!whisper_encode_external(wstate)) {
@@ -2248,9 +2240,7 @@ static bool whisper_encode_internal(
     {
         auto & alloc = wstate.alloc_conv.alloc;
 
-        bool encode_external = whisper_encode_external(wstate);
-
-        ggml_cgraph * gf = whisper_build_graph_conv(wctx, wstate, mel_offset, encode_external);
+        ggml_cgraph * gf = whisper_build_graph_conv(wctx, wstate, mel_offset);
 
         if (!ggml_gallocr_alloc_graph(alloc, gf)) {
             // should never happen as we pre-allocate the memory
@@ -2261,7 +2251,7 @@ static bool whisper_encode_internal(
             return false;
         }
 
-        if (encode_external) {
+        if (whisper_encode_external(wstate)) {
             ggml_tensor * mel = gf->nodes[gf->n_nodes - 1];
             assert(mel->ne[1] == wctx.model.hparams.n_mels);
             GGML_UNUSED(mel);
@@ -3427,7 +3417,7 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) {
     {
         bool ok = whisper_allocr_graph_init(state->alloc_conv, ctx->backend,
                 [&]() {
-                    return whisper_build_graph_conv(*ctx, *state, 0, false);
+                    return whisper_build_graph_conv(*ctx, *state, 0);
                 });
 
         if (!ok) {
```

Code after:
        const int mel_s = i1 - i0;

        assert(mel_inp->type == GGML_TYPE_F32);
        assert(mel_inp->ne[1] == n_mels);

        ggml_tensor * cur = ggml_view_2d(ctx0, mel_inp, out_s, n_mels, mel_inp->nb[1], ggml_row_size(mel_inp->type, i0));

        if (mel_s < out_s) {
            mel = ggml_pad(ctx0, cur, out_s - mel_s, 0, 0, 0);
        }
        else {
            mel = ggml_cont(ctx0, cur);
        }
    }
    else {
        // just create some tensor so that the graph/buffer size estimation is correct
        mel = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 2 * n_ctx, n_mels);
    }
    ggml_set_name(mel, "mel"); // used with external encoding

    struct ggml_tensor * cur = nullptr;

    if (!whisper_encode_external(wstate)) {
        // convolution + gelu
        {
            cur = ggml_conv_1d_ph(ctx0, model.e_conv_1_w, mel, 1, 1);
            cur = ggml_add(ctx0, cur, model.e_conv_1_b);

            cur = ggml_gelu(ctx0, cur);

            cur = ggml_conv_1d_ph(ctx0, model.e_conv_2_w, cur, 2, 1);
            cur = ggml_add(ctx0, cur, model.e_conv_2_b);

            cur = ggml_gelu(ctx0, cur);
        }

        ggml_set_name(cur, "embd_conv");
        wstate.embd_conv = cur;
    } else {
        ggml_build_forward_expand(gf, mel);

        cur = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_state, n_ctx);

        ggml_set_name(cur, "embd_enc");
        wstate.embd_enc = cur;
    }

    ggml_set_output(cur);

    ggml_build_forward_expand(gf, cur);