Repository: ggerganov/whisper.cpp
File: src/whisper.cpp (C++)

Code before:

                            // Check if first segment has skipped audio (multiple timestamp tokens after [_BEG_]) by
                            // peeking at a few of the tokens following, and see if their timestamps are increasing.
                            // If this is the case we adjust the first segment's start timestamp (t0) to account for
                            // the skipped audio.
                            //
                            // Note that this will only happen for models that have been trained to skip
                            // introductions/credits like the medium models for example.
                            const int n_check_tokens = 6;
                            const int thold_intro_detect = 300;
                            auto & segment = result_all[0];
                            auto & tokens  = segment.tokens;
                            if (segment.t0 == 0 && tokens.size() > n_check_tokens) {
                                int sum = 0;
                                for (int i = 0; i < n_check_tokens; ++i) {
                                    sum += tokens[i].tid - whisper_token_beg(ctx);
                                }
                                // If the total sum of the first 6 timestamp tokens is larger than 300
                                // centiseconds (3 seconds) then we take it to mean that there is an
                                // introduction/credits that the model skipped.
                                if (sum > thold_intro_detect) {
                                    segment.t0 = state->t_beg + 2*(tokens[2].tid - whisper_token_beg(ctx));
                                    WHISPER_LOG_INFO("%s: detected skipped audio sequence, setting segment.t0 to %d\n", __func__, (int)segment.t0);
                                }
                            }

                            int n_new = 1;

                            if (params.token_timestamps) {
                                whisper_exp_compute_token_level_timestamps(
                                        *ctx, *state, result_all.size() - 1, params.thold_pt, params.thold_ptsum);

                                if (params.max_len > 0) {
                                    n_new = whisper_wrap_segment(*ctx, *state, params.max_len, params.split_on_word);
                                }
                            }
                            if (params.new_segment_callback && !ctx->params.dtw_token_timestamps) {
                                params.new_segment_callback(ctx, state, n_new, params.new_segment_callback_user_data);
                            }
                        }
                        text = "";
                        while (i < (int) tokens_cur.size() && tokens_cur[i].id > whisper_token_beg(ctx)) {
                            i++;
                        }
                        i--;
                        t0 = t1;
                        i0 = i + 1;
                        speaker_turn_next = false;
                    }
                }


Review comment (suggestion):
I think this fix is not necessary. I have doubts that this will work in all cases and also skipping intro of trained audio is not a big problem because we are mainly interested on applying the model on new audio. So let's remove it for now and merge only the timestamp fixes.

Code after:

                            int n_new = 1;

                            if (params.token_timestamps) {
                                whisper_exp_compute_token_level_timestamps(
                                        *ctx, *state, result_all.size() - 1, params.thold_pt, params.thold_ptsum);

                                if (params.max_len > 0) {
                                    n_new = whisper_wrap_segment(*ctx, *state, params.max_len, params.split_on_word);
                                }
                            }
                            if (params.new_segment_callback && !ctx->params.dtw_token_timestamps) {
                                params.new_segment_callback(ctx, state, n_new, params.new_segment_callback_user_data);
                            }
                        }
                        text = "";
                        while (i < (int) tokens_cur.size() && tokens_cur[i].id > whisper_token_beg(ctx)) {
                            i++;
                        }
                        i--;
                        t0 = t1;
                        i0 = i + 1;
                        speaker_turn_next = false;
                    }
                }

                if (!text.empty()) {
                    const auto t1 = seek + seek_delta;

                    const auto tt0 = t0;
                    const auto tt1 = t1;

                    if (params.print_realtime) {
                        if (params.print_timestamps) {
                            printf("[%s --> %s]  %s\n", to_timestamp(tt0).c_str(), to_timestamp(tt1).c_str(), text.c_str());
                        } else {
                            printf("%s", text.c_str());
                            fflush(stdout);
                        }
                    }

                    result_all.push_back({ tt0, tt1, text, state->no_speech_prob, {}, speaker_turn_next });
                    for (int j = i0; j < (int) tokens_cur.size(); j++) {
                        result_all.back().tokens.push_back(tokens_cur[j]);
                    }

                    int n_new = 1;

                    if (params.token_timestamps) {
                        whisper_exp_compute_token_level_timestamps(
                                *ctx, *state, result_all.size() - 1, params.thold_pt, params.thold_ptsum);