Duplicate from k2-fsa/streaming-automatic-speech-recognition

Co-authored-by: fangjun <csukuangfj@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+flagged/

README.md

@@ -0,0 +1,14 @@
+---
+title: Streaming Automatic Speech Recognition
+emoji: 📚
+colorFrom: pink
+colorTo: green
+sdk: gradio
+sdk_version: 2.9.0
+app_file: app.py
+pinned: false
+license: apache-2.0
+duplicated_from: k2-fsa/streaming-automatic-speech-recognition
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+#
+# Copyright      2022-2023  Xiaomi Corp.        (authors: Fangjun Kuang)
+#
+# See LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# References:
+# https://gradio.app/docs/#dropdown
+
+import logging
+import os
+from typing import List, Optional
+
+import gradio as gr
+import torchaudio
+
+from model import create_recognizer, language_to_models
+
+title = "Next-gen Kaldi: Real-time streaming speech recognition"
+description = """
+This space shows how to do **real-time** streaming speech recognition
+with **Next-gen Kaldi**.
+
+Please visit
+<https://huggingface.co/spaces/k2-fsa/automatic-speech-recognition>
+for non-streaming speech recognition with **Next-gen Kaldi**.
+
+It is running on CPU within a docker container provided by Hugging Face.
+
+**Caution**: You may see **significant delay** since HuggingFace sends
+your recorded data by chunks and the interval between chunks is
+unknown, e.g., may be 2 seconds.
+"""
+
+article = """
+See more information by visiting the following links:
+
+- <https://github.com/k2-fsa/icefall>
+- <https://github.com/k2-fsa/sherpa>
+- <https://github.com/k2-fsa/k2>
+- <https://github.com/lhotse-speech/lhotse>
+
+If you want to deploy it locally, please see
+<https://k2-fsa.github.io/sherpa/>
+
+Usage instructions:
+
+(1) Select a language and a model from the dropdown box
+
+(2) Click the Record button to start
+
+(3) Speak
+
+(4) Click the Stop Recording button to stop
+
+(5) **Remember to click the Clear button before you re-click the Record button**
+
+(6) **Remember to click the Clear button before you re-click the Record button**
+
+(7) **Remember to click the Clear button before you re-click the Record button**
+"""
+
+
+def convert_to_wav(in_filename: str) -> str:
+    """Convert the input audio file to a wave file"""
+    out_filename = in_filename + ".wav"
+    #  logging.info(f"Converting '{in_filename}' to '{out_filename}'")
+    _ = os.system(
+        f"ffmpeg -hide_banner -loglevel error -i '{in_filename}' -ar 16000 '{out_filename}'"
+    )
+
+    return out_filename
+
+
+def get_language_and_model() -> List[str]:
+    """
+    Each entry is of the following format:
+        language | repo_id
+    """
+    ans = []
+    for language, repo_id_list in language_to_models.items():
+        for repo_id in repo_id_list:
+            ans.append(f"{language} | {repo_id}")
+    return ans
+
+
+language_model_list = get_language_and_model()
+
+
+def process(language_and_repo_id: str, audio: Optional[str], state=None):
+    """
+    Args:
+      language_and_repo_id:
+        It contains "language | repo_id"
+      audio:
+        Path to the audio file. Not necessarily in wave format.
+      state:
+        If not None, it contains a list:
+          - error message if any
+          - language_and_repo_id
+          - recognizer
+          - stream
+          - wasOk
+    """
+    language, repo_id = language_and_repo_id.split("|")
+    language = language.strip()
+    repo_id = repo_id.strip()
+
+    if state is None:
+        print("language", language)
+        print("repo_id", repo_id)
+        recognizer = create_recognizer(repo_id)
+        stream = recognizer.create_stream()
+        state = ["", language_and_repo_id, recognizer, stream, True]
+
+    if not state[-1]:
+        return state[0], state
+
+    if audio is None:
+        if "Error" in state[0]:
+            return state[0], state
+        else:
+            recognizer = state[2]
+            stream = state[3]
+            return recognizer.get_result(stream).text.lower()
+
+    if state[1] != language_and_repo_id:
+        state[0] = (
+            "Error: Please don't change the language and model during recognition "
+            + "or "
+            + "please press the Clear button before you re-click Record or re-select "
+            + "language and model.\n\n\n"
+            + "Hint: Click Stop Recording and then press Clear to fix this error."
+        )
+        state[-1] = False
+
+        return state[0], state
+
+    filename = convert_to_wav(audio)
+
+    samples, sample_rate = torchaudio.load(filename)
+    assert sample_rate == 16000, (sample_rate, 16000)
+    samples = samples.squeeze(0)
+    duration = samples.numel() / 16000
+    #  logging.info(f"duration: {duration} s")
+
+    recognizer = state[2]
+    stream = state[3]
+    stream.accept_waveform(16000, samples)
+
+    while recognizer.is_ready(stream):
+        recognizer.decode_stream(stream)
+
+    text = recognizer.get_result(stream).text.lower()
+    logging.info(text)
+
+    return text, state
+
+
+language_dropdown = gr.inputs.Dropdown(
+    label="Select a language and a model",
+    choices=language_model_list,
+    default=language_model_list[0],
+)
+
+itf1 = gr.Interface(
+    title=title,
+    description=description,
+    article=article,
+    fn=process,
+    inputs=[
+        language_dropdown,
+        gr.inputs.Audio(
+            source="microphone",
+            type="filepath",
+            label="Press me to start recognition",
+        ),
+        "state",
+    ],
+    outputs=[
+        gr.outputs.Textbox(type="str", label="result"),
+        gr.outputs.State(label=""),
+    ],
+    live=True,
+)
+
+
+if __name__ == "__main__":
+    formatter = "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s"
+
+    logging.basicConfig(format=formatter, level=logging.INFO)
+
+    itf1.launch()

model.py

@@ -0,0 +1,162 @@
+# Copyright      2022-2023  Xiaomi Corp.        (authors: Fangjun Kuang)
+#
+# See LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+
+from huggingface_hub import hf_hub_download
+
+os.system(
+    "cp -v /home/user/.local/lib/python3.8/site-packages/k2/lib/*.so /home/user/.local/lib/python3.8/site-packages/sherpa/lib/"
+)
+
+import sherpa  # noqa
+
+
+def _get_nn_model_filename(
+    repo_id: str,
+    filename: str,
+    subfolder: str = "exp",
+) -> str:
+    nn_model_filename = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        subfolder=subfolder,
+    )
+    return nn_model_filename
+
+
+def _get_token_filename(
+    repo_id: str,
+    filename: str = "tokens.txt",
+    subfolder: str = "data/lang_char",
+) -> str:
+    token_filename = hf_hub_download(
+        repo_id=repo_id,
+        filename=filename,
+        subfolder=subfolder,
+    )
+    return token_filename
+
+
+def get_english_model_2022_12_19(repo_id: str):
+    encoder = _get_nn_model_filename(repo_id=repo_id, filename="encoder_jit_trace.pt")
+    decoder = _get_nn_model_filename(repo_id=repo_id, filename="decoder_jit_trace.pt")
+    joiner = _get_nn_model_filename(repo_id=repo_id, filename="joiner_jit_trace.pt")
+    tokens = _get_token_filename(repo_id=repo_id, subfolder="data/lang_bpe_500")
+
+    feat_config = sherpa.FeatureConfig()
+    feat_config.fbank_opts.frame_opts.samp_freq = 16000
+    feat_config.fbank_opts.mel_opts.num_bins = 80
+    feat_config.fbank_opts.frame_opts.dither = 0
+
+    config = sherpa.OnlineRecognizerConfig(
+        nn_model="",
+        encoder_model=encoder,
+        decoder_model=decoder,
+        joiner_model=joiner,
+        tokens=tokens,
+        use_gpu=False,
+        feat_config=feat_config,
+        decoding_method="greedy_search",
+        chunk_size=32,
+    )
+
+    recognizer = sherpa.OnlineRecognizer(config)
+    return recognizer
+
+
+def get_chinese_english_mixed_model_conv_emformer_transducer_stateless2_zh(
+    repo_id: str,
+) -> sherpa.OnlineRecognizer:
+    nn_model = _get_nn_model_filename(
+        repo_id=repo_id, filename="cpu_jit-epoch-11-avg-1.pt"
+    )
+    tokens = _get_token_filename(repo_id=repo_id, subfolder="data/lang_char_bpe")
+
+    feat_config = sherpa.FeatureConfig()
+    feat_config.fbank_opts.frame_opts.samp_freq = 16000
+    feat_config.fbank_opts.mel_opts.num_bins = 80
+    feat_config.fbank_opts.frame_opts.dither = 0
+
+    config = sherpa.OnlineRecognizerConfig(
+        nn_model=nn_model,
+        tokens=tokens,
+        use_gpu=False,
+        feat_config=feat_config,
+        decoding_method="greedy_search",
+    )
+    recognizer = sherpa.OnlineRecognizer(config)
+    return recognizer
+
+
+def get_chinese_english_mixed_model_k2fsa_zipformer_chinese_english_mixed(
+    repo_id: str,
+) -> sherpa.OnlineRecognizer:
+    encoder = _get_nn_model_filename(repo_id=repo_id, filename="encoder_jit_trace.pt")
+    decoder = _get_nn_model_filename(repo_id=repo_id, filename="decoder_jit_trace.pt")
+    joiner = _get_nn_model_filename(repo_id=repo_id, filename="joiner_jit_trace.pt")
+    tokens = _get_token_filename(repo_id=repo_id, subfolder="data/lang_char_bpe")
+
+    feat_config = sherpa.FeatureConfig()
+    feat_config.fbank_opts.frame_opts.samp_freq = 16000
+    feat_config.fbank_opts.mel_opts.num_bins = 80
+    feat_config.fbank_opts.frame_opts.dither = 0
+
+    config = sherpa.OnlineRecognizerConfig(
+        nn_model="",
+        encoder_model=encoder,
+        decoder_model=decoder,
+        joiner_model=joiner,
+        tokens=tokens,
+        use_gpu=False,
+        feat_config=feat_config,
+        decoding_method="greedy_search",
+        chunk_size=32,
+    )
+
+    recognizer = sherpa.OnlineRecognizer(config)
+    return recognizer
+
+
+def create_recognizer(repo_id: str) -> sherpa.OnlineRecognizer:
+    if repo_id in english_models:
+        return english_models[repo_id](repo_id)
+    elif repo_id in chinese_english_mixed_models:
+        return chinese_english_mixed_models[repo_id](repo_id)
+    else:
+        raise ValueError(f"Unsupported repo_id: {repo_id}")
+
+
+english_models = {
+    # https://huggingface.co/Zengwei/icefall-asr-librispeech-pruned-transducer-stateless7-streaming-2022-12-29
+    "Zengwei/icefall-asr-librispeech-pruned-transducer-stateless7-streaming-2022-12-29": get_english_model_2022_12_19
+}
+
+chinese_english_mixed_models = {
+    # https://huggingface.co/pfluo/k2fsa-zipformer-chinese-english-mixed
+    "pfluo/k2fsa-zipformer-chinese-english-mixed": get_chinese_english_mixed_model_k2fsa_zipformer_chinese_english_mixed,
+    # https://huggingface.co/ptrnull/icefall-asr-conv-emformer-transducer-stateless2-zh
+    "ptrnull/icefall-asr-conv-emformer-transducer-stateless2-zh": get_chinese_english_mixed_model_conv_emformer_transducer_stateless2_zh,
+}
+
+all_models = {
+    **english_models,
+    **chinese_english_mixed_models,
+}
+
+language_to_models = {
+    "English": list(english_models.keys()),
+    "Chinese+English": list(chinese_english_mixed_models.keys()),
+}

requirements.txt

@@ -0,0 +1,11 @@
+https://download.pytorch.org/whl/cpu/torch-1.13.1%2Bcpu-cp38-cp38-linux_x86_64.whl
+https://download.pytorch.org/whl/cpu/torchaudio-0.13.1%2Bcpu-cp38-cp38-linux_x86_64.whl
+
+https://huggingface.co/csukuangfj/wheels/resolve/main/2023-01-30/k2-1.23.4.dev20230130%2Bcpu.torch1.13.1-cp38-cp38-linux_x86_64.whl
+https://huggingface.co/csukuangfj/wheels/resolve/main/2023-01-30/k2_sherpa-1.1-cp38-cp38-linux_x86_64.whl
+https://huggingface.co/csukuangfj/wheels/resolve/main/2023-01-30/kaldifeat-1.22-cp38-cp38-linux_x86_64.whl
+
+sentencepiece>=0.1.96
+numpy
+
+huggingface_hub