Initial commit

.gitignore

@@ -0,0 +1,6 @@
+checkpoint-*/
+*results.json
+training_args.bin
+trainer_state.json
+normalizer.json
+wer_summary/

README.md

@@ -0,0 +1,129 @@
+---
+license: apache-2.0
+language: de
+library_name: transformers
+thumbnail: null
+tags:
+- automatic-speech-recognition
+- whisper-event
+datasets:
+- mozilla-foundation/common_voice_11_0
+metrics:
+- wer
+model-index:
+- name: Fine-tuned whisper-large-v2 model for ASR in German
+  results:
+  - task:
+      name: Automatic Speech Recognition
+      type: automatic-speech-recognition
+    dataset:
+      name: Common Voice 11.0
+      type: mozilla-foundation/common_voice_11_0
+      config: de
+      split: test
+      args: de
+    metrics:
+    - name: WER (Greedy)
+      type: wer
+      value: 5.76
+---
+
+<style>
+img {
+ display: inline;
+}
+</style>
+
+![Model architecture](https://img.shields.io/badge/Model_Architecture-seq2seq-lightgrey)
+![Model size](https://img.shields.io/badge/Params-1550M-lightgrey)
+![Language](https://img.shields.io/badge/Language-German-lightgrey)
+
+# Fine-tuned whisper-large-v2 model for ASR in German
+
+This model is a fine-tuned version of [openai/whisper-large-v2](https://huggingface.co/openai/whisper-large-v2), trained on the mozilla-foundation/common_voice_11_0 de dataset. When using the model make sure that your speech input is also sampled at 16Khz. **This model also predicts casing and punctuation.**
+
+## Usage
+
+Inference with 🤗 Pipeline
+
+```python
+import torch
+
+from datasets import load_dataset
+from transformers import pipeline
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# Load pipeline
+pipe = pipeline("automatic-speech-recognition", model="bofenghuang/whisper-large-v2-cv11-german-punct", device=device)
+
+# NB: set forced_decoder_ids for generation utils
+pipe.model.config.forced_decoder_ids = pipe.tokenizer.get_decoder_prompt_ids(language="de", task="transcribe")
+
+# Load data
+ds_mcv_test = load_dataset("mozilla-foundation/common_voice_11_0", "de", split="test", streaming=True)
+test_segment = next(iter(ds_mcv_test))
+waveform = test_segment["audio"]
+
+# NB: decoding option
+# limit the maximum number of generated tokens to 225
+pipe.model.config.max_length = 225 + 1
+# sampling
+# pipe.model.config.do_sample = True
+# beam search
+# pipe.model.config.num_beams = 5
+# return
+# pipe.model.config.return_dict_in_generate = True
+# pipe.model.config.output_scores = True
+# pipe.model.config.num_return_sequences = 5
+
+# Run
+generated_sentences = pipe(waveform)["text"]
+```
+
+Inference with 🤗 low-level APIs
+
+```python
+import torch
+import torchaudio
+
+from datasets import load_dataset
+from transformers import AutoProcessor, AutoModelForSpeechSeq2Seq
+
+device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
+
+# Load model
+model = AutoModelForSpeechSeq2Seq.from_pretrained("bofenghuang/whisper-large-v2-cv11-german-punct").to(device)
+processor = AutoProcessor.from_pretrained("bofenghuang/whisper-large-v2-cv11-german-punct", language="german", task="transcribe")
+
+# NB: set forced_decoder_ids for generation utils
+model.config.forced_decoder_ids = processor.get_decoder_prompt_ids(language="de", task="transcribe")
+
+# 16_000
+model_sample_rate = processor.feature_extractor.sampling_rate
+
+# Load data
+ds_mcv_test = load_dataset("mozilla-foundation/common_voice_11_0", "de", split="test", streaming=True)
+test_segment = next(iter(ds_mcv_test))
+waveform = torch.from_numpy(test_segment["audio"]["array"])
+sample_rate = test_segment["audio"]["sampling_rate"]
+
+# Resample
+if sample_rate != model_sample_rate:
+    resampler = torchaudio.transforms.Resample(sample_rate, model_sample_rate)
+    waveform = resampler(waveform)
+
+# Get feat
+inputs = processor(waveform, sampling_rate=model_sample_rate, return_tensors="pt")
+input_features = inputs.input_features
+input_features = input_features.to(device)
+
+# Generate
+generated_ids = model.generate(inputs=input_features, max_new_tokens=225)  # greedy
+# generated_ids = model.generate(inputs=input_features, max_new_tokens=225, num_beams=5)  # beam search
+
+# Detokenize
+generated_sentences = processor.batch_decode(generated_ids, skip_special_tokens=True)[0]
+
+# Normalise predicted sentences if necessary
+```

added_tokens.json

@@ -0,0 +1,109 @@
+{
+  "<|af|>": 50327,
+  "<|am|>": 50334,
+  "<|ar|>": 50272,
+  "<|as|>": 50350,
+  "<|az|>": 50304,
+  "<|ba|>": 50355,
+  "<|be|>": 50330,
+  "<|bg|>": 50292,
+  "<|bn|>": 50302,
+  "<|bo|>": 50347,
+  "<|br|>": 50309,
+  "<|bs|>": 50315,
+  "<|ca|>": 50270,
+  "<|cs|>": 50283,
+  "<|cy|>": 50297,
+  "<|da|>": 50285,
+  "<|de|>": 50261,
+  "<|el|>": 50281,
+  "<|endoftext|>": 50257,
+  "<|en|>": 50259,
+  "<|es|>": 50262,
+  "<|et|>": 50307,
+  "<|eu|>": 50310,
+  "<|fa|>": 50300,
+  "<|fi|>": 50277,
+  "<|fo|>": 50338,
+  "<|fr|>": 50265,
+  "<|gl|>": 50319,
+  "<|gu|>": 50333,
+  "<|haw|>": 50352,
+  "<|ha|>": 50354,
+  "<|hi|>": 50276,
+  "<|hr|>": 50291,
+  "<|ht|>": 50339,
+  "<|hu|>": 50286,
+  "<|hy|>": 50312,
+  "<|id|>": 50275,
+  "<|is|>": 50311,
+  "<|it|>": 50274,
+  "<|iw|>": 50279,
+  "<|ja|>": 50266,
+  "<|jw|>": 50356,
+  "<|ka|>": 50329,
+  "<|kk|>": 50316,
+  "<|km|>": 50323,
+  "<|kn|>": 50306,
+  "<|ko|>": 50264,
+  "<|la|>": 50294,
+  "<|lb|>": 50345,
+  "<|ln|>": 50353,
+  "<|lo|>": 50336,
+  "<|lt|>": 50293,
+  "<|lv|>": 50301,
+  "<|mg|>": 50349,
+  "<|mi|>": 50295,
+  "<|mk|>": 50308,
+  "<|ml|>": 50296,
+  "<|mn|>": 50314,
+  "<|mr|>": 50320,
+  "<|ms|>": 50282,
+  "<|mt|>": 50343,
+  "<|my|>": 50346,
+  "<|ne|>": 50313,
+  "<|nl|>": 50271,
+  "<|nn|>": 50342,
+  "<|nocaptions|>": 50362,
+  "<|notimestamps|>": 50363,
+  "<|no|>": 50288,
+  "<|oc|>": 50328,
+  "<|pa|>": 50321,
+  "<|pl|>": 50269,
+  "<|ps|>": 50340,
+  "<|pt|>": 50267,
+  "<|ro|>": 50284,
+  "<|ru|>": 50263,
+  "<|sa|>": 50344,
+  "<|sd|>": 50332,
+  "<|si|>": 50322,
+  "<|sk|>": 50298,
+  "<|sl|>": 50305,
+  "<|sn|>": 50324,
+  "<|so|>": 50326,
+  "<|sq|>": 50317,
+  "<|sr|>": 50303,
+  "<|startoflm|>": 50360,
+  "<|startofprev|>": 50361,
+  "<|startoftranscript|>": 50258,
+  "<|su|>": 50357,
+  "<|sv|>": 50273,
+  "<|sw|>": 50318,
+  "<|ta|>": 50287,
+  "<|te|>": 50299,
+  "<|tg|>": 50331,
+  "<|th|>": 50289,
+  "<|tk|>": 50341,
+  "<|tl|>": 50348,
+  "<|transcribe|>": 50359,
+  "<|translate|>": 50358,
+  "<|tr|>": 50268,
+  "<|tt|>": 50351,
+  "<|uk|>": 50280,
+  "<|ur|>": 50290,
+  "<|uz|>": 50337,
+  "<|vi|>": 50278,
+  "<|yi|>": 50335,
+  "<|yo|>": 50325,
+  "<|zh|>": 50260
+}

config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "openai/whisper-large-v2",
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "architectures": [
+    "WhisperForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "begin_suppress_tokens": [
+    220,
+    50257
+  ],
+  "bos_token_id": 50257,
+  "d_model": 1280,
+  "decoder_attention_heads": 20,
+  "decoder_ffn_dim": 5120,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 32,
+  "decoder_start_token_id": 50258,
+  "dropout": 0.05,
+  "encoder_attention_heads": 20,
+  "encoder_ffn_dim": 5120,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 32,
+  "eos_token_id": 50257,
+  "forced_decoder_ids": null,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "max_length": 448,
+  "max_source_positions": 1500,
+  "max_target_positions": 448,
+  "model_type": "whisper",
+  "num_hidden_layers": 32,
+  "num_mel_bins": 80,
+  "pad_token_id": 50257,
+  "scale_embedding": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.26.0.dev0",
+  "use_cache": false,
+  "vocab_size": 51865
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e10d90936368fbc11d8e23cbed4ed795129ec728f911b2579e27f4af58c51c85
+size 3086784957

results_cv11_greedy/eval_results.txt

@@ -0,0 +1,2 @@
+WER: 0.05761999110428208
+CER: 0.01686471998368856

runs/Dec15_21-21-27_129-146-0-103/1671139337.3378797/events.out.tfevents.1671139337.129-146-0-103.1994026.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff71dadfdb1627b162c0835ad2b02ddcb9793c4914e4411786b70000f7c50ebe
+size 6127

runs/Dec15_21-21-27_129-146-0-103/events.out.tfevents.1671139337.129-146-0-103.1994026.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0417271c6cc163963515a024ecb3d1f1400e462620fe8855e84b396dc540b44f
+size 25611

runs/Dec15_21-21-27_129-146-0-103/events.out.tfevents.1671309779.129-146-0-103.1994026.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8937fc2447265df86427ba5688d2ac1923ca3d124a2b81f4940857d80dda6809
+size 358

special_tokens_map.json

@@ -0,0 +1,133 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|startoftranscript|>",
+    "<|en|>",
+    "<|zh|>",
+    "<|de|>",
+    "<|es|>",
+    "<|ru|>",
+    "<|ko|>",
+    "<|fr|>",
+    "<|ja|>",
+    "<|pt|>",
+    "<|tr|>",
+    "<|pl|>",
+    "<|ca|>",
+    "<|nl|>",
+    "<|ar|>",
+    "<|sv|>",
+    "<|it|>",
+    "<|id|>",
+    "<|hi|>",
+    "<|fi|>",
+    "<|vi|>",
+    "<|iw|>",
+    "<|uk|>",
+    "<|el|>",
+    "<|ms|>",
+    "<|cs|>",
+    "<|ro|>",
+    "<|da|>",
+    "<|hu|>",
+    "<|ta|>",
+    "<|no|>",
+    "<|th|>",
+    "<|ur|>",
+    "<|hr|>",
+    "<|bg|>",
+    "<|lt|>",
+    "<|la|>",
+    "<|mi|>",
+    "<|ml|>",
+    "<|cy|>",
+    "<|sk|>",
+    "<|te|>",
+    "<|fa|>",
+    "<|lv|>",
+    "<|bn|>",
+    "<|sr|>",
+    "<|az|>",
+    "<|sl|>",
+    "<|kn|>",
+    "<|et|>",
+    "<|mk|>",
+    "<|br|>",
+    "<|eu|>",
+    "<|is|>",
+    "<|hy|>",
+    "<|ne|>",
+    "<|mn|>",
+    "<|bs|>",
+    "<|kk|>",
+    "<|sq|>",
+    "<|sw|>",
+    "<|gl|>",
+    "<|mr|>",
+    "<|pa|>",
+    "<|si|>",
+    "<|km|>",
+    "<|sn|>",
+    "<|yo|>",
+    "<|so|>",
+    "<|af|>",
+    "<|oc|>",
+    "<|ka|>",
+    "<|be|>",
+    "<|tg|>",
+    "<|sd|>",
+    "<|gu|>",
+    "<|am|>",
+    "<|yi|>",
+    "<|lo|>",
+    "<|uz|>",
+    "<|fo|>",
+    "<|ht|>",
+    "<|ps|>",
+    "<|tk|>",
+    "<|nn|>",
+    "<|mt|>",
+    "<|sa|>",
+    "<|lb|>",
+    "<|my|>",
+    "<|bo|>",
+    "<|tl|>",
+    "<|mg|>",
+    "<|as|>",
+    "<|tt|>",
+    "<|haw|>",
+    "<|ln|>",
+    "<|ha|>",
+    "<|ba|>",
+    "<|jw|>",
+    "<|su|>",
+    "<|translate|>",
+    "<|transcribe|>",
+    "<|startoflm|>",
+    "<|startofprev|>",
+    "<|nocaptions|>",
+    "<|notimestamps|>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "<|endoftext|>",
+  "unk_token": {
+    "content": "",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,36 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "bos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "__type": "AddedToken",
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "errors": "replace",
+  "model_max_length": 1024,
+  "name_or_path": "openai/whisper-large-v2",
+  "pad_token": null,
+  "processor_class": "WhisperProcessor",
+  "return_attention_mask": false,
+  "special_tokens_map_file": null,
+  "tokenizer_class": "WhisperTokenizer",
+  "unk_token": {
+    "__type": "AddedToken",
+    "content": "",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}