|
--- |
|
language: de |
|
datasets: |
|
- common_voice |
|
metrics: |
|
- wer |
|
- cer |
|
tags: |
|
- audio |
|
- automatic-speech-recognition |
|
- speech |
|
- hf-asr-leaderboard |
|
license: apache-2.0 |
|
model-index: |
|
- name: wav2vec2-xls-r-1b-5gram-german with LM by Florian Zimmermeister @A\\Ware |
|
results: |
|
- task: |
|
name: Speech Recognition |
|
type: automatic-speech-recognition |
|
dataset: |
|
name: Common Voice de |
|
type: common_voice |
|
args: de |
|
metrics: |
|
- name: Test WER |
|
type: wer |
|
value: 4.382541642219636 |
|
- name: Test CER |
|
type: cer |
|
value: 1.6235493024026488 |
|
- task: |
|
name: Speech Recognition |
|
type: automatic-speech-recognition |
|
dataset: |
|
name: Common Voice 8 de |
|
type: mozilla-foundation/common_voice_8_0 |
|
args: de |
|
metrics: |
|
- name: Test WER |
|
type: wer |
|
value: 4.382541642219636 |
|
- name: Test CER |
|
type: cer |
|
value: 1.6235493024026488 |
|
--- |
|
|
|
## Evaluation |
|
The model can be evaluated as follows on the German test data of Common Voice. |
|
|
|
```python |
|
import torch |
|
from transformers import AutoModelForCTC, AutoProcessor |
|
from unidecode import unidecode |
|
import re |
|
from datasets import load_dataset, load_metric |
|
import datasets |
|
|
|
counter = 0 |
|
wer_counter = 0 |
|
cer_counter = 0 |
|
device = "cuda" if torch.cuda.is_available() else "cpu" |
|
|
|
|
|
special_chars = [["Ä"," AE "], ["Ö"," OE "], ["Ü"," UE "], ["ä"," ae "], ["ö"," oe "], ["ü"," ue "]] |
|
def clean_text(sentence): |
|
for special in special_chars: |
|
sentence = sentence.replace(special[0], special[1]) |
|
|
|
sentence = unidecode(sentence) |
|
|
|
for special in special_chars: |
|
sentence = sentence.replace(special[1], special[0]) |
|
|
|
sentence = re.sub("[^a-zA-Z0-9öäüÖÄÜ ,.!?]", " ", sentence) |
|
|
|
return sentence |
|
|
|
def main(model_id): |
|
print("load model") |
|
model = AutoModelForCTC.from_pretrained(model_id).to(device) |
|
print("load processor") |
|
processor = AutoProcessor.from_pretrained(processor_id) |
|
|
|
print("load metrics") |
|
wer = load_metric("wer") |
|
cer = load_metric("cer") |
|
|
|
ds = load_dataset("mozilla-foundation/common_voice_8_0","de") |
|
ds = ds["test"] |
|
|
|
ds = ds.cast_column( |
|
"audio", datasets.features.Audio(sampling_rate=16_000) |
|
) |
|
|
|
def calculate_metrics(batch): |
|
global counter, wer_counter, cer_counter |
|
resampled_audio = batch["audio"]["array"] |
|
|
|
input_values = processor(resampled_audio, return_tensors="pt", sampling_rate=16_000).input_values |
|
|
|
with torch.no_grad(): |
|
logits = model(input_values.to(device)).logits.cpu().numpy()[0] |
|
|
|
|
|
decoded = processor.decode(logits) |
|
pred = decoded.text.lower() |
|
|
|
ref = clean_text(batch["sentence"]).lower() |
|
|
|
wer_result = wer.compute(predictions=[pred], references=[ref]) |
|
cer_result = cer.compute(predictions=[pred], references=[ref]) |
|
|
|
counter += 1 |
|
wer_counter += wer_result |
|
cer_counter += cer_result |
|
|
|
if counter % 100 == True: |
|
print(f"WER: {(wer_counter/counter)*100} | CER: {(cer_counter/counter)*100}") |
|
|
|
return batch |
|
|
|
|
|
ds.map(calculate_metrics, remove_columns=ds.column_names) |
|
print(f"WER: {(wer_counter/counter)*100} | CER: {(cer_counter/counter)*100}") |
|
|
|
model_id = "flozi00/wav2vec2-xls-r-1b-5gram-german" |
|
main(model_id) |
|
``` |
