Update README.md

README.md

@@ -27,10 +27,10 @@ model-index:
     metrics:
        - name: Test WER
          type: wer
-         value: 6.358
+         value: 5.860
        - name: Test CER
          type: cer
-         value: 1.711
+         value: 1.228
   - task:
       name: Speech Recognition
       type: automatic-speech-recognition
@@ -41,10 +41,10 @@ model-index:
     metrics:
        - name: Test WER
          type: wer
-         value: 15.402
+         value: 15.330
        - name: Test CER
          type: cer
-         value: 4.315
+         value: 4.299
 ---
 
 # Wav2Vec2-Large-Ru-Golos
@@ -66,7 +66,7 @@ import torch
 processor = Wav2Vec2Processor.from_pretrained("bond005/wav2vec2-large-ru-golos")
 model = Wav2Vec2ForCTC.from_pretrained("bond005/wav2vec2-large-ru-golos")
      
-# load test part of Golos dataset and read first soundfile
+# load the test part of Golos dataset and read first soundfile
 ds = load_dataset("bond005/sberdevices_golos_10h_crowd", split="test")
  
 # tokenize
@@ -81,6 +81,79 @@ transcription = processor.batch_decode(predicted_ids)[0]
 print(transcription)
 ```
 
+ ## Evaluation
+ 
+This code snippet shows how to evaluate **bond005/wav2vec2-large-ru-golos** on Golos dataset's "crowd" and "farfield" test data.
+ 
+```python
+from datasets import load_dataset
+from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
+import torch
+from jiwer import wer, cer  # we need word error rate (WER) and character error rate (CER)
+
+# load the test part of Golos Crowd and remove samples with empty "true" transcriptions
+golos_crowd_test = load_dataset("bond005/sberdevices_golos_10h_crowd", split="test")
+golos_crowd_test = golos_crowd_test.filter(
+    lambda it1: (it1["transcription"] is not None) and (len(it1["transcription"].strip()) > 0)
+)
+
+# load the test part of Golos Farfield and remove sampels with empty "true" transcriptions
+golos_farfield_test = load_dataset("bond005/sberdevices_golos_100h_farfield", split="test")
+golos_farfield_test = golos_farfield_test.filter(
+    lambda it2: (it2["transcription"] is not None) and (len(it2["transcription"].strip()) > 0)
+)
+
+# load model and tokenizer
+model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h").to("cuda")
+processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h")
+
+# recognize one sound
+def map_to_pred(batch):
+    # tokenize and vectorize
+    processed = processor(
+        batch["audio"]["array"], sampling_rate=batch["audio"]["sampling_rate"],
+        return_tensors="pt", padding="longest"
+    )
+    input_values = processed.input_values.to("cuda")
+    attention_mask = processed.attention_mask.to("cuda")
+
+    # recognize
+    with torch.no_grad():
+        logits = model(input_values, attention_mask=attention_mask).logits
+    predicted_ids = torch.argmax(logits, dim=-1)
+
+    # decode
+    transcription = processor.batch_decode(predicted_ids)
+    batch["text"] = transcription[0]
+    return batch
+
+# calculate WER and CER on the crowd domain
+crowd_result = golos_crowd_test.map(map_to_pred, remove_columns=["audio"])
+crowd_wer = wer(crowd_result["transcription"], crowd_result["text"])
+crowd_cer = cer(crowd_result["transcription"], crowd_result["text"])
+print("Word error rate on the Crowd domain:", crowd_wer)
+print("Character error rate on the Crowd domain:", crowd_cer)
+
+# calculate WER and CER on the farfield domain
+farfield_result = golos_farfield_test.map(map_to_pred, remove_columns=["audio"])
+farfield_wer = wer(farfield_result["transcription"], farfield_result["text"])
+farfield_cer = cer(farfield_result["transcription"], farfield_result["text"])
+print("Word error rate on the Farfield domain:", farfield_wer)
+print("Character error rate on the Farfield domain:", farfield_cer)
+```
+
+*Result (WER, %)*:
+
+| "crowd" | "farfield" |
+|---------|------------|
+|   5.860 |     15.330 |
+
+*Result (CER, %)*:
+
+| "crowd" | "farfield" |
+|---------|------------|
+|   1.228 |      4.299 |
+
 ## Citation
 If you want to cite this model you can use this: