add conformer

.gitattributes

@@ -33,5 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
-conformer.png filter=lfs diff=lfs merge=lfs -text
-whisper.png filter=lfs diff=lfs merge=lfs -text
+conformer.png filter=lfs diff=lfs merge=lfs -text
+whisper.png filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,12 +1,76 @@
+import os 
+import torch
+import torch.nn as nn
+import torchaudio
 import numpy as np  # type: ignore
 import gradio as gr  # type: ignore
 from transformers import pipeline
+from huggingface_hub import hf_hub_download
+from torchaudio.models import Conformer
+
+class ASRConformerModel(nn.Module):
+    def __init__(self, input_dim, vocab_size):
+        super().__init__()
+        self.encoder = Conformer(
+            input_dim=input_dim,
+            num_heads=4,
+            ffn_dim=512,
+            num_layers=4,
+            depthwise_conv_kernel_size=31,
+            dropout=0.1
+        )
+        self.classifier = nn.Linear(input_dim, vocab_size)
+
+    def forward(self, x, lengths):
+        x, lengths = self.encoder(x, lengths=lengths)
+        x = self.classifier(x)
+        return x, lengths
+
+
+VOCAB = set("abcdefghijklmnopqrstuvwxyz '")
+char_to_idx = {ch: i + 1 for i, ch in enumerate(sorted(VOCAB))}  # 0 for CTC blank
+
+def greedy_decode(log_probs, blank=0):
+    pred_ids = log_probs.argmax(dim=-1)  # [T, B]
+    pred_ids = pred_ids.transpose(0, 1)  # [B, T]
+    predictions = []
+    for seq in pred_ids:
+        prev = blank
+        pred = []
+        for i in seq:
+            if i != prev and i != blank:
+                pred.append(i.item())
+            prev = i
+        predictions.append(pred)
+    return predictions
+
+def encode(text):
+    return torch.tensor([char_to_idx[c] for c in text.lower() if c in char_to_idx], dtype=torch.long)
+
+def decode_to_text(predictions, idx_to_char):
+    return [''.join(idx_to_char[i] for i in pred if i in idx_to_char) for pred in predictions]
 
 # Load fine-tuned Whisper model
-transcriber = pipeline("automatic-speech-recognition", model="OwLim/whisper-java-SLR41-SLR35")
+transcriber_whisper = pipeline("automatic-speech-recognition", model="OwLim/whisper-sundanese-finetune")
+transcriber_wav2vec = pipeline("automatic-speech-recognition", model="indonesian-nlp/wav2vec2-indonesian-javanese-sundanese")
+
+
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+SAMPLE_RATE = 16_000
+
+model_path = hf_hub_download(repo_id="Blebbyblub/javanese-conformer-asrV2", filename="pytorch_model.bin")
+model = ASRConformerModel(input_dim=80, vocab_size=29).to(device)
+model.load_state_dict(torch.load(model_path, map_location=device))
+
+examples_audio = [
+    file for file in os.listdir("./") if file.endswith(".wav")
+]
+
+idx_to_char = {v: k for k, v in char_to_idx.items()}
 
-def transcribe(audio):
+def transcribe(audio, model_selection):
     sr, waveform = audio
+
     # Change into Mono Audio
     if waveform.ndim > 1:
         waveform = waveform.mean(axis=1)
@@ -15,7 +79,33 @@ def transcribe(audio):
     waveform = waveform.astype(np.float32)
     waveform /= np.max(np.abs(waveform))
 
-    return transcriber({
+    if "Conformer" == model_selection :
+        mel_transform = torchaudio.transforms.MelSpectrogram(sample_rate=SAMPLE_RATE, n_mels=80)
+
+        waveform = torch.from_numpy(waveform).float()
+        if sr != SAMPLE_RATE:
+            waveform = torchaudio.transforms.Resample(sr, SAMPLE_RATE)(waveform)
+
+        waveform = waveform.unsqueeze(0)
+        mel = mel_transform(waveform).squeeze(0).transpose(0, 1)  # [time, mel]
+        mel = mel.unsqueeze(0).to(device)
+        input_length = torch.tensor([mel.size(1)]).to(device)
+
+        model.eval()
+        with torch.no_grad():
+            output, output_lengths = model(mel, input_length)
+            log_probs = output.log_softmax(2).transpose(0, 1)
+            pred_ids = greedy_decode(log_probs)
+            pred_text = decode_to_text(pred_ids, idx_to_char)[0]
+
+        return pred_text
+
+    if "Wav2Vec" == model_selection :
+        selected_model = transcriber_wav2vec
+    elif "Whisper" == model_selection:
+        selected_model = transcriber_whisper
+
+    return selected_model({
         "sampling_rate" : sr,
         "raw" : waveform
     })["text"]
@@ -25,6 +115,7 @@ def clear():
 
 # --- Tab 1: Transcribe ---
 with gr.Blocks() as tab_transcribe:
+    model_selector = gr.Radio(choices=["Whisper", "Conformer", "Wav2Vec"], label="Choose Model", info="This will effect the model that you use for transcribing", )    
     with gr.Row():
         with gr.Column(scale=1):
             audio_input = gr.Audio(sources="microphone", label="Record Your Voice")
@@ -35,7 +126,13 @@ with gr.Blocks() as tab_transcribe:
         with gr.Column(scale=1):
             output_text = gr.Textbox(label="Transcription", placeholder="Waiting for Input", lines=3)
 
-    subBtn.click(fn=transcribe, inputs=audio_input, outputs=output_text)
+    gr.Examples(
+        examples=examples_audio,  # List of audio file paths
+        inputs=audio_input,
+        label="Try with Example Audio"
+    )
+
+    subBtn.click(fn=transcribe, inputs=[audio_input, model_selector], outputs=output_text)
     clrBtn.click(fn=clear, outputs=[audio_input, output_text])
 
 # --- Tab 2: Penjelasan Model Fine-Tuned ---
@@ -52,19 +149,20 @@ with gr.Blocks() as tab_background:
         Model yang telah kami fine tune merupakan hasil <b>fine-tuning dari Whisper dan Conformer</b> untuk mendukung bahasa lokal di Indonesia, khususnya bahasa Jawa dan Sunda.
         Model dilatih menggunakan kombinasi dataset <b>OpenSLR</b> berikut:
             <br>
-            <a href="https://openslr.org/35/" target="_blank" style="text-decoration:none;> 
-                    <b>SLR35</b> - Large Javanese ASR
-            </a>
-
+                <a href="https://openslr.org/35/" target="_blank" style="text-decoration:none;">             
+                        <b>SLR35</b> - Large Javanese ASR training data set
+                </a>        
             <br>
-            <a href="https://openslr.org/41/" target="_blank" style="text-decoration:none;">             
-                    <b>SLR41</b> - High quality TTS data for Javanese
-            </a>        
-
+                <a href="https://openslr.org/36/" target="_blank" style="text-decoration:none;">             
+                        <b>SLR36</b> - Large Sundanese ASR training data set
+                </a>        
+            <br>
+                <a href="https://openslr.org/41/" target="_blank" style="text-decoration:none;">             
+                        <b>SLR41</b> - High quality TTS data for Javanese
+                </a>        
             <br>
-            <a href="https://openslr.org/36" target="_blank" style="text-decoration:none;">             
-                    <b>SLR36</b>    
-                    <b>SLR44</b> - Bilingual speech datasets
+            <a href="https://openslr.org/44" target="_blank" style="text-decoration:none;">                 
+                    <b>SLR44</b> - High quality TTS data for Sundanese.
             </a>
         
                 <br>
@@ -172,7 +270,7 @@ demo = gr.TabbedInterface(
     [tab_transcribe, tab_background, tab_architecture, tab_results, tab_authors],
     ["Transcribe", "Latar Belakang", "Arsitektur", "Evaluasi", "Fine-Tuned By"],
     theme=gr.themes.Soft(),
-    title="Whisper VS Conformer Model"
+    title="Multilingual ASR Model"
 )
 
 if __name__ == "__main__":

jawa.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28e88d2a129ae797fde52637b187fef218c30eddc891b8189eed8c0b40bf9dec
+size 200812

requirements.txt

@@ -1,3 +1,6 @@
+os
+torch
 numpy
 torchaudio
-transformers
+transformers
+huggingface_hub

sunda.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbc472fcba6f3f5203a9ccde45219f1dac1242a829451f6e397c905c3774eeac
+size 615864

test.py

@@ -0,0 +1,5 @@
+import os
+examples_audio = [
+    'data/'+ file for file in os.listdir("data")
+]
+print(examples_audio)