Implemented everything

app.py

@@ -8,26 +8,17 @@ import io
 import base64
 from strsimpy.jaro_winkler import JaroWinkler
 
-# base64 to audio ✅
-# audio to transcription ✅
-# audio to text ✅
-# text to phoneme ✅
-# accuracy = jarowinkler(transcription, phoneme) ✅
-# band = getBandFromAccuracy(accuracy)  ✅
-# return accuracy, band ✅
 
-
-def lark(audioAsB64):
-    # base64 to wav data conversion
+def speechToPhonemeWS(audioAsB64):
     wav_data = base64.b64decode(audioAsB64.encode("utf-8"))
-
-    # audio to transcription
     processor = Wav2Vec2Processor.from_pretrained(
         "facebook/wav2vec2-xlsr-53-espeak-cv-ft"
     )
     model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-xlsr-53-espeak-cv-ft")
 
-    waveform, sample_rate = librosa.load(io.BytesIO(wav_data), sr=16000)
+    waveform, sample_rate = librosa.load(
+        io.BytesIO(wav_data), sr=16000
+    )  # Downsample 44.1kHz to 8kHz
 
     input_values = processor(
         waveform, sampling_rate=sample_rate, return_tensors="pt"
@@ -37,55 +28,68 @@ def lark(audioAsB64):
         logits = model(input_values).logits
 
     predicted_ids = torch.argmax(logits, dim=-1)
-    speechToPhonemeTranscription = processor.batch_decode(predicted_ids)[0]
+    transcription = processor.batch_decode(predicted_ids)
+    speechToPhonemeTranscription = transcription[0]
+    speechToPhonemeTranscription = speechToPhonemeTranscription.replace(" ", "")
+    return speechToPhonemeTranscription
+
+
+def speechToTextToPhonemeWS(audioAsB64):
+    wav_data = base64.b64decode(audioAsB64.encode("utf-8"))
 
-    # audio to text
-    processorSTT = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h")
+    waveform, sample_rate = librosa.load(
+        io.BytesIO(wav_data), sr=16000
+    )  # Downsample 44.1kHz to 8kHz
+    processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h")
     model = Wav2Vec2ForCTC.from_pretrained("facebook/wav2vec2-base-960h")
 
-    input_values = processorSTT(
+    input_values = processor(
         waveform, sampling_rate=sample_rate, return_tensors="pt"
     ).input_values
 
     logits = model(input_values).logits
 
     predicted_ids = torch.argmax(logits, dim=-1)
-    speechToTextTranscripition = processor.batch_decode(predicted_ids)[0]
+    speechToTextTranscription = processor.batch_decode(predicted_ids)
 
-    # text to phoneme
-    graphemeToPhonemeTranscription = phonemizer.phonemize(speechToTextTranscripition)
+    graphemeToPhonemeTranscription = phonemizer.phonemize(speechToTextTranscription[0])
+    graphemeToPhonemeTranscription = graphemeToPhonemeTranscription.replace(" ", "")
+    return [speechToTextTranscription[0], graphemeToPhonemeTranscription]
 
-    # accuracy = jaroWinkler(transcription, phoneme)
 
+def similarity(S2P, G2P2T):
     jarowinkler = JaroWinkler()
-    similarity_score = jarowinkler.similarity(
-        speechToPhonemeTranscription, graphemeToPhonemeTranscription
-    )
+    similarity_score = jarowinkler.similarity(S2P, G2P2T)
+    return similarity_score
+
+
+def similarityScoreToBand(similarity_score):
+    if similarity_score >= 0.91:
+        return 9
+    elif similarity_score >= 0.81:
+        return 8
+    elif similarity_score >= 0.73:
+        return 7
+    elif similarity_score >= 0.65:
+        return 6
+    elif similarity_score >= 0.60:
+        return 5
+    elif similarity_score >= 0.46:
+        return 4
+    elif similarity_score >= 0.35:
+        return 3
+    elif similarity_score >= 0.1:
+        return 2
+    else:
+        return 1
 
-    # ielts pronunciation band estimation
-    def getBandFromSimilarityScore(similarity_score):
-        if similarity_score >= 0.91:
-            return 9
-        elif similarity_score >= 0.81:
-            return 8
-        elif similarity_score >= 0.73:
-            return 7
-        elif similarity_score >= 0.65:
-            return 6
-        elif similarity_score >= 0.60:
-            return 5
-        elif similarity_score >= 0.46:
-            return 4
-        elif similarity_score >= 0.35:
-            return 3
-        elif similarity_score >= 0.1:
-            return 2
-        else:
-            return 1
-
-    IELTSband = getBandFromSimilarityScore(similarity_score)
-
-    return [similarity_score, IELTSband, speechToTextTranscripition]
+
+def lark(audioAsB64):
+    s2p = speechToPhonemeWS(audioAsB64)
+    [s2t, s2t2p] = speechToTextToPhonemeWS(audioAsB64)
+    ss = similarity(s2t2p, s2p)
+    band = similarityScoreToBand(ss)
+    return [ss, band, s2t]
 
 
 iface = gr.Interface(fn=lark, inputs="text", outputs=["text", "text", "text"])