Upload 4 files

app.py

@@ -0,0 +1,60 @@
+import gradio as gr
+from scipy.io import wavfile
+from wav2vec_aligen import speaker_pronunciation_assesment
+
+
+
+def analyze_audio(audio):
+# Write the processed audio to a temporary WAV file
+    if audio is None:
+      return 'the audio is missing'
+    temp_filename = 'temp_audio.wav'
+    wavfile.write(temp_filename, audio[0], audio[1])
+
+
+    result = speaker_pronunciation_assesment(temp_filename)
+    accuracy_score = result['pronunciation_accuracy']
+    fluency_score   = result['fluency_score']
+    total_score       = result['total_score']
+    content_scores = result['content_scores']
+  
+    result_markdown = f"""|Language Aspect| Score|
+    |---|---| 
+    |Pronunciation Accuracy| {accuracy_score}|
+    |Fluency| {fluency_score}|
+    |Total Score| {total_score}|
+    |Content Score| {content_scores}|
+    """
+    return result_markdown
+  
+import gradio as gr
+
+CHOICES     = ['Daibers', 'Carbon', 'Reptiles']
+SENTENCES = [
+    """In Germany, over 100,000 tons of diapers are discarded each year, resulting in the wastage of valuable resources. Diaper liners, which contain special polymers known as superabsorbers, are among the materials that end up in landfills. However, researchers have made significant progress in enhancing the recycling process for these liners, leading to substantial improvements.""", 
+             
+             """Across the globe, there is a wide spread effort to explore methods for extracting carbon dioxide from the atmosphere or power plant emissions and transforming it into a valuable resource. Among the various ideas being explored, the concept of converting car bondioxide into a stable fuel shows significant promise.""",
+             
+             """Around 250 million years ago,700 species of reptiles closely related to the modern-day crocodile roamed the earth, now new research reveals how a complex interplay between climate change, species competition and habitat can help explain why just 23 species of crocodile survive today."""
+             ]
+
+PAIRED_TEXT = {k: v for k, v in zip(CHOICES, SENTENCES)}
+
+def get_paired_text(value):
+    text = '## ' + PAIRED_TEXT.get(value, '')
+    return text
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        with gr.Column():
+            with gr.Row():
+                drp_down = gr.Dropdown(choices=CHOICES, scale=2)
+                show_text_btn = gr.Button("Select", scale=1)
+            read_text = gr.Markdown(label='Read the follwing text')
+            show_text_btn.click(get_paired_text, inputs=drp_down, outputs=read_text)
+            audio_area = gr.Audio(label='Read the sentence')
+            analyize_audio_btn = gr.Button("Submit", scale=1)
+        with gr.Column():
+            capt_area = gr.Markdown(label='CAPT Scores')
+            analyize_audio_btn.click(analyze_audio, inputs=audio_area, outputs=capt_area)
+demo.launch()

model.py

@@ -0,0 +1,97 @@
+from transformers import Wav2Vec2BertPreTrainedModel, Wav2Vec2BertModel
+from transformers.modeling_outputs import SequenceClassifierOutput
+from typing import Optional, Tuple, Union
+from torch.nn import MSELoss
+import torch
+import torch.nn as nn
+
+class Wav2Vec2BertForSequenceClassification(Wav2Vec2BertPreTrainedModel):
+    # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForSequenceClassification.__init__ with Wav2Vec2->Wav2Vec2Bert,wav2vec2->wav2vec2_bert
+    def __init__(self, config):
+        super().__init__(config)
+
+        if hasattr(config, "add_adapter") and config.add_adapter:
+            raise ValueError(
+                "Sequence classification does not support the use of Wav2Vec2Bert adapters (config.add_adapter=True)"
+            )
+        self.wav2vec2_bert = Wav2Vec2BertModel(config)
+        num_layers = config.num_hidden_layers + 1  # transformer layers + input embeddings
+        if config.use_weighted_layer_sum:
+            self.layer_weights = nn.Parameter(torch.ones(num_layers) / num_layers)
+        self.projector = nn.Linear(config.hidden_size, config.classifier_proj_size)
+        self.classifier = nn.Linear(config.classifier_proj_size, config.num_labels)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def freeze_base_model(self):
+        """
+        Calling this function will disable the gradient computation for the base model so that its parameters will not
+        be updated during training. Only the classification head will be updated.
+        """
+        for param in self.wav2vec2_bert.parameters():
+            param.requires_grad = False
+
+    
+    # Copied from transformers.models.wav2vec2.modeling_wav2vec2.Wav2Vec2ForSequenceClassification.forward with Wav2Vec2->Wav2Vec2Bert,wav2vec2->wav2vec2_bert,WAV_2_VEC_2->WAV2VEC2_BERT, input_values->input_features
+    def forward(
+        self,
+        input_features: Optional[torch.Tensor],
+        attention_mask: Optional[torch.Tensor] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        labels: Optional[torch.Tensor] = None,
+    ) -> Union[Tuple, SequenceClassifierOutput]:
+        r"""
+        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
+            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
+            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
+            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
+        """
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        output_hidden_states = True if self.config.use_weighted_layer_sum else output_hidden_states
+
+        outputs = self.wav2vec2_bert(
+            input_features,
+            attention_mask=attention_mask,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        if self.config.use_weighted_layer_sum:
+            hidden_states = outputs[_HIDDEN_STATES_START_POSITION]
+            hidden_states = torch.stack(hidden_states, dim=1)
+            norm_weights = nn.functional.softmax(self.layer_weights, dim=-1)
+            hidden_states = (hidden_states * norm_weights.view(-1, 1, 1)).sum(dim=1)
+        else:
+            hidden_states = outputs[0]
+
+        hidden_states = self.projector(hidden_states)
+        if attention_mask is None:
+            pooled_output = hidden_states.mean(dim=1)
+        else:
+            padding_mask = self._get_feature_vector_attention_mask(hidden_states.shape[1], attention_mask)
+            hidden_states[~padding_mask] = 0.0
+            pooled_output = hidden_states.sum(dim=1) / padding_mask.sum(dim=1).view(-1, 1)
+
+        logits = self.classifier(pooled_output)
+        logits = nn.functional.relu(logits)
+
+        loss = None
+        if labels is not None:
+            loss_fct = MSELoss()
+            loss = loss_fct(logits.view(-1, self.config.num_labels), labels.view(-1, self.config.num_labels))
+
+        if not return_dict:
+            output = (logits,) + outputs[_HIDDEN_STATES_START_POSITION:]
+            return ((loss,) + output) if loss is not None else output
+
+        return SequenceClassifierOutput(
+            loss=loss,
+            logits=logits,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )

requirements.txt

@@ -0,0 +1,9 @@
+wave
+torch
+optimum
+scipy
+numpy 
+resampy
+gradio
+librosa
+transformers

wav2vec_aligen.py

@@ -0,0 +1,51 @@
+import torch
+import librosa
+import os
+from model import Wav2Vec2BertForSequenceClassification
+from transformers import AutoFeatureExtractor
+# from optimum.bettertransformer import BetterTransformer
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+# os.environ['PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION'] = 'python'
+# os.environ['TRANSFORMERS_NO_ADVISORY_WARNINGS'] = '1'
+# os.environ['TRANSFORMERS_VERBOSITY'] = 'error'
+torch.random.manual_seed(0); 
+# protobuf==3.20.0
+
+model_name = "arslanarjumand/wav2vec-reptiles"
+processor = AutoFeatureExtractor.from_pretrained(model_name)
+model = Wav2Vec2BertForSequenceClassification.from_pretrained(model_name).to(device)
+# model = BetterTransformer.transform(model)
+
+def load_audio(audio_path, processor):
+    audio, sr = librosa.load(audio_path, sr=16000)
+
+    input_values = processor(audio, sampling_rate=16000, return_tensors="pt").input_features
+    return input_values
+        
+@torch.inference_mode()
+def get_emissions(input_values, model):
+    results = model(input_values,).logits[0]
+    return results
+
+
+def speaker_pronunciation_assesment(audio_path):
+    input_values = load_audio(audio_path, processor)
+    result_scores = get_emissions(input_values, model)
+
+    pronunciation_score = round(result_scores[0].cpu().item())
+    fluency_score           = round(result_scores[1].cpu().item())
+    total_score              = round(result_scores[2].cpu().item())
+    content_scores         = round(result_scores[3].cpu().item())
+
+     
+
+    result = {'pronunciation_accuracy': pronunciation_score,
+              'content_scores': content_scores,
+              'total_score': total_score,
+              'fluency_score': fluency_score}
+    return result
+
+if __name__ == '__main__':
+    pass
+