Adding the voice feature

Dockerfile

@@ -1,6 +1,3 @@
-# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker
-# you will also find guides on how best to write your Dockerfile
-
 FROM python:3.9
 
 WORKDIR /code
@@ -8,7 +5,13 @@ WORKDIR /code
 # Copy requirements.txt separately to leverage Docker layer caching
 COPY ./requirements.txt /code/requirements.txt
 
-# Install dependencies
+# Install system dependencies for audio processing
+RUN apt-get update && apt-get install -y \
+    ffmpeg \
+    libsndfile1 \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Python dependencies
 RUN pip install --no-cache-dir --upgrade -r requirements.txt
 
 # Set the HF_HOME environment variable
@@ -19,6 +22,7 @@ RUN mkdir -p $HF_HOME && chmod -R 777 $HF_HOME
 
 # Copy the model files into the image
 COPY ./DistillMDPI1 /code/DistillMDPI1
+COPY ./Neptune /code/Neptune
 
 # Copy the rest of the application files
 COPY . .
@@ -27,4 +31,4 @@ COPY . .
 RUN mkdir -p /code/static/uploads && chmod -R 777 /code/static/uploads
 
 # Command to run your application
-CMD ["gunicorn", "-b", "0.0.0.0:7860", "app:app"]
+CMD ["gunicorn", "-b", "0.0.0.0:7860", "app:app"]

Model.py

@@ -0,0 +1,162 @@
+
+import torch
+import torch.nn as nn
+import numpy as np
+from transformers import  AutoTokenizer 
+import pickle
+
+# Load the tokenizer
+tokenizer = AutoTokenizer.from_pretrained("DistillMDPI1/DistillMDPI1/saved_tokenizer")
+
+
+
+# Step 1: Ensure the tokenizer has the [MULT] token
+tokenizer.add_special_tokens({'additional_special_tokens': ['<MULT>']})
+mult_token_id = tokenizer.convert_tokens_to_ids('<MULT>')
+cls_token_id = tokenizer.cls_token_id
+sep_token_id = tokenizer.sep_token_id
+pad_token_id = tokenizer.pad_token_id
+
+## Voice Part functions
+maxlen = 255 # maximum of length
+batch_size = 32
+max_pred = 5  # max tokens of prediction
+n_layers = 6 # number of Encoder of Encoder Layer
+n_heads = 12 # number of heads in Multi-Head Attention
+d_model = 768 # Embedding Size
+d_ff = 768 * 4  # 4*d_model, FeedForward dimension
+d_k = d_v = 64  # dimension of K(=Q), V
+n_segments = 2
+vocab_size = tokenizer.vocab_size +1
+
+def get_attn_pad_mask(seq_q, seq_k):
+    batch_size, len_q = seq_q.size()
+    batch_size, len_k = seq_k.size()
+    # eq(zero) is PAD token
+    pad_attn_mask = seq_k.data.eq(1).unsqueeze(1)  # batch_size x 1 x len_k(=len_q), one is masking
+    return pad_attn_mask.expand(batch_size, len_q, len_k) # batch_size x len_q x len_k
+
+class Embedding(nn.Module):
+    def __init__(self):
+        super(Embedding, self).__init__()
+        self.tok_embed = nn.Embedding(vocab_size, d_model)  # token embedding
+        self.pos_embed = nn.Embedding(maxlen, d_model)  # position embedding
+        self.seg_embed = nn.Embedding(n_segments, d_model)  # segment(token type) embedding
+        self.norm = nn.LayerNorm(d_model)
+
+    def forward(self, x, seg):
+        seq_len = x.size(1)
+        pos = torch.arange(seq_len, dtype=torch.long, device=x.device)
+        pos = pos.unsqueeze(0).expand_as(x)  # (seq_len,) -> (batch_size, seq_len)
+        embedding = self.tok_embed(x)
+        embedding += self.pos_embed(pos)
+        embedding += self.seg_embed(seg)
+        return self.norm(embedding)
+
+class ScaledDotProductAttention(nn.Module):
+    def __init__(self):
+        super(ScaledDotProductAttention, self).__init__()
+
+    def forward(self, Q, K, V, attn_mask):
+        scores = torch.matmul(Q, K.transpose(-1, -2)) / np.sqrt(d_k) # scores : [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]
+        scores.masked_fill_(attn_mask, -1e9) # Fills elements of self tensor with value where mask is one.
+        attn = nn.Softmax(dim=-1)(scores)
+        context = torch.matmul(attn, V)
+        return scores , context, attn
+
+class MultiHeadAttention(nn.Module):
+    def __init__(self):
+        super(MultiHeadAttention, self).__init__()
+        self.W_Q = nn.Linear(d_model, d_k * n_heads)
+        self.W_K = nn.Linear(d_model, d_k * n_heads)
+        self.W_V = nn.Linear(d_model, d_v * n_heads)
+        self.fc = nn.Linear(n_heads * d_v, d_model)
+        self.norm = nn.LayerNorm(d_model)
+    def forward(self, Q, K, V, attn_mask):
+        # q: [batch_size x len_q x d_model], k: [batch_size x len_k x d_model], v: [batch_size x len_k x d_model]
+        residual, batch_size = Q, Q.size(0)
+        device = Q.device
+        Q, K, V = Q.to(device), K.to(device), V.to(device)
+        # (B, S, D) -proj-> (B, S, D) -split-> (B, S, H, W) -trans-> (B, H, S, W)
+        q_s = self.W_Q(Q).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # q_s: [batch_size x n_heads x len_q x d_k]
+        k_s = self.W_K(K).view(batch_size, -1, n_heads, d_k).transpose(1,2)  # k_s: [batch_size x n_heads x len_k x d_k]
+        v_s = self.W_V(V).view(batch_size, -1, n_heads, d_v).transpose(1,2)  # v_s: [batch_size x n_heads x len_k x d_v]
+
+        attn_mask = attn_mask.unsqueeze(1).repeat(1, n_heads, 1, 1) # attn_mask : [batch_size x n_heads x len_q x len_k]
+
+        # context: [batch_size x n_heads x len_q x d_v], attn: [batch_size x n_heads x len_q(=len_k) x len_k(=len_q)]
+        scores ,context, attn = ScaledDotProductAttention()(q_s, k_s, v_s, attn_mask)
+        context = context.transpose(1, 2).contiguous().view(batch_size, -1, n_heads * d_v) # context: [batch_size x len_q x n_heads * d_v]
+        output = self.fc(context)
+        return self.norm(output + residual), attn # output: [batch_size x len_q x d_model]
+
+class PoswiseFeedForwardNet(nn.Module):
+    def __init__(self):
+        super(PoswiseFeedForwardNet, self).__init__()
+        self.fc1 = nn.Linear(d_model, d_ff)
+        self.fc2 = nn.Linear(d_ff, d_model)
+        self.gelu = nn.GELU()
+    def forward(self, x):
+        # (batch_size, len_seq, d_model) -> (batch_size, len_seq, d_ff) -> (batch_size, len_seq, d_model)
+        return self.fc2(self.gelu(self.fc1(x)))
+
+class EncoderLayer(nn.Module):
+    def __init__(self):
+        super(EncoderLayer, self).__init__()
+        self.enc_self_attn = MultiHeadAttention()
+        self.pos_ffn = PoswiseFeedForwardNet()
+
+    def forward(self, enc_inputs, enc_self_attn_mask):
+        enc_outputs, attn = self.enc_self_attn(enc_inputs, enc_inputs, enc_inputs, enc_self_attn_mask.to(enc_inputs.device)) # enc_inputs to same Q,K,V
+        enc_outputs = self.pos_ffn(enc_outputs) # enc_outputs: [batch_size x len_q x d_model]
+        return enc_outputs, attn
+
+class BERT(nn.Module):
+    def __init__(self):
+        super(BERT, self).__init__()
+        self.embedding = Embedding()
+        self.layers = nn.ModuleList([EncoderLayer() for _ in range(n_layers)])
+        self.fc = nn.Linear(d_model, d_model)
+        self.activ1 = nn.Tanh()
+        self.linear = nn.Linear(d_model, d_model)
+        self.activ2 = nn.GELU()
+        self.norm = nn.LayerNorm(d_model)
+        self.classifier = nn.Linear(d_model, 2)
+        # decoder is shared with embedding layer
+        embed_weight = self.embedding.tok_embed.weight
+        n_vocab, n_dim = embed_weight.size()
+        self.decoder = nn.Linear(n_dim, n_vocab, bias=False)
+        self.decoder.weight = embed_weight
+        self.decoder_bias = nn.Parameter(torch.zeros(n_vocab))
+        self.mclassifier = nn.Linear(d_model, 17)
+
+    def forward(self, input_ids, segment_ids, masked_pos):
+        output = self.embedding(input_ids, segment_ids)
+        enc_self_attn_mask = get_attn_pad_mask(input_ids, input_ids).to(output.device)
+        for layer in self.layers:
+            output, enc_self_attn = layer(output, enc_self_attn_mask)
+        # output : [batch_size, len, d_model], attn : [batch_size, n_heads, d_mode, d_model]
+        # it will be decided by first token(CLS)
+        h_pooled = self.activ1(self.fc(output[:, 0])) # [batch_size, d_model]
+        logits_clsf = self.classifier(h_pooled) # [batch_size, 2]
+
+        masked_pos = masked_pos[:, :, None].expand(-1, -1, output.size(-1)) # [batch_size, max_pred, d_model]
+        # get masked position from final output of transformer.
+        h_masked = torch.gather(output, 1, masked_pos) # masking position [batch_size, max_pred, d_model]
+        h_masked = self.norm(self.activ2(self.linear(h_masked)))
+        logits_lm = self.decoder(h_masked) + self.decoder_bias # [batch_size, max_pred, n_vocab]
+
+        h_mult_sent1 = self.activ1(self.fc(output[:, 1]))
+        logits_mclsf1 = self.mclassifier(h_mult_sent1)
+
+        mult2_token_id = mult_token_id  # Assuming mult_token_id is defined globally
+        mult2_positions = (input_ids == mult2_token_id).nonzero(as_tuple=False)  # Find positions of [MULT2] tokens
+        # Ensure there are exactly two [MULT] tokens in each input sequence
+        assert mult2_positions.size(0) == 2 * input_ids.size(0)
+        mult2_positions = mult2_positions[1::2][:, 1]
+        # Gather the hidden states corresponding to the second [MULT] token
+        h_mult_sent2 = output[torch.arange(output.size(0)), mult2_positions]
+
+        logits_mclsf2 = self.mclassifier(h_mult_sent2)
+        logits_mclsf2 = self.mclassifier(h_mult_sent2)
+        return logits_lm, logits_clsf , logits_mclsf1 , logits_mclsf2

Neptune/Neptune/added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<MULT>": 30522
+}

Neptune/Neptune/model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7f0cace6edaecd1f2e30cd8cbe3ad1d6d43829c3029279308d4ebb2ec8542c0
+size 269574874

Neptune/Neptune/special_tokens_map.json

@@ -0,0 +1,16 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<MULT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

Neptune/Neptune/tokenizer_config.json

@@ -0,0 +1,66 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "101": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "102": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "103": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30522": {
+      "content": "<MULT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<MULT>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_lower_case": true,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "DistilBertTokenizer",
+  "unk_token": "[UNK]"
+}

Neptune/Neptune/unique_labels.json

@@ -0,0 +1 @@
+["Physics", "Societies", "admsci", "agriculture", "ai", "applsci", "asi", "biology", "economies", "energies", "environments", "make", "mathematics", "robotics", "sports", "technologies", "vehicles"]

app.py

@@ -1,13 +1,32 @@
-from flask import Flask, render_template,request, redirect,url_for, jsonify
-from helper_functions import predict_class
+from flask import Flask, render_template,request, redirect,url_for, jsonify , session
+from helper_functions import predict_class , prepare_text , inference , predict , align_predictions_with_sentences , load_models
 import fitz  # PyMuPDF
 import os, shutil
 import torch
-import pickle
-
+import tempfile
+from pydub import AudioSegment
+import logging
+ 
 app = Flask(__name__)
 app.config['UPLOAD_FOLDER'] = 'static/uploads' 
 
+# Global variables for models
+global_model = None
+global_neptune = None
+global_tokenizer = None
+global_pipe = None
+
+def init_app():
+    global global_model, global_neptune, global_pipe
+    print("Loading models...")
+    global_model, global_neptune, global_pipe = load_models()
+    print("Models loaded successfully!")
+
+# Call init_app() before the first request
+@app.before_first_request
+def before_first_request():
+    init_app()
+
 @app.route("/")
 def home():
     predict_class = ""
@@ -24,6 +43,7 @@ def pdf():
 
 @app.route('/pdf/upload' , methods = ['POST'])
 def treatment():
+    global global_model, global_tokenizer
     if request.method == 'POST' :
         # Récupérer le fichier PDF de la requête
         file = request.files['file']
@@ -53,7 +73,7 @@ def treatment():
         # Fermer le fichier PDF
         pdf_document.close()
         # Prepare data for the chart
-        predicted_class , class_probabilities = predict_class([extracted_text])
+        predicted_class , class_probabilities = predict_class([extracted_text] , global_model)
         chart_data = {
             'datasets': [{
                 'data': list(class_probabilities.values()),
@@ -77,17 +97,20 @@ def treatment():
         return render_template('pdf.html',extracted_text = extracted_text, class_probabilities=class_probabilities, predicted_class=predicted_class, chart_data = chart_data)
     return render_template('pdf.html')
 
+## Sentence
+
 @app.route('/sentence' , methods = ['GET' , 'POST'])
 def sentence():
+    global global_model, global_tokenizer
     if request.method == 'POST':
         # Get the form data
         text = [request.form['text']]
-        predicted_class , class_probabilities = predict_class(text)
+        predicted_class , class_probabilities = predict_class(text , global_model)
         # Prepare data for the chart
         chart_data = {
             'datasets': [{
                 'data': list(class_probabilities.values()),
-                'backgroundColor': [color[2] for color in class_probabilities.keys()],
+                'backgroundColor': [color[2 ] for color in class_probabilities.keys()],
                 'borderColor': [color[2] for color in class_probabilities.keys()]
             }],
             'labels': [label[0] for label in class_probabilities.keys()]
@@ -108,6 +131,121 @@ def sentence():
     # Render the initial form page
     return render_template('sentence.html')
 
+## Voice 
+@app.route("/voice_backup")
+def slu_backup():
+    input_file = "static/uploads/2022.jep-architectures-neuronales.pdf"
+    # Ouvrir le fichier PDF
+    pdf_document = fitz.open(input_file)
+    # Initialiser une variable pour stocker le texte extrait
+    extracted_text = ""
+    # Boucler à travers chaque page pour extraire le texte
+    for page_num in range(len(pdf_document)):
+        # Récupérer l'objet de la page
+        page = pdf_document.load_page(page_num)
+
+        # Extraire le texte de la page
+        page_text = page.get_text()
+
+        # Ajouter le texte de la page à la variable d'extraction
+        extracted_text += f"\nPage {page_num + 1}:\n{page_text}"
+
+    # Fermer le fichier PDF
+    pdf_document.close()
+    # Prepare data for the chart
+    inference_batch, sentences = inference(extracted_text)
+    predictions = predict(inference_batch)
+    sentences_prediction = align_predictions_with_sentences(sentences, predictions)
+    predicted_class , class_probabilities = predict_class([extracted_text] , global_model)
+
+    chart_data = {
+            'datasets': [{
+                'data': list(class_probabilities.values()),
+                'backgroundColor': [color[2 ] for color in class_probabilities.keys()],
+                'borderColor': [color[2] for color in class_probabilities.keys()]
+            }],
+            'labels': [label[0] for label in class_probabilities.keys()]
+        }
+    print(class_probabilities)
+    print(chart_data)
+    print(sentences_prediction)
+    return render_template('voice_backup.html',extracted_text = extracted_text, class_probabilities=class_probabilities, predicted_class=predicted_class, chart_data = chart_data, sentences_prediction = sentences_prediction)
+
+logging.basicConfig(level=logging.DEBUG)
+
+@app.route("/voice", methods=['GET', 'POST'])
+def slu():
+    global global_neptune, global_pipe, global_model
+
+    if request.method == 'POST':
+        logging.debug("Received POST request")
+        audio_file = request.files.get('audio')
+
+        if audio_file:
+            logging.debug(f"Received audio file: {audio_file.filename}")
+            
+            # Save audio data to a temporary file
+            with tempfile.NamedTemporaryFile(delete=False, suffix='.wav') as temp_audio:
+                audio_file.save(temp_audio)
+                temp_audio_path = temp_audio.name
+
+            logging.debug(f"Saved audio to temporary file: {temp_audio_path}")
+
+            try:
+                # Transcribe audio using Whisper
+                result = global_pipe(temp_audio_path)
+                extracted_text = result["text"]
+                logging.debug(f"Transcribed text: {extracted_text}")
+
+                # Process the transcribed text
+                inference_batch, sentences = inference(extracted_text)
+                predictions = predict(inference_batch, global_neptune)
+                sentences_prediction = align_predictions_with_sentences(sentences, predictions)
+                predicted_class, class_probabilities = predict_class([extracted_text], global_model)
+
+                chart_data = {
+                    'datasets': [{
+                        'data': list(class_probabilities.values()),
+                        'backgroundColor': [color[2] for color in class_probabilities.keys()],
+                        'borderColor': [color[2] for color in class_probabilities.keys()]
+                    }],
+                    'labels': [label[0] for label in class_probabilities.keys()]
+                }
+
+                response_data = {
+                    'extracted_text': extracted_text,
+                    'class_probabilities' : class_probabilities,
+                    'predicted_class': predicted_class,
+                    'chart_data': chart_data,
+                    'sentences_prediction': sentences_prediction
+                }
+                logging.debug(f"Prepared response data: {response_data}")
+
+                return render_template('voice.html', 
+                           class_probabilities= class_probabilities, 
+                           predicted_class= predicted_class, 
+                           chart_data= chart_data, 
+                           sentences_prediction=sentences_prediction)
+
+            except Exception as e:
+                logging.error(f"Error processing audio: {str(e)}")
+                return jsonify({'error': str(e)}), 500
+
+            finally:
+                # Remove temporary file
+                os.unlink(temp_audio_path)
+
+        else:
+            logging.error("No audio file received")
+            return jsonify({'error': 'No audio file received'}), 400
+
+    # For GET request
+    logging.debug("Received GET request")
+    return render_template('voice.html', 
+                           class_probabilities={}, 
+                           predicted_class=[""], 
+                           chart_data={}, 
+                           sentences_prediction={})
 
 if __name__ == '__main__':
     app.run(debug=True)

helper_functions.py

@@ -4,17 +4,38 @@ from transformers import  AutoTokenizer , DistilBertForSequenceClassification
 from transformers import BatchEncoding, PreTrainedTokenizerBase
 from typing import Optional
 from torch import Tensor
+import numpy as np 
+from random import shuffle
+from Model import BERT
+from Model import tokenizer , mult_token_id , cls_token_id , pad_token_id , max_pred , maxlen , sep_token_id 
+from transformers import pipeline
 
+device = "cpu"
 # Load the model
-model = DistilBertForSequenceClassification.from_pretrained("DistillMDPI1/DistillMDPI1/saved_model")
-
-# Load the tokenizer
-tokenizer = AutoTokenizer.from_pretrained("DistillMDPI1/DistillMDPI1/saved_tokenizer")
-
-# Charger le label encoder
-with open("DistillMDPI1/DistillMDPI1/label_encoder.pkl", "rb") as f:
-    label_encoder = pickle.load(f)
-
+def load_models():
+    print("Loading DistilBERT model...")
+    model = DistilBertForSequenceClassification.from_pretrained("DistillMDPI1/DistillMDPI1/saved_model")
+    
+    print("Loading BERT model...")
+    neptune = BERT()
+    device = "cpu"
+    model_save_path = "Neptune/Neptune/model.pt"
+    neptune.load_state_dict(torch.load(model_save_path, map_location=torch.device('cpu')))
+    neptune.to(device)
+    
+    print("Loading speech recognition pipeline...")
+    pipe = pipeline(
+        "automatic-speech-recognition",
+        model="openai/whisper-tiny.en",
+        chunk_length_s=30,
+        device=device,
+    )
+    print(pipe)
+    # Charger le label encoder
+    with open("DistillMDPI1/DistillMDPI1/label_encoder.pkl", "rb") as f:
+        label_encoder = pickle.load(f)
+    
+    return model, neptune, pipe
 
 class_labels = {
     16: ('vehicles','info' , '#4f9ef8'),
@@ -35,7 +56,7 @@ class_labels = {
     2: ('administration','pink', '#d63384'),
     7: ('biology' ,'cambridge' , '#88aa99')}
 
-def predict_class(text):
+def predict_class(text , model):
     # Tokenisation du texte
     inputs = transform_list_of_texts(text, tokenizer, 510, 510, 1, 2550)
     # Extraire le tenseur de la liste
@@ -91,7 +112,7 @@ def transform_single_text(
         tokens = tokenize_whole_text(text, tokenizer)
     input_id_chunks, mask_chunks = split_tokens_into_smaller_chunks(tokens, chunk_size, stride, minimal_chunk_length)
     add_special_tokens_at_beginning_and_end(input_id_chunks, mask_chunks)
-    add_padding_tokens(input_id_chunks, mask_chunks)
+    input_ids, attention_mask = stack_tokens_from_all_chunks(input_id_chunks, mask_chunks)
     input_ids, attention_mask = stack_tokens_from_all_chunks(input_id_chunks, mask_chunks)
     return input_ids, attention_mask
 
@@ -137,16 +158,19 @@ def add_special_tokens_at_beginning_and_end(input_id_chunks: list[Tensor], mask_
         mask_chunks[i] = torch.cat([Tensor([1]), mask_chunks[i], Tensor([1])])
 
 
-def add_padding_tokens(input_id_chunks: list[Tensor], mask_chunks: list[Tensor]) -> None:
-    """Adds padding tokens (token id = 0) at the end to make sure that all chunks have exactly 512 tokens."""
+
+def add_padding_tokens(input_id_chunks: list[Tensor], mask_chunks: list[Tensor], chunk_size: int) -> None:
+    """Adds padding tokens at the end to make sure that all chunks have exactly chunk_size tokens."""
+    pad_token_id = 0  # Assuming this is defined somewhere in your code
     for i in range(len(input_id_chunks)):
         # get required padding length
-        pad_len = 512 - input_id_chunks[i].shape[0]
+        pad_len = chunk_size +2 - input_id_chunks[i].shape[0]
         # check if tensor length satisfies required chunk size
         if pad_len > 0:
             # if padding length is more than 0, we must add padding
-            input_id_chunks[i] = torch.cat([input_id_chunks[i], Tensor([0] * pad_len)])
-            mask_chunks[i] = torch.cat([mask_chunks[i], Tensor([0] * pad_len)])
+            input_id_chunks[i] = torch.cat([input_id_chunks[i], torch.tensor([pad_token_id] * pad_len)])
+            mask_chunks[i] = torch.cat([mask_chunks[i], torch.tensor([0] * pad_len)])
+
 
 
 def stack_tokens_from_all_chunks(input_id_chunks: list[Tensor], mask_chunks: list[Tensor]) -> tuple[Tensor, Tensor]:
@@ -165,3 +189,144 @@ def split_overlapping(tensor: Tensor, chunk_size: int, stride: int, minimal_chun
         result = [x for x in result if len(x) >= minimal_chunk_length]
     return result
 
+## Voice part
+
+def transform_for_inference_text(text: str,
+    tokenizer: PreTrainedTokenizerBase,
+    chunk_size: int,
+    stride: int,
+    minimal_chunk_length: int,
+    maximal_text_length: Optional[int],) -> BatchEncoding:
+    if maximal_text_length:
+        tokens = tokenize_text_with_truncation(text, tokenizer, maximal_text_length)
+    else:
+        tokens = tokenize_whole_text(text, tokenizer)
+    input_id_chunks, mask_chunks = split_tokens_into_smaller_chunks(tokens, chunk_size, stride, minimal_chunk_length)
+    add_special_tokens_at_beginning_and_end_inference(input_id_chunks, mask_chunks)
+    add_padding_tokens_inference(input_id_chunks, mask_chunks, chunk_size)
+    input_ids, attention_mask = stack_tokens_from_all_chunks(input_id_chunks, mask_chunks)
+    return {"input_ids": input_ids, "attention_mask": attention_mask}
+
+def add_special_tokens_at_beginning_and_end_inference(input_id_chunks: list[Tensor], mask_chunks: list[Tensor]) -> None:
+    """
+    Adds special MULT token, CLS token at the beginning.
+    Adds SEP token at the end of each chunk.
+    Adds corresponding attention masks equal to 1 (attention mask is boolean).
+    """
+    for i in range(len(input_id_chunks)):
+        # adding MULT, CLS, and SEP tokens
+        input_id_chunks[i] = torch.cat([input_id_chunks[i]])
+        # adding attention masks corresponding to special tokens
+        mask_chunks[i] = torch.cat([mask_chunks[i]])
+
+def add_padding_tokens_inference(input_id_chunks: list[Tensor], mask_chunks: list[Tensor], chunk_size: int) -> None:
+    """Adds padding tokens at the end to make sure that all chunks have exactly chunk_size tokens."""
+    pad_token_id = 0  # Assuming this is defined somewhere in your code
+    for i in range(len(input_id_chunks)):
+        # get required padding length
+        pad_len = chunk_size - input_id_chunks[i].shape[0]
+        # check if tensor length satisfies required chunk size
+        if pad_len > 0:
+            # if padding length is more than 0, we must add padding
+            input_id_chunks[i] = torch.cat([input_id_chunks[i], torch.tensor([pad_token_id] * pad_len)])
+            mask_chunks[i] = torch.cat([mask_chunks[i], torch.tensor([0] * pad_len)])
+
+def prepare_text(tokens_splitted: BatchEncoding):
+    batch = []
+    sentences = []
+    input_ids_list = tokens_splitted['input_ids']
+    
+    for i in range(0, len(input_ids_list), 2):  # Adjust loop to stop at second last index
+        k = i + 1
+        if k == len(input_ids_list):
+            input_ids_a = input_ids_list[i]
+            input_ids_a = [token for token in input_ids_a.view(-1).tolist() if token != pad_token_id]
+            input_ids_b = []
+            input_ids = [cls_token_id] + [mult_token_id] + input_ids_a + [sep_token_id] + [mult_token_id] + input_ids_b + [sep_token_id]
+            text_input_a = tokenizer.decode(input_ids_a)
+            sentences.append(text_input_a)
+            segment_ids = [0] * (1 + 1 + len(input_ids_a) + 1) + [1] * (1 + len(input_ids_b) + 1)
+            
+            # MASK LM
+            n_pred = min(max_pred, max(1, int(round(len(input_ids) * 0.15))))
+            cand_masked_pos = [idx for idx, token in enumerate(input_ids) if token not in [cls_token_id, sep_token_id, mult_token_id]]
+            shuffle(cand_masked_pos)
+            masked_tokens, masked_pos = [], []
+            for pos in cand_masked_pos[:n_pred]:
+                masked_pos.append(pos)
+                masked_tokens.append(input_ids[pos])
+                input_ids[pos] = tokenizer.mask_token_id
+
+            # Zero Padding
+            n_pad = maxlen - len(input_ids)
+            input_ids.extend([pad_token_id] * n_pad)
+            segment_ids.extend([0] * n_pad)
+
+            # Zero Padding for masked tokens
+            if max_pred > n_pred:
+                n_pad = max_pred - n_pred
+                masked_tokens.extend([0] * n_pad)
+                masked_pos.extend([0] * n_pad)
+        else:
+            input_ids_a = input_ids_list[i]  # Correct the indexing here
+            input_ids_b = input_ids_list[k]  # Correct the indexing here
+            input_ids_a = [token for token in input_ids_a.view(-1).tolist() if token != pad_token_id]
+            input_ids_b = [token for token in input_ids_b.view(-1).tolist() if token != pad_token_id]
+            input_ids = [cls_token_id] + [mult_token_id] + input_ids_a + [sep_token_id] + [mult_token_id] + input_ids_b + [sep_token_id]
+            segment_ids = [0] * (1 + 1 + len(input_ids_a) + 1) + [1] * (1 + len(input_ids_b) + 1)
+            text_input_a = tokenizer.decode(input_ids_a)
+            text_input_b = tokenizer.decode(input_ids_b)
+            sentences.append(text_input_a)
+            sentences.append(text_input_b)
+
+            # MASK LM
+            n_pred = min(max_pred, max(1, int(round(len(input_ids) * 0.15))))
+            cand_masked_pos = [idx for idx, token in enumerate(input_ids) if token not in [cls_token_id, sep_token_id, mult_token_id]]
+            shuffle(cand_masked_pos)
+            masked_tokens, masked_pos = [], []
+            for pos in cand_masked_pos[:n_pred]:
+                masked_pos.append(pos)
+                masked_tokens.append(input_ids[pos])
+                input_ids[pos] = tokenizer.mask_token_id
+
+            # Zero Padding
+            n_pad = maxlen - len(input_ids)
+            input_ids.extend([pad_token_id] * n_pad)
+            segment_ids.extend([0] * n_pad)
+
+            # Zero Padding for masked tokens
+            if max_pred > n_pred:
+                n_pad = max_pred - n_pred
+                masked_tokens.extend([0] * n_pad)
+                masked_pos.extend([0] * n_pad)
+
+        batch.append([input_ids, segment_ids, masked_pos])
+    return batch, sentences
+
+def inference(text: str):
+    encoded_text = transform_for_inference_text(text, tokenizer, 125, 125, 1, 2550)
+    batch, sentences = prepare_text(encoded_text)
+    return batch, sentences
+
+def predict(inference_batch,neptune , device = device):
+    all_preds_mult1 = []
+    neptune.eval()
+    with torch.no_grad():
+        for batch in inference_batch:
+            input_ids = torch.tensor(batch[0], device=device, dtype=torch.long).unsqueeze(0)
+            segment_ids = torch.tensor(batch[1], device=device, dtype=torch.long).unsqueeze(0)
+            masked_pos = torch.tensor(batch[2], device=device, dtype=torch.long).unsqueeze(0)
+            _, _, logits_mclsf1, logits_mclsf2 = neptune(input_ids, segment_ids, masked_pos)
+            preds_mult1 = torch.argmax(logits_mclsf1, dim=1).cpu().detach().numpy()
+            preds_mult2 = torch.argmax(logits_mclsf2, dim=1).cpu().detach().numpy()
+            
+            all_preds_mult1.extend(preds_mult1)
+            all_preds_mult1.extend(preds_mult2)
+
+    return all_preds_mult1
+
+def align_predictions_with_sentences(sentences, preds):
+    dc = {}  # Initialize an empty dictionary
+    for sentence, pred in zip(sentences, preds):  # Iterate through sentences and predictions
+        dc[sentence] = class_labels.get(pred, "Unknown")  # Look up the label for each prediction
+    return dc

requirements.txt

@@ -3,4 +3,6 @@ gunicorn
 PyPDF2
 PyMuPDF
 torch
-transformers
+transformers
+pydub
+whisper

static/css/style2.css

@@ -3518,6 +3518,11 @@ textarea.form-control-lg {
   background-color: #d6293e;
   border-color: #d6293e;
 }
+.btn-green {
+  color: #fff;
+  background-color: #72AB5A;
+  border-color: #72AB5A;
+}
 .btn-danger:hover {
   color: #fff;
   background-color: #b62335;
@@ -16576,6 +16581,7 @@ html[data-theme=dark] .light-mode-item {
   z-index: 0;
   border: 2px dashed #eef0f2;
 }
+
 .dropzone .dz-preview.dz-image-preview {
   background: transparent;
 }
@@ -17334,5 +17340,48 @@ html[data-theme=dark] .light-mode-item {
 .mw-80 {
   max-width: 80%;
 }
+.card-body {
+  display: flex;
+  flex-direction: column;
+}
+
+.card-body > div:first-child {
+  flex-grow: 1;
+}
+
+.card-body .mt-auto {
+  margin-top: auto;
+}
+
+.file-upload-wrapper {
+  position: relative;
+  width: 100%;
+  height: 100px;
+  border: 2px dashed #72AB5A;
+  border-radius: 5px;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  overflow: hidden;
+  cursor: pointer;
+  background-color: #303131;
+}
+.file-upload-wrapper input[type=file] {
+  position: absolute;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  opacity: 0;
+  cursor: pointer;
+}
+.file-upload-text {
+  color: #fff;
+  text-align: center;
+}
+.file-name {
+  margin-top: 10px;
+  color: #72AB5A;
+}
 /*User CSS*/
 /*# sourceMappingURL=style.css.map */

static/js/dashboard_voice.js

@@ -0,0 +1,54 @@
+// Récupérez les données, les couleurs de fond, les couleurs de bordure et les étiquettes des paramètres de l'URL
+var urlParams = new URLSearchParams(window.location.search);
+var data = JSON.parse(decodeURIComponent(urlParams.get('data')));
+var backgroundColor = JSON.parse(decodeURIComponent(urlParams.get('backgroundColor')));
+var borderColor = JSON.parse(decodeURIComponent(urlParams.get('borderColor')));
+var labels = JSON.parse(decodeURIComponent(urlParams.get('labels')));
+
+// Function to initialize or update the chart with dynamic data
+function initializeOrUpdateChart(data, backgroundColor, borderColor, labels) {
+    // Check if a chart instance exists
+    if (window.myChart) {
+        // Update the existing chart
+        window.myChart.data.datasets[0].data = data;
+        window.myChart.data.datasets[0].backgroundColor = backgroundColor;
+        window.myChart.data.datasets[0].borderColor = borderColor;
+        window.myChart.data.labels = labels;
+        window.myChart.update();
+    } else {
+        // Create a new chart instance
+        var ctx = document.getElementById('bestSellers').getContext('2d');
+        window.myChart = new Chart(ctx, {
+            type: 'doughnut',
+            data: {
+                datasets: [{
+                    data: data,
+                    backgroundColor: backgroundColor,
+                    borderColor: borderColor
+                }],
+                labels: labels
+            },
+            options: {
+                responsive: true,
+                cutoutPercentage: 80,
+                legend: {
+                    display: false
+                },
+                animation: {
+                    animateScale: true,
+                    animateRotate: true
+                },
+                plugins: {
+                    datalabels: {
+                        display: false,
+                        align: 'center',
+                        anchor: 'center'
+                    }
+                }
+            }
+        });
+    }
+}
+
+// Initialize or update the chart when the script is loaded
+initializeOrUpdateChart(data, backgroundColor, borderColor, labels);

static/js/voice.js

@@ -0,0 +1,233 @@
+const reset = document.getElementById("reset");
+const currentClassProbabilitiesList = document.getElementById("class-probabilities");
+const currentPredictedClass = document.getElementById('predicted-class');
+const staticDiv = document.getElementById("static");
+const dynamicDiv = document.getElementById("dynamic");
+var chartData;
+
+let mediaRecorder;
+let audioChunks = [];
+
+document.addEventListener('DOMContentLoaded', function() {
+    loadResults();
+    attachEventListeners();
+});
+
+function attachEventListeners() {
+    document.getElementById('startRecord').addEventListener('click', startRecording);
+    document.getElementById('stopRecord').addEventListener('click', stopRecording);
+    document.getElementById('uploadAudio').addEventListener('click', handleAudioUpload);
+}
+
+function initializeChart(data, backgroundColor, borderColor) {
+    const canvas = document.getElementById('bestSellers');
+    
+    // Destroy existing chart if it exists
+    const existingChart = Chart.getChart(canvas);
+    if (existingChart) {
+        existingChart.destroy();
+    }
+
+    // Clear the canvas
+    const context = canvas.getContext('2d');
+    context.clearRect(0, 0, canvas.width, canvas.height);
+
+    data = data.map(function (element) {
+        return parseFloat(element).toFixed(2);
+    });
+
+    new Chart(canvas, {
+        type: 'doughnut',
+        data: {
+            datasets: [{
+                data: data,
+                backgroundColor: backgroundColor,
+                borderColor: borderColor,
+                borderWidth: 1
+                
+            }]
+        },
+        options: {
+            responsive: true,
+            cutout: '80%',
+            plugins: {
+                legend: {
+                    display: true,
+                },
+                tooltip: {
+                    enabled: false
+                }
+            },
+            layout: {
+                padding: 0
+            },
+            elements: {
+                arc: {
+                    borderWidth: 0
+                }
+            },
+            plugins: {
+                datalabels: {
+                    display: false,
+                    align: 'center',
+                    anchor: 'center'
+                }
+            }
+        }
+    });
+}
+
+function loadResults() {
+    fetch('/voice')
+        .then(response => response.text())
+        .then(html => {
+            const responseDOM = new DOMParser().parseFromString(html, "text/html");
+            const classProbabilitiesList = responseDOM.getElementById("class-probabilities");
+            currentClassProbabilitiesList.innerHTML = classProbabilitiesList.innerHTML;
+            const PredictedClass = responseDOM.getElementById("predicted-class")
+            currentPredictedClass.innerHTML = PredictedClass.innerHTML;
+
+            var canvasElement = responseDOM.querySelector('.bestSellers');
+            console.log(canvasElement);
+            chartData = canvasElement.getAttribute('data-chart');
+            console.log(chartData);
+            if (chartData) {
+                var parsedChartData = JSON.parse(chartData);
+                var data = parsedChartData.datasets[0].data.slice(0, 5);
+                var backgroundColor = parsedChartData.datasets[0].backgroundColor.slice(0, 5);
+                var borderColor = parsedChartData.datasets[0].borderColor.slice(0, 5);
+                var labels = parsedChartData.labels.slice(0, 5);
+
+                initializeChart(data, backgroundColor, borderColor, labels);
+            }
+        })
+        .catch(error => console.error('Error:', error));
+}
+
+function startRecording() {
+    navigator.mediaDevices.getUserMedia({ audio: true })
+        .then(stream => {
+            mediaRecorder = new MediaRecorder(stream);
+            mediaRecorder.start();
+
+            audioChunks = [];
+            mediaRecorder.addEventListener("dataavailable", event => {
+                audioChunks.push(event.data);
+            });
+
+            document.getElementById('startRecord').disabled = true;
+            document.getElementById('stopRecord').disabled = false;
+        });
+}
+
+function stopRecording() {
+    mediaRecorder.stop();
+    document.getElementById('startRecord').disabled = false;
+    document.getElementById('stopRecord').disabled = true;
+
+    mediaRecorder.addEventListener("stop", () => {
+        const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
+        sendAudioToServer(audioBlob);
+    });
+}
+
+function handleAudioUpload() {
+    const fileInput = document.getElementById('audioFileInput');
+    if (fileInput.files.length > 0) {
+        const file = fileInput.files[0];
+        sendAudioToServer(file);
+    } else {
+        console.error('No file selected');
+    }
+}
+function sendAudioToServer(audioData) {
+    // ... (existing code)
+
+    
+}
+function sendAudioToServer(audioData) {
+    const formData = new FormData();
+    formData.append('audio', audioData);
+
+    document.getElementById('loadingIndicator').style.display = 'block';
+
+    // Clear existing chart
+    const canvas = document.getElementById('bestSellers');
+    const existingChart = Chart.getChart(canvas);
+    if (existingChart) {
+        existingChart.destroy();
+    }
+    const context = canvas.getContext('2d');
+    context.clearRect(0, 0, canvas.width, canvas.height);
+
+    fetch('/voice', {
+        method: 'POST',
+        body: formData
+    })
+    .then(response => response.text())
+    .then(html => {
+        const parser = new DOMParser();
+        const newDocument = parser.parseFromString(html, 'text/html');
+        
+        // Update other parts of the page as before
+        // Update only the necessary parts of the page
+        document.getElementById('class-probabilities').innerHTML = newDocument.getElementById('class-probabilities').innerHTML;
+        document.getElementById('predicted-class').innerHTML = newDocument.getElementById('predicted-class').innerHTML;
+        document.getElementById('transcribedText').innerHTML = newDocument.getElementById('transcribedText').innerHTML;
+        document.getElementById('classifiedText').innerHTML = newDocument.getElementById('classifiedText').innerHTML;
+        dynamicDiv.classList.remove('d-none');
+        staticDiv.classList.add('d-none');
+        // Update chart
+        const newCanvasElement = newDocument.querySelector('.bestSellers');
+        if (newCanvasElement) {
+            const newChartData = newCanvasElement.getAttribute('data-chart');
+            if (newChartData) {
+                const parsedChartData = JSON.parse(newChartData);
+                initializeChart(
+                    parsedChartData.datasets[0].data.slice(0, 5),
+                    parsedChartData.datasets[0].backgroundColor.slice(0, 5),
+                    parsedChartData.datasets[0].borderColor.slice(0, 5),
+                    parsedChartData.labels.slice(0, 5)
+                );
+            }
+        }
+
+        document.getElementById('loadingIndicator').style.display = 'none';
+    })
+    .catch(error => {
+        console.error('Error:', error);
+        document.getElementById('loadingIndicator').style.display = 'none';
+    });
+}
+fetch('/voice', {
+    method: 'POST',
+    body: formData
+})
+.then(response => response.text())
+.then(html => {
+    const parser = new DOMParser();
+    const newDocument = parser.parseFromString(html, 'text/html');
+    
+    
+    
+    // Update chart
+    const newCanvasElement = newDocument.querySelector('.bestSellers');
+    if (newCanvasElement) {
+        const newChartData = newCanvasElement.getAttribute('data-chart');
+        if (newChartData) {
+            const parsedChartData = JSON.parse(newChartData);
+            initializeChart(
+                parsedChartData.datasets[0].data.slice(0, 5),
+                parsedChartData.datasets[0].backgroundColor.slice(0, 5),
+                parsedChartData.datasets[0].borderColor.slice(0, 5),
+                parsedChartData.labels.slice(0, 5)
+            );
+        }
+    }
+
+    document.getElementById('loadingIndicator').style.display = 'none';
+})
+.catch(error => {
+    console.error('Error:', error);
+    document.getElementById('loadingIndicator').style.display = 'none';
+});

static/js/voice_backup.js

@@ -0,0 +1,115 @@
+const reset = document.getElementById("reset");
+const currentClassProbabilitiesList = document.getElementById("class-probabilities");
+const currentPredictedClass = document.getElementById('predicted-class')
+const staticDiv = document.getElementById("static");
+const dynamicDiv = document.getElementById("dynamic");
+var chartData;
+
+
+
+// Définir la fonction initializeChart en premier
+function initializeChart(data, backgroundColor, borderColor, labels) {
+  // Créer une nouvelle instance Chart.js pour chaque élément canvas avec la classe 'bestSellers'
+  data = data.map(function (element) {
+    return parseFloat(element).toFixed(2);
+  });
+  document.querySelectorAll('.bestSellers').forEach(function (canvas) {
+    // Initialiser le graphique
+    new Chart(canvas, {
+      type: 'doughnut', // Définir le type de graphique sur doughnut
+      data: {
+        datasets: [{
+          data: data,
+          backgroundColor: backgroundColor,
+          borderColor: borderColor,
+        }],
+        labels: labels
+      },
+      options: {
+        responsive: true, // Rendre le graphique responsive
+        cutoutPercentage: 80, // Définir le pourcentage de découpe
+        legend: {
+          display: false, // Masquer la légende
+        },
+        animation: {
+          animateScale: true,
+          animateRotate: true
+        },
+        plugins: {
+          datalabels: {
+            display: false,
+            align: 'center',
+            anchor: 'center'
+          }
+        }
+      }
+    });
+  });
+}
+document.addEventListener('DOMContentLoaded', function() {
+    loadResults();
+});
+function loadResults() {
+    fetch('/voice')
+        .then(response => response.text())
+        .then(html => {
+            const responseDOM = new DOMParser().parseFromString(html, "text/html");
+            const classProbabilitiesList = responseDOM.getElementById("class-probabilities");
+            currentClassProbabilitiesList.innerHTML = classProbabilitiesList.innerHTML;
+            const PredictedClass = responseDOM.getElementById("predicted-class")
+            currentPredictedClass.innerHTML = PredictedClass.innerHTML;
+
+            var canvasElement = responseDOM.querySelector('.bestSellers'); // Sélectionnez le premier élément avec la classe 'bestSellers'
+            console.log(canvasElement);
+            chartData = canvasElement.getAttribute('data-chart');
+            console.log(chartData);
+            var data = JSON.parse(chartData).datasets[0].data.slice(0, 5);
+            var backgroundColor = JSON.parse(chartData).datasets[0].backgroundColor.slice(0, 5);
+            var borderColor = JSON.parse(chartData).datasets[0].borderColor.slice(0, 5);
+            var labels = JSON.parse(chartData).labels.slice(0, 5);
+
+            // Créer de nouveaux graphiques
+            loadDashboardScript(data, backgroundColor, borderColor, labels);
+        })
+        .catch(error => console.error('Error:', error));
+}
+
+
+
+function loadDashboardScript(data, backgroundColor, borderColor, labels) { // Correction ici
+  var scriptElement = document.createElement('script');
+  scriptElement.type = 'text/javascript';
+  scriptElement.src = '../static/js/dashboard_pdf.js';
+  // Attendez que le script soit chargé avant d'appeler la fonction d'initialisation
+  scriptElement.onload = function () {
+    initializeChart(data, backgroundColor, borderColor, labels);
+  };
+  document.body.appendChild(scriptElement);
+}
+
+function destroyPreviousCharts() {
+  // Trouver tous les éléments canvas avec la classe 'bestSellers'
+  document.querySelectorAll('.bestSellers').forEach(function (canvas) {
+    // Récupérer l'instance du graphique
+    var chartInstance = Chart.getChart(canvas);
+    // Si une instance existe, détruire le graphique
+    if (chartInstance) {
+      chartInstance.destroy();
+    }
+  });
+}
+
+function createResponseElement(response) {
+  var pdfResponseElement = document.createElement('ul');
+  pdfResponseElement.classList.add('graph-legend-rectangle');
+  pdfResponseElement.innerHTML = response;
+  return pdfResponseElement;
+}
+
+// Modifiez la fonction reloadDashboardScript pour accepter les données supplémentaires
+function reloadDashboardScript(data, backgroundColor, borderColor, labels) {
+  var scriptElement = document.createElement('script');
+  scriptElement.type = 'text/javascript';
+  scriptElement.src = `../static/js/dashboard_pdf.js?data=${encodeURIComponent(JSON.stringify(data))}&backgroundColor=${encodeURIComponent(JSON.stringify(backgroundColor))}&borderColor=${encodeURIComponent(JSON.stringify(borderColor))}&labels=${encodeURIComponent(JSON.stringify(labels))}`;
+  document.body.appendChild(scriptElement);
+}

templates/pdf.html

@@ -42,6 +42,9 @@
                     <li class="nav-item nav-pills nav-pills-ocr">
                         <a class="nav-item nav-link " href="sentence">Text Classifier</a>
                     </li>
+                    <li class="nav-item nav-pills nav-pills-ocr">
+                        <a class="nav-item nav-link " href="voice">SLU</a>
+                    </li>
                 </ul>
             </div>
             <!-- Main navbar END -->

templates/sentence.html

@@ -44,6 +44,9 @@
                         <li class="nav-item nav-pills nav-pills-ocr">
                             <a class="nav-item nav-link active" href="sentence">Text Classifier</a>
                         </li>
+                        <li class="nav-item nav-pills nav-pills-ocr">
+                            <a class="nav-item nav-link " href="voice">SLU</a>
+                        </li>
                     </ul>
                 </div>
                 <!-- Main navbar END -->

templates/voice.html

@@ -0,0 +1,228 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Speech Transcription</title>
+    <link rel="icon" href="https://cdn-icons-png.flaticon.com/512/5262/5262072.png">
+    <link rel="stylesheet" href="../static/css/style2.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.3/font/bootstrap-icons.min.css">
+    <link rel="stylesheet" href="../static/css/vendor.bundle.base.css">
+</head>
+
+<body style="background-color: #1F2020;">
+    <nav class="navbar navbar-expand-lg bg-ocr mb-5">
+        <div class="container h-100">
+            <!-- Logo START -->
+            <a class="navbar-brand" href="pdf">
+                <img class="navbar-brand-item" src="../static/icons/avignon_universite_blanc_RVB.png" alt="logo">
+            </a>
+            <!-- Logo END -->
+
+            <!-- Responsive navbar toggler -->
+            <button class="navbar-toggler ms-auto icon-md btn btn-light p-0 collapsed" type="button"
+                data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse"
+                aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-animation">
+                    <span></span>
+                    <span></span>
+                    <span></span>
+                </span>
+            </button>
+
+            <!-- Main navbar START -->
+            <div class="navbar-collapse collapse" id="navbarCollapse">
+                <ul class="navbar-nav navbar-nav-scroll ms-auto p-xl-0">
+                    <li class="nav-item nav-pills nav-pills-ocr">
+                        <a class="nav-item nav-link  mb-5" href="pdf">PDF Classifier</a>
+                    </li>
+                    <li class="nav-item nav-pills nav-pills-ocr">
+                        <a class="nav-item nav-link " href="sentence">Text Classifier</a>
+                    </li>
+                    <li class="nav-item nav-pills nav-pills-ocr">
+                        <a class="nav-item nav-link active" href="voice">SLU</a>
+                    </li>
+                </ul>
+            </div>
+            <!-- Main navbar END -->
+        </div>
+    </nav>
+
+    <main class="pt-3 pt-md-5 pb-md-5 pt-lg-0 mt-3">
+        <div class="container mt-lg-7">
+            <div class="row">
+                <div class="col-lg-6 mb-4">
+                    <div class="card bg-ocr h-100">
+                        <div class="card-body">
+                            <h5 class="card-title text-white">Record Speech</h5>
+                            <p class="card-text text-white-50">Click the button to start recording your speech.</p>
+                            <div class="mt-auto text-center">
+                                <button id="startRecord" class="btn btn-ocr">Start Recording</button>
+                                <button id="stopRecord" class="btn btn-green" disabled>Stop Recording</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                <div class="col-lg-6 mb-4">
+                    <div class="card bg-ocr">
+                        <div class="card-body">
+                            <h5 class="card-title text-white">Upload Audio File</h5>
+                            <p class="card-text text-white-50">Click or drag and drop to upload an audio file.</p>
+                            <div class="file-upload-wrapper" id="fileUploadWrapper">
+                                <input type="file" id="audioFileInput" accept="audio/*">
+                                <div class="file-upload-text">
+                                    <i class="bi bi-cloud-upload" style="font-size: 2rem;"></i>
+                                    <p>Click or drag audio file here</p>
+                                    <p class="file-name"></p>
+                                </div>
+                            </div>
+                            <div class="text-center">
+                                <button id="uploadAudio" class="btn btn-ocr mt-3 text-center">Upload and
+                                    Transcribe</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            <h1 class="text-center text-white-50"> Results</h1>
+            <div class="card" id = 'static' style="background-color: #303131; border: 2px dashed rgb(82, 82, 82);">
+                <div class="card-body text-center" id="static">
+                    <div class="col-12 d-flex justify-content-center">
+                        <div class="col-sm-12 d-inline align-items-center" style="height: 175px; width: 229px;">
+                            <div class="flex-shrink-0 avatar avatar-lg  me-2 mb-3 mt-4">
+                                <img class="avatar-img rounded-circle"
+                                    src="../static/icons/logo_header_128x128.png" alt=""> 
+                            </div>
+                            <h5 class="card-title text-white-50">Get more insights about your speech 📣📢.</h5>
+                        </div>
+                        
+                    </div>
+                </div>
+            </div>
+            <div class="row d-none" id = 'dynamic'>
+                <div class="col-4">
+                    <div class="nav flex-column position-sticky top-10 shadow-lg" style="background-color: #222424;">
+                        <div class="card mb-3" style="background-color: #303131; border: 2px dashed rgb(82, 82, 82);">
+                            <div class="card-body">
+                                <h4 class="card-title text-white">Text's Category</h4>
+                                <div class="row d-flex">
+                                    <div class="col-sm-7 col-6">
+                                        <ul class="graphl-legend-rectangle" id="class-probabilities">
+                                            {% if class_probabilities %}
+                                                {% for class_label, probability in class_probabilities.items() %}
+                                                {% if loop.index <= 5 %}
+                                                <li class="text-white-50">
+                                                    <span class="bg-{{ class_label[1] }}"></span>
+                                                    <div class="d-flex justify-content-center">
+                                                        {{ class_label[0] }}:
+                                                        <span class="text-white w-100">&nbsp;{{ "%.2f" % (probability) }}%</span>
+                                                    </div>
+                                                </li>
+                                                {% endif %}
+                                                {% endfor %}
+                                            {% endif %}
+                                        </ul>
+                                    </div>
+                                    <div class="col-sm-5 grid-margin col-6">
+                                        <canvas class="bestSellers" data-chart='{{ chart_data | tojson}}'
+                                            id="bestSellers"></canvas>
+                                    </div>
+                                </div>
+                                <div class="mb-lg-0 text-white-50">
+                                    la classe la plus dominante est <span class="fw-bolder text-white"
+                                        id="predicted-class">{{
+                                        predicted_class[0] if predicted_class else ""
+                                        }}</span>
+                                </div>
+                            </div>
+                        </div>
+                        <div class="card my-auto mt-3"
+                            style="background-color: #303131; border: 2px dashed rgb(82, 82, 82);">
+                            <div class="card-body">
+                                <h4 class="card-title text-white">Legend</h4>
+                                <div class="row d-flex">
+                                    <div class="col-sm-6 col-6">
+                                        <ul class="graphl-legend-rectangle">
+                                            <li class="text-white-50"><span class="bg-info "></span>vehicles
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-success"></span>environments
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-danger"></span>energies
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-primary"></span>Physics
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-moss"></span>robotics
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-agri"></span>agriculture
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-yellow"></span>ML
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-warning"></span>economies
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-vanila"></span>technologies
+                                            </li>
+                                        </ul>
+                                    </div>
+                                    <div class="col-sm-6 col-6">
+                                        <ul class="graphl-legend-rectangle">
+
+                                            <li class="text-white-50"><span class="bg-coffe"></span>mathematics
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-orange "></span>sports
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-cyan"></span>AI
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-rosy"></span>Innovation
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-picton"></span>Science
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-purple"></span>Societies
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-pink"></span>administration
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-cambridge"></span>biology
+                                            </li>
+                                        </ul>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                
+                <div class="col-8">
+                    <div class="card" style="background-color: #303131; border: 2px dashed rgb(82, 82, 82);">
+                        <div class="card-body">
+                            <h5 class="card-title text-white mb-3">Transcribed and Classified Text</h5>
+                            <div id="transcribedText" class="text-white-50 mb-4"></div>
+                            <div id="static" class="text-center">
+                                <div class="col-12 d-flex justify-content-center">
+                                    <div class="col-sm-12 d-inline align-items-center" id="classifiedText">
+                                        {% if sentences_prediction %}
+                                            {% for sentence, color in sentences_prediction.items() %}
+                                            <span class="text-bold text-start bg-{{color[1]}}">
+                                                {{sentence}}
+                                            </span>
+                                            {% endfor %}
+                                        {% endif %}
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+        </div>
+    </main>
+
+    <div id="loadingIndicator" style="display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background-color: rgba(0,0,0,0.5); z-index: 9999;">
+        <div style="position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); color: white;">
+            Processing...
+        </div>
+    </div>
+
+    <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+    <script src="../static/js/voice.js" type="text/javascript"></script>
+    <script src="../static/js/vendor.bundle.base.js"></script>
+</body>
+
+</html>

templates/voice_backup.html

@@ -0,0 +1,209 @@
+<!DOCTYPE html>
+<html lang="en-US">
+
+<head>
+    <title>Speech Transcription</title>
+    <link rel="icon" href="https://cdn-icons-png.flaticon.com/512/5262/5262072.png">
+    <link rel="stylesheet" href="../static/css/style2.css">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bootstrap-icons@1.11.3/font/bootstrap-icons.min.css">
+    <link rel="stylesheet" href="../static/css/vendor.bundle.base.css">
+    <style>
+
+    </style>
+</head>
+
+<body style="background-color: #1F2020;">
+    <nav class="navbar navbar-expand-lg bg-ocr mb-5">
+        <div class="container h-100">
+            <!-- Logo START -->
+            <a class="navbar-brand" href="pdf">
+                <img class="navbar-brand-item" src="../static/icons/avignon_universite_blanc_RVB.png" alt="logo">
+            </a>
+            <!-- Logo END -->
+
+            <!-- Responsive navbar toggler -->
+            <button class="navbar-toggler ms-auto icon-md btn btn-light p-0 collapsed" type="button"
+                data-bs-toggle="collapse" data-bs-target="#navbarCollapse" aria-controls="navbarCollapse"
+                aria-expanded="false" aria-label="Toggle navigation">
+                <span class="navbar-toggler-animation">
+                    <span></span>
+                    <span></span>
+                    <span></span>
+                </span>
+            </button>
+
+            <!-- Main navbar START -->
+            <div class="navbar-collapse collapse" id="navbarCollapse">
+                <ul class="navbar-nav navbar-nav-scroll ms-auto p-xl-0">
+                    <!-- Nav item 1 Demos -->
+                    <li class="nav-item nav-pills nav-pills-ocr">
+                        <a class="nav-item nav-link  mb-5" href="pdf">PDF Classifier</a>
+                    </li>
+                    <!-- Nav item 2 Pages -->
+                    <li class="nav-item nav-pills nav-pills-ocr">
+                        <a class="nav-item nav-link " href="sentence">Text Classifier</a>
+                    </li>
+                    <li class="nav-item nav-pills nav-pills-ocr">
+                        <a class="nav-item nav-link active" href="voice">SLU</a>
+                    </li>
+                </ul>
+            </div>
+            <!-- Main navbar END -->
+        </div>
+    </nav>
+
+    <main class="pt-3 pt-md-5 pb-md-5 pt-lg-0 mt-3">
+        <div class="container mt-lg-7">
+            <div class="row">
+                <div class="col-lg-6 mb-4">
+                    <div class="card bg-ocr h-100">
+                        <div class="card-body">
+                            <h5 class="card-title text-white">Record Speech</h5>
+                            <p class="card-text text-white-50">Click the button to start recording your speech.</p>
+                            <div class="mt-auto text-center">
+                                <button id="startRecord" class="btn btn-ocr">Start Recording</button>
+                                <button id="stopRecord" class="btn btn-green" disabled>Stop Recording</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                <div class="col-lg-6 mb-4">
+                    <div class="card bg-ocr">
+                        <div class="card-body">
+                            <h5 class="card-title text-white">Upload Audio File</h5>
+                            <p class="card-text text-white-50">Click or drag and drop to upload an audio file.</p>
+                            <div class="file-upload-wrapper" id="fileUploadWrapper">
+                                <input type="file" id="audioFileInput" accept="audio/*">
+                                <div class="file-upload-text">
+                                    <i class="bi bi-cloud-upload" style="font-size: 2rem;"></i>
+                                    <p>Click or drag audio file here</p>
+                                    <p class="file-name"></p>
+                                </div>
+                            </div>
+                            <div class="text-center">
+                                <button id="uploadAudio" class="btn btn-ocr mt-3 text-center">Upload and
+                                    Transcribe</button>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+            <h1 class="text-center text-white-50"> Results</h1>
+            <div class="row">
+                <div class="col-4">
+                    <div class="nav flex-column position-sticky top-10 shadow-lg" style="background-color: #222424;">
+                        <div class="card mb-3" style="background-color: #303131; border: 2px dashed rgb(82, 82, 82);">
+                            <div class="card-body">
+                                <h4 class="card-title text-white">Text's Category</h4>
+                                <div class="row d-flex">
+                                    <div class="col-sm-7 col-6">
+                                        <ul class="graphl-legend-rectangle" id="class-probabilities">
+                                            {% for class_label, probability in class_probabilities.items() %}
+                                            {% if loop.index <= 5 %} <li class="text-white-50">
+                                                <span class="bg-{{ class_label[1] }}"></span>
+                                                <div class="d-flex justify-content-center">
+                                                    {{ class_label[0] }}:
+                                                    <span class="text-white w-100">&nbsp;{{ "%.2f" % probability
+                                                        }}%</span>
+                                                </div>
+                                                </li>
+                                                {% endif %}
+                                                {% endfor %}
+                                        </ul>
+                                    </div>
+                                    <div class="col-sm-5 grid-margin col-6">
+                                        <canvas class="bestSellers" data-chart='{{ chart_data | tojson }}'
+                                            id="bestSellers#"></canvas>
+                                    </div>
+                                </div>
+                                <div class="mb-lg-0 text-white-50">
+                                    la classe la plus dominante est <span class="fw-bolder text-white"
+                                        id="predicted-class">{{
+                                        predicted_class[0]
+                                        }}</span>
+                                </div>
+                            </div>
+                        </div>
+                        <div class="card my-auto mt-3"
+                            style="background-color: #303131; border: 2px dashed rgb(82, 82, 82);">
+                            <div class="card-body">
+                                <h4 class="card-title text-white">Legend</h4>
+                                <div class="row d-flex">
+                                    <div class="col-sm-6 col-6">
+                                        <ul class="graphl-legend-rectangle">
+                                            <li class="text-white-50"><span class="bg-info "></span>vehicles
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-success"></span>environments
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-danger"></span>energies
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-primary"></span>Physics
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-moss"></span>robotics
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-agri"></span>agriculture
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-yellow"></span>ML
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-warning"></span>economies
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-vanila"></span>technologies
+                                            </li>
+                                        </ul>
+                                    </div>
+                                    <div class="col-sm-6 col-6">
+                                        <ul class="graphl-legend-rectangle">
+
+                                            <li class="text-white-50"><span class="bg-coffe"></span>mathematics
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-orange "></span>sports
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-cyan"></span>AI
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-rosy"></span>Innovation
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-picton"></span>Science
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-purple"></span>Societies
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-pink"></span>administration
+                                            </li>
+                                            <li class="text-white-50"><span class="bg-cambridge"></span>biology
+                                            </li>
+                                        </ul>
+                                    </div>
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+                
+                <div class="col-8">
+                    <div class="card" style="background-color: #303131; border: 2px dashed rgb(82, 82, 82);">
+                        <div class="card-body text-center" id="static">
+                            <div class="col-12 d-flex justify-content-center">
+                                <div class="col-sm-12 d-inline align-items-center">
+                                    {% for sentence , color in sentences_prediction.items() %}
+                                    <span class="text-bold text-start bg-{{color[1]}}">
+                                        {{sentence}}
+                                    </span>
+                                    {% endfor %}
+                                </div>
+
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+        </div>
+    </main>
+    <!-- Container END -->
+    <script src="../static/js/dashboard_voice.js"></script>
+    <script src="../static/js/Chart.min.js"></script>
+    <script src="../static/js/voice.js" type="text/javascript"></script>
+    <script src="../static/js/vendor.bundle.base.js"></script>
+
+</body>
+
+</html>