Update app.py

app.py

@@ -2,6 +2,47 @@ import gradio as gr
 import onnxruntime
 from transformers import AutoTokenizer
 import torch
+import os
+from transformers import pipeline
+### --- Audio/Video to txt ---###
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+pipe = pipeline("automatic-speech-recognition", 
+                    model="openai/whisper-base.en",
+                    chunk_length_s=30, device=device)
+
+summarizer = pipeline("summarization", model="facebook/bart-large-cnn", device=device)
+     
+
+def video_identity(video):
+    transcription = pipe(video)["text"]
+    return transcription
+
+def summary(text):
+    text = text.split('.')
+    max_chunk = 500
+    current_chunk = 0
+    chunks = []
+    
+    
+    for t in text:
+        if len(chunks) == current_chunk + 1:
+            if len(chunks[current_chunk]) + len(t.split(' ')) <= max_chunk:
+                chunks[current_chunk].extend(t.split(' '))
+            else:
+                current_chunk += 1
+                chunks.append(t.split(' '))
+        else:
+          chunks.append(t.split(' '))
+    
+    for chunk in range(len(chunks)):
+        chunks[chunk] =' '.join(chunks[chunk])
+    
+    summ = summarizer(chunks,max_length = 100)
+
+    return summ
+    
+    
+
 
 
 token  = AutoTokenizer.from_pretrained('distilroberta-base')
@@ -12,16 +53,21 @@ output_name = inf_session.get_outputs()[0].name
 
 classes = ['Art', 'Astrology', 'Biology', 'Chemistry', 'Economics', 'History', 'Literature', 'Philosophy', 'Physics', 'Politics', 'Psychology', 'Sociology']
 
-def classify(vid,review):
-    input_ids = token(review)['input_ids'][:512]
+def classify(vid):
+    full_text = video_identity(vid)
+    sum = summary(full_text)[0]['summary_text']
+
+    
+    input_ids = token(sum)['input_ids'][:512]
     logits = inf_session.run([output_name],{input_name : [input_ids]})[0]
     logits = torch.FloatTensor(logits)
     probs = torch.sigmoid(logits)[0]
-    x = 2
-    return dict(zip(classes,map(float,probs)))
+    return full_text, sum, dict(zip(classes,map(float,probs)))
 
-label = gr.outputs.Label(num_top_classes=5)
-iface = gr.Interface(fn=classify,inputs=[gr.Video(),'text'],outputs = label)
+# label = gr.outputs.Label(num_top_classes=5)
+iface = gr.Interface(fn=classify,
+                     inputs=gr.inputs.Audio(source="upload", type="filepath"),
+                     outputs = ['text','text',gr.outputs.Label(num_top_classes=3)])
 iface.launch(inline=False)