added emotion analysis

app.py

@@ -8,7 +8,7 @@ import base64
 from deepface import DeepFace
 import torchaudio
 import moviepy.editor as mp
-from transformers import WhisperProcessor, WhisperForConditionalGeneration
+from transformers import WhisperProcessor, WhisperForConditionalGeneration, pipeline
 
 # import pathlib
 # temp = pathlib.PosixPath
@@ -23,6 +23,8 @@ backends = [
   'mediapipe'
 ]
 
+emotion_pipeline = pipeline("text-classification", model="cardiffnlp/twitter-roberta-base-emotion")
+
 def getTranscription(path):
     # Insert Local Video File Path
     clip = mp.VideoFileClip(path)
@@ -51,6 +53,10 @@ def getTranscription(path):
 
 model = load_learner("gaze-recognizer-v3.pkl")
 
+def analyze_emotion(text):
+    result = emotion_pipeline(text)
+    return result
+
 def video_processing(video_file, encoded_video):
     angry = 0
     disgust = 0
@@ -74,6 +80,8 @@ def video_processing(video_file, encoded_video):
 
     transcription = getTranscription(video_file)
     print(transcription)
+    text_emotion = analyze_emotion(transcription)
+    print(text_emotion)
     
     video_capture = cv2.VideoCapture(video_file)
     on_camera = 0

optimized.py

@@ -0,0 +1,97 @@
+import base64
+import cv2
+import face_recognition
+import gradio as gr
+import moviepy.editor as mp
+import os
+import time
+import torchaudio
+from fastai.vision.all import load_learner
+from transformers import WhisperProcessor, WhisperForConditionalGeneration, pipeline
+
+emotion_pipeline = pipeline("text-classification", model="cardiffnlp/twitter-roberta-base-emotion")
+
+model = load_learner("gaze-recognizer-v3.pkl")
+
+def extract_audio(video_path):
+    clip = mp.VideoFileClip(video_path)
+    clip.audio.write_audiofile("audio.wav")
+
+def get_transcription(path):
+    extract_audio(path)
+
+    waveform, sample_rate = torchaudio.load("audio.wav")
+    resampler = torchaudio.transforms.Resample(sample_rate, 16000)
+    waveform = resampler(waveform)[0]
+
+    processor = WhisperProcessor.from_pretrained("openai/whisper-tiny")
+    model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny")
+    model.config.forced_decoder_ids = None
+
+    input_features = processor(waveform.squeeze(dim=0), return_tensors="pt").input_features 
+    predicted_ids = model.generate(input_features)
+
+    transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True)
+    return transcription[0]
+
+def analyze_emotion(text):
+    result = emotion_pipeline(text)
+    return result
+
+def process_frame(frame):
+    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+    face_locations = face_recognition.face_locations(gray)
+
+    if len(face_locations) > 0:
+        for top, right, bottom, left in face_locations:
+            face_image = gray[top:bottom, left:right]
+            resized_face_image = cv2.resize(face_image, (128, 128))
+            result = model.predict(resized_face_image)
+
+            return result[0]
+
+    return None
+
+def video_processing(video_file, encoded_video):
+    if encoded_video != "":
+        decoded_file_data = base64.b64decode(encoded_video)
+        with open("temp_video.mp4", "wb") as f:
+            f.write(decoded_file_data)
+        video_file = "temp_video.mp4"
+
+    transcription = get_transcription(video_file)
+    print(transcription)
+
+    video_capture = cv2.VideoCapture(video_file)
+    on_camera = 0
+    off_camera = 0
+    total = 0
+    emotions = []
+
+    while True:
+        for _ in range(24 * 3):
+            ret, frame = video_capture.read()
+            if not ret:
+                break
+
+        if not ret:
+            break
+
+        result = process_frame(frame)
+        if result:
+            if result == 'on_camera':
+                on_camera += 1
+            elif result == 'off_camera':
+                off_camera += 1
+            total += 1
+
+        emotion_results = analyze_emotion(transcription)
+        emotions.append(emotion_results)
+
+    video_capture.release()
+    cv2.destroyAllWindows()
+
+    if os.path.exists("temp_video.mp4"):
+        os.remove("temp_video.mp4")
+
+    gaze_percentage = on_camera / total * 100 if total > 0