import base64 import cv2 import face_recognition import gradio as gr import moviepy.editor as mp import os import time import torchaudio from fastai.vision.all import load_learner from transformers import WhisperProcessor, WhisperForConditionalGeneration, pipeline emotion_pipeline = pipeline("text-classification", model="cardiffnlp/twitter-roberta-base-emotion") model = load_learner("gaze-recognizer-v3.pkl") def extract_audio(video_path): clip = mp.VideoFileClip(video_path) clip.audio.write_audiofile("audio.wav") def get_transcription(path): extract_audio(path) waveform, sample_rate = torchaudio.load("audio.wav") resampler = torchaudio.transforms.Resample(sample_rate, 16000) waveform = resampler(waveform)[0] processor = WhisperProcessor.from_pretrained("openai/whisper-tiny") model = WhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny") model.config.forced_decoder_ids = None input_features = processor(waveform.squeeze(dim=0), return_tensors="pt").input_features predicted_ids = model.generate(input_features) transcription = processor.batch_decode(predicted_ids, skip_special_tokens=True) return transcription[0] def analyze_emotion(text): result = emotion_pipeline(text) return result def process_frame(frame): gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) face_locations = face_recognition.face_locations(gray) if len(face_locations) > 0: for top, right, bottom, left in face_locations: face_image = gray[top:bottom, left:right] resized_face_image = cv2.resize(face_image, (128, 128)) result = model.predict(resized_face_image) return result[0] return None def video_processing(video_file, encoded_video): if encoded_video != "": decoded_file_data = base64.b64decode(encoded_video) with open("temp_video.mp4", "wb") as f: f.write(decoded_file_data) video_file = "temp_video.mp4" transcription = get_transcription(video_file) print(transcription) video_capture = cv2.VideoCapture(video_file) on_camera = 0 off_camera = 0 total = 0 emotions = [] while True: for _ in range(24 * 3): ret, frame = video_capture.read() if not ret: break if not ret: break result = process_frame(frame) if result: if result == 'on_camera': on_camera += 1 elif result == 'off_camera': off_camera += 1 total += 1 emotion_results = analyze_emotion(transcription) emotions.append(emotion_results) video_capture.release() cv2.destroyAllWindows() if os.path.exists("temp_video.mp4"): os.remove("temp_video.mp4") gaze_percentage = on_camera / total * 100 if total > 0