Spaces:
Running
Running
File size: 2,838 Bytes
ab4fd59 a22f055 aa35242 18a71f5 aa35242 a22f055 aa35242 a22f055 aa35242 18a71f5 aa35242 18a71f5 aa35242 18a71f5 aa35242 18a71f5 aa35242 18a71f5 aa35242 18a71f5 aa35242 18a71f5 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 |
import gradio as gr
from transformers import LlamaTokenizer, LlamaForCausalLM
import tempfile
import numpy as np
# Initialize LLaMA Model for Question Answering
llama_tokenizer = LlamaTokenizer.from_pretrained('huggingface/llama-7b')
llama_model = LlamaForCausalLM.from_pretrained('huggingface/llama-7b')
# Updated transcribe_and_predict_video function from your code
def transcribe_and_predict_video(video):
# Process video frames for image-based emotion recognition
image_emotion = process_video(video)
# Process audio for text and audio-based emotion recognition
text_emotion, audio_emotion = process_audio_from_video(video)
# Determine the overall emotion (could be based on majority vote or some other logic)
overall_emotion = Counter([text_emotion, audio_emotion, image_emotion]).most_common(1)[0][0]
return overall_emotion
# Emotion-aware Question Answering with LLM
def emotion_aware_qa(question, video):
# Get the emotion from the video (this uses the emotion detection you already implemented)
detected_emotion = transcribe_and_predict_video(video)
# Create a custom response context based on the detected emotion
if detected_emotion == 'joy':
emotion_context = "You're in a good mood! Let's keep the positivity going."
elif detected_emotion == 'sadness':
emotion_context = "It seems like you're feeling a bit down. Let me help with that."
elif detected_emotion == 'anger':
emotion_context = "I sense some frustration. Let's work through it together."
elif detected_emotion == 'fear':
emotion_context = "It sounds like you're anxious. How can I assist in calming things down?"
elif detected_emotion == 'neutral':
emotion_context = "You're feeling neutral. How can I help you today?"
else:
emotion_context = "You're in an uncertain emotional state. Let me guide you."
# Prepare the prompt for LLaMA, including emotion context and user question
prompt = f"{emotion_context} User asks: {question}"
# Tokenize and generate response from LLaMA
inputs = llama_tokenizer(prompt, return_tensors="pt")
outputs = llama_model.generate(inputs['input_ids'], max_length=150)
answer = llama_tokenizer.decode(outputs[0], skip_special_tokens=True)
return answer
# Create Gradio interface to interact with the LLM and video emotion detection
def gradio_interface(question, video):
response = emotion_aware_qa(question, video)
return response
iface = gr.Interface(fn=gradio_interface,
inputs=["text", gr.Video()],
outputs="text",
title="Emotion-Aware Question Answering",
description="Ask a question and get an emotion-aware response based on the video.")
iface.launch()
|