Spaces:
Runtime error
Runtime error
Analyze frames from video
Browse files- app.py +54 -42
- requirements.txt +2 -1
app.py
CHANGED
@@ -1,9 +1,10 @@
|
|
1 |
import os
|
2 |
-
import
|
3 |
from dataclasses import dataclass
|
4 |
from time import sleep
|
5 |
-
from typing import Dict, List, Generator
|
6 |
|
|
|
7 |
import gradio as gr
|
8 |
from openai import OpenAI
|
9 |
from dotenv import load_dotenv
|
@@ -58,22 +59,48 @@ class MockInterviewer:
|
|
58 |
model='whisper-1',
|
59 |
file=file,
|
60 |
)
|
|
|
61 |
os.remove(video)
|
62 |
config = Config(job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count)
|
63 |
-
response = self._chat(transcriptions.text, config)
|
64 |
return [(transcriptions.text, response)]
|
65 |
|
66 |
def clear_thread(self) -> None:
|
67 |
print('Initializing new thread')
|
68 |
self._thread = self._client.beta.threads.create()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
69 |
|
70 |
-
def _chat(self, message: str, config: Config) -> str:
|
71 |
print('Started chat')
|
72 |
assistant_id = self._init_assistant(config)
|
73 |
-
return self._send_message(message, assistant_id)
|
74 |
|
75 |
-
def _send_message(self, message: str, assistant_id: str) -> str:
|
76 |
-
self._client.beta.threads.messages.create(thread_id=self._thread.id, role='user', content=message)
|
77 |
print('Message created')
|
78 |
run = self._client.beta.threads.runs.create(thread_id=self._thread.id, assistant_id=assistant_id)
|
79 |
print('Run created')
|
@@ -92,21 +119,7 @@ class MockInterviewer:
|
|
92 |
response = messages.data[0].content[0].text.value
|
93 |
print(f'Assistant response: {response}')
|
94 |
return response
|
95 |
-
|
96 |
-
def _create_files(self, company: str) -> List[str]:
|
97 |
-
if company.lower() == 'amazon':
|
98 |
-
url = 'https://www.aboutamazon.com/about-us/leadership-principles'
|
99 |
-
filename = 'leadership_principles.html'
|
100 |
-
else:
|
101 |
-
return []
|
102 |
-
|
103 |
-
filename, headers = urllib.request.urlretrieve(url, filename)
|
104 |
-
with open(filename, 'rb') as file:
|
105 |
-
assistant_file = self._client.files.create(file=file, purpose='assistants')
|
106 |
-
file_ids = [assistant_file.id]
|
107 |
-
os.remove(filename)
|
108 |
-
return file_ids
|
109 |
-
|
110 |
def _init_assistant(self, config: Config) -> str:
|
111 |
cache_key = config
|
112 |
if cache_key in self._assistant_id_cache:
|
@@ -114,38 +127,38 @@ class MockInterviewer:
|
|
114 |
return self._assistant_id_cache.get(cache_key)
|
115 |
else:
|
116 |
print(f'Initializing new assistant for key {cache_key}')
|
117 |
-
file_ids = self._create_files(config.company)
|
118 |
-
|
119 |
assistant = self._client.beta.assistants.create(
|
120 |
name='Mock Interviewer',
|
121 |
instructions=self._generate_assistant_instructions(config),
|
122 |
-
model='gpt-4-0125-preview'
|
123 |
-
tools=[
|
124 |
-
{
|
125 |
-
'type': 'retrieval' # This adds the knowledge base as a tool
|
126 |
-
}
|
127 |
-
],
|
128 |
-
file_ids=file_ids)
|
129 |
|
130 |
self._assistant_id_cache[cache_key] = assistant.id
|
131 |
return assistant.id
|
132 |
|
133 |
def _generate_assistant_instructions(self, config: Config) -> str:
|
134 |
if config.job_role and config.company:
|
135 |
-
purpose = f'You are
|
136 |
elif config.job_role:
|
137 |
-
purpose = f'You are
|
138 |
elif config.company:
|
139 |
-
purpose = f'You are
|
140 |
else:
|
141 |
-
purpose = 'You are
|
142 |
|
143 |
if config.job_description:
|
144 |
specifics = f'Tailor your questions based on the following job posting: {config.job_description}.'
|
145 |
else:
|
146 |
specifics = ''
|
147 |
|
148 |
-
return f
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
149 |
|
150 |
mock_interviewer = MockInterviewer()
|
151 |
|
@@ -170,8 +183,6 @@ with gr.Blocks(theme=theme) as demo:
|
|
170 |
placeholder='Key job responsibilities, basic qualifications, preferred qualifications, about the company, etc.'
|
171 |
)
|
172 |
with gr.Accordion("Question Preferences", open=False):
|
173 |
-
label='Question Type and Count'
|
174 |
-
info='Please indicate how many questions you would like asked on the following question types:',
|
175 |
behavioral_count = gr.Slider(label="Behavioral", maximum=10, value=1, step=1)
|
176 |
technical_count = gr.Slider(label="Technical", maximum=10, value=1, step=1)
|
177 |
situational_count = gr.Slider(label="Situational", maximum=10, value=1, step=1)
|
@@ -180,20 +191,21 @@ with gr.Blocks(theme=theme) as demo:
|
|
180 |
with gr.Column(variant='panel', scale=6):
|
181 |
chat_interface = gr.ChatInterface(
|
182 |
fn=mock_interviewer.chat_with_text,
|
183 |
-
title="Hi! I'm Ami, your AI Mock Interviewer.",
|
184 |
-
description='You can begin by clicking record and introducing yourself!',
|
185 |
additional_inputs=[job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
|
186 |
retry_btn=None,
|
187 |
undo_btn=None)
|
188 |
|
|
|
|
|
189 |
chat_interface.load(mock_interviewer.clear_thread)
|
190 |
chat_interface.clear_btn.click(mock_interviewer.clear_thread)
|
191 |
-
|
|
|
192 |
video = gr.Video(sources='webcam', include_audio=True)
|
193 |
video.stop_recording(fn=mock_interviewer.chat_with_video,
|
194 |
inputs=[video, job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
|
195 |
outputs=[chat_interface.chatbot],
|
196 |
-
api_name=False).then(lambda:None, None, video, queue=False)
|
197 |
|
198 |
if __name__ == '__main__':
|
199 |
demo.launch()
|
|
|
1 |
import os
|
2 |
+
import random
|
3 |
from dataclasses import dataclass
|
4 |
from time import sleep
|
5 |
+
from typing import Dict, List, Generator, Optional
|
6 |
|
7 |
+
import cv2
|
8 |
import gradio as gr
|
9 |
from openai import OpenAI
|
10 |
from dotenv import load_dotenv
|
|
|
59 |
model='whisper-1',
|
60 |
file=file,
|
61 |
)
|
62 |
+
video_frame_file_ids = self._extract_frames(video)
|
63 |
os.remove(video)
|
64 |
config = Config(job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count)
|
65 |
+
response = self._chat(transcriptions.text, config, video_frame_file_ids)
|
66 |
return [(transcriptions.text, response)]
|
67 |
|
68 |
def clear_thread(self) -> None:
|
69 |
print('Initializing new thread')
|
70 |
self._thread = self._client.beta.threads.create()
|
71 |
+
|
72 |
+
def _extract_frames(self, video_path: str) -> List[str]:
|
73 |
+
video = cv2.VideoCapture(video_path)
|
74 |
+
num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
|
75 |
+
|
76 |
+
counter = 0
|
77 |
+
for i in random.sample(range(num_frames), 10):
|
78 |
+
video.set(cv2.CAP_PROP_FRAME_COUNT, i)
|
79 |
+
success, frame = video.read()
|
80 |
+
if not success:
|
81 |
+
print('Error in video frame extraction')
|
82 |
+
break
|
83 |
+
cv2.imwrite(f'{counter}.jpg', frame)
|
84 |
+
counter += 1
|
85 |
+
|
86 |
+
video.release()
|
87 |
+
|
88 |
+
file_ids = []
|
89 |
+
for i in range(counter):
|
90 |
+
with open(f'{i}.jpg', 'rb') as image:
|
91 |
+
file = self._client.files.create(file=image, purpose='assistants')
|
92 |
+
file_ids.append(file.id)
|
93 |
+
os.remove(f'{i}.jpg')
|
94 |
+
|
95 |
+
return file_ids
|
96 |
|
97 |
+
def _chat(self, message: str, config: Config, video_frame_file_ids: List[str] = list()) -> str:
|
98 |
print('Started chat')
|
99 |
assistant_id = self._init_assistant(config)
|
100 |
+
return self._send_message(message, assistant_id, video_frame_file_ids)
|
101 |
|
102 |
+
def _send_message(self, message: str, assistant_id: str, video_frame_file_ids: List[str]) -> str:
|
103 |
+
self._client.beta.threads.messages.create(thread_id=self._thread.id, role='user', content=message, file_ids=video_frame_file_ids)
|
104 |
print('Message created')
|
105 |
run = self._client.beta.threads.runs.create(thread_id=self._thread.id, assistant_id=assistant_id)
|
106 |
print('Run created')
|
|
|
119 |
response = messages.data[0].content[0].text.value
|
120 |
print(f'Assistant response: {response}')
|
121 |
return response
|
122 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
123 |
def _init_assistant(self, config: Config) -> str:
|
124 |
cache_key = config
|
125 |
if cache_key in self._assistant_id_cache:
|
|
|
127 |
return self._assistant_id_cache.get(cache_key)
|
128 |
else:
|
129 |
print(f'Initializing new assistant for key {cache_key}')
|
|
|
|
|
130 |
assistant = self._client.beta.assistants.create(
|
131 |
name='Mock Interviewer',
|
132 |
instructions=self._generate_assistant_instructions(config),
|
133 |
+
model='gpt-4-0125-preview')
|
|
|
|
|
|
|
|
|
|
|
|
|
134 |
|
135 |
self._assistant_id_cache[cache_key] = assistant.id
|
136 |
return assistant.id
|
137 |
|
138 |
def _generate_assistant_instructions(self, config: Config) -> str:
|
139 |
if config.job_role and config.company:
|
140 |
+
purpose = f'You are AiMI, an AI mock interviewer for {config.job_role} roles at {config.company}.'
|
141 |
elif config.job_role:
|
142 |
+
purpose = f'You are AiMI, an AI mock interviewer for {config.job_role} roles.'
|
143 |
elif config.company:
|
144 |
+
purpose = f'You are AiMI, an AI mock interviewer for roles at {config.company}.'
|
145 |
else:
|
146 |
+
purpose = 'You are AiMI, an AI mock interviewer.'
|
147 |
|
148 |
if config.job_description:
|
149 |
specifics = f'Tailor your questions based on the following job posting: {config.job_description}.'
|
150 |
else:
|
151 |
specifics = ''
|
152 |
|
153 |
+
return f'''
|
154 |
+
{purpose} Please greet the candidate and begin the mock interview when the candidate sends you the first message. {specifics} Ask {config.behavioral_count} number of behavioral questions, {config.technical_count} number of technical questions, {config.situational_count} number of situational questions, and {config.case_count} number of case-like questions, one question at a time.
|
155 |
+
|
156 |
+
After the candidate gives a response, evaluate the response of the candidate by addressing the candidate as if you were giving feedback to them (i.e. address them as you). Keep in mind what your company values in candidates (if you have been assigned a company). Provide a detailed analysis of the candidate's response based on the question type. In your feedback, comment on 1) avoiding filler words and non-words such as um or like, 2) avoiding jargon, and 3) flow (ideas flow logically with clear transitions between main ideas).
|
157 |
+
|
158 |
+
The candidate may have included frames from a video recording of their response. If so, please analyze the provided images from a mock interview setting, focusing on the following key aspects to evaluate the subject's presentation and the interview environment. Provide recommendations for improvement (limit observations to be brief). Focus on these nonverbal criteria: Facial Expressions: Assess the subject's facial expressions, considering if they convey confidence, engagement, and professionalism. Offer insights into how facial expressions could impact the interviewer's perception. Energy: If they appear energetic (conveying energy to engage viewers). Please provide detailed feedback on each aspect, including what is done well and what could be enhanced to improve the subject's presentation and the overall interview setting. Also comment on the following briefly only if it really needs improvement. It is not necessary to comment on the following, only if it needs improvement: Lighting: Describe the quality and direction of the lighting in the image. Note whether the subject is well-lit, if there are any harsh shadows on the face, and if the background is appropriately illuminated. Apparel: Comment on the appropriateness of the subject's attire for a professional interview. Mention the colors, fit, and formality of the clothing, and suggest any adjustments if needed. Speaking Environment/Background: Analyze the speaking environment and background for any distractions or elements that could detract from the focus on the subject. Recommend changes to create a more neutral and professional background. Limit your complete comments on the candidate's video to 100 words.
|
159 |
+
|
160 |
+
Finally, rate the complete response (content and video) on a scale from 1 to 10, where 1 is inadequate and 10 is exceptional.
|
161 |
+
'''
|
162 |
|
163 |
mock_interviewer = MockInterviewer()
|
164 |
|
|
|
183 |
placeholder='Key job responsibilities, basic qualifications, preferred qualifications, about the company, etc.'
|
184 |
)
|
185 |
with gr.Accordion("Question Preferences", open=False):
|
|
|
|
|
186 |
behavioral_count = gr.Slider(label="Behavioral", maximum=10, value=1, step=1)
|
187 |
technical_count = gr.Slider(label="Technical", maximum=10, value=1, step=1)
|
188 |
situational_count = gr.Slider(label="Situational", maximum=10, value=1, step=1)
|
|
|
191 |
with gr.Column(variant='panel', scale=6):
|
192 |
chat_interface = gr.ChatInterface(
|
193 |
fn=mock_interviewer.chat_with_text,
|
|
|
|
|
194 |
additional_inputs=[job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
|
195 |
retry_btn=None,
|
196 |
undo_btn=None)
|
197 |
|
198 |
+
chat_interface.chatbot.value = [(None, "Hi! I'm AiMI, your AI Mock Interviewer. You can begin by clicking record and introducing yourself!")]
|
199 |
+
chat_interface.chatbot.height = '70vh'
|
200 |
chat_interface.load(mock_interviewer.clear_thread)
|
201 |
chat_interface.clear_btn.click(mock_interviewer.clear_thread)
|
202 |
+
|
203 |
+
with gr.Column(variant='panel', scale=1):
|
204 |
video = gr.Video(sources='webcam', include_audio=True)
|
205 |
video.stop_recording(fn=mock_interviewer.chat_with_video,
|
206 |
inputs=[video, job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
|
207 |
outputs=[chat_interface.chatbot],
|
208 |
+
api_name=False).then(lambda: None, None, video, queue=False)
|
209 |
|
210 |
if __name__ == '__main__':
|
211 |
demo.launch()
|
requirements.txt
CHANGED
@@ -1,3 +1,4 @@
|
|
1 |
openai==1.16.2
|
2 |
gradio==4.25.0
|
3 |
-
python-dotenv==1.0.1
|
|
|
|
1 |
openai==1.16.2
|
2 |
gradio==4.25.0
|
3 |
+
python-dotenv==1.0.1
|
4 |
+
opencv-python==4.9.0.80
|