Letsch22 commited on
Commit
435d273
1 Parent(s): 010c52a

Analyze frames from video

Browse files
Files changed (2) hide show
  1. app.py +54 -42
  2. requirements.txt +2 -1
app.py CHANGED
@@ -1,9 +1,10 @@
1
  import os
2
- import urllib.request
3
  from dataclasses import dataclass
4
  from time import sleep
5
- from typing import Dict, List, Generator
6
 
 
7
  import gradio as gr
8
  from openai import OpenAI
9
  from dotenv import load_dotenv
@@ -58,22 +59,48 @@ class MockInterviewer:
58
  model='whisper-1',
59
  file=file,
60
  )
 
61
  os.remove(video)
62
  config = Config(job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count)
63
- response = self._chat(transcriptions.text, config)
64
  return [(transcriptions.text, response)]
65
 
66
  def clear_thread(self) -> None:
67
  print('Initializing new thread')
68
  self._thread = self._client.beta.threads.create()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
 
70
- def _chat(self, message: str, config: Config) -> str:
71
  print('Started chat')
72
  assistant_id = self._init_assistant(config)
73
- return self._send_message(message, assistant_id)
74
 
75
- def _send_message(self, message: str, assistant_id: str) -> str:
76
- self._client.beta.threads.messages.create(thread_id=self._thread.id, role='user', content=message)
77
  print('Message created')
78
  run = self._client.beta.threads.runs.create(thread_id=self._thread.id, assistant_id=assistant_id)
79
  print('Run created')
@@ -92,21 +119,7 @@ class MockInterviewer:
92
  response = messages.data[0].content[0].text.value
93
  print(f'Assistant response: {response}')
94
  return response
95
-
96
- def _create_files(self, company: str) -> List[str]:
97
- if company.lower() == 'amazon':
98
- url = 'https://www.aboutamazon.com/about-us/leadership-principles'
99
- filename = 'leadership_principles.html'
100
- else:
101
- return []
102
-
103
- filename, headers = urllib.request.urlretrieve(url, filename)
104
- with open(filename, 'rb') as file:
105
- assistant_file = self._client.files.create(file=file, purpose='assistants')
106
- file_ids = [assistant_file.id]
107
- os.remove(filename)
108
- return file_ids
109
-
110
  def _init_assistant(self, config: Config) -> str:
111
  cache_key = config
112
  if cache_key in self._assistant_id_cache:
@@ -114,38 +127,38 @@ class MockInterviewer:
114
  return self._assistant_id_cache.get(cache_key)
115
  else:
116
  print(f'Initializing new assistant for key {cache_key}')
117
- file_ids = self._create_files(config.company)
118
-
119
  assistant = self._client.beta.assistants.create(
120
  name='Mock Interviewer',
121
  instructions=self._generate_assistant_instructions(config),
122
- model='gpt-4-0125-preview',
123
- tools=[
124
- {
125
- 'type': 'retrieval' # This adds the knowledge base as a tool
126
- }
127
- ],
128
- file_ids=file_ids)
129
 
130
  self._assistant_id_cache[cache_key] = assistant.id
131
  return assistant.id
132
 
133
  def _generate_assistant_instructions(self, config: Config) -> str:
134
  if config.job_role and config.company:
135
- purpose = f'You are Ami, an AI mock interviewer for {config.job_role} roles at {config.company}.'
136
  elif config.job_role:
137
- purpose = f'You are Ami, an AI mock interviewer for {config.job_role} roles.'
138
  elif config.company:
139
- purpose = f'You are Ami, an AI mock interviewer for roles at {config.company}.'
140
  else:
141
- purpose = 'You are Ami, an AI mock interviewer.'
142
 
143
  if config.job_description:
144
  specifics = f'Tailor your questions based on the following job posting: {config.job_description}.'
145
  else:
146
  specifics = ''
147
 
148
- return f"{purpose} Please state your purpose when the candidate sends you the first message. If you have been provided a file, use it as an interview guide. {specifics} Ask {config.behavioral_count} number of behavioral questions, {config.technical_count} number of technical questions, {config.situational_count} number of situational questions, and {config.case_count} number of case-like questions. After the candidate gives a response, evaluate the response of the candidate by addressing the candidate as if you were giving feedback to them (i.e. address them as you). Keep in mind what your company values in candidates. Provide a detailed analysis of the candidate's response based on the question type. Also, rate the response on a scale from 1 to 10, where 1 is inadequate and 10 is exceptional."
 
 
 
 
 
 
 
 
149
 
150
  mock_interviewer = MockInterviewer()
151
 
@@ -170,8 +183,6 @@ with gr.Blocks(theme=theme) as demo:
170
  placeholder='Key job responsibilities, basic qualifications, preferred qualifications, about the company, etc.'
171
  )
172
  with gr.Accordion("Question Preferences", open=False):
173
- label='Question Type and Count'
174
- info='Please indicate how many questions you would like asked on the following question types:',
175
  behavioral_count = gr.Slider(label="Behavioral", maximum=10, value=1, step=1)
176
  technical_count = gr.Slider(label="Technical", maximum=10, value=1, step=1)
177
  situational_count = gr.Slider(label="Situational", maximum=10, value=1, step=1)
@@ -180,20 +191,21 @@ with gr.Blocks(theme=theme) as demo:
180
  with gr.Column(variant='panel', scale=6):
181
  chat_interface = gr.ChatInterface(
182
  fn=mock_interviewer.chat_with_text,
183
- title="Hi! I'm Ami, your AI Mock Interviewer.",
184
- description='You can begin by clicking record and introducing yourself!',
185
  additional_inputs=[job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
186
  retry_btn=None,
187
  undo_btn=None)
188
 
 
 
189
  chat_interface.load(mock_interviewer.clear_thread)
190
  chat_interface.clear_btn.click(mock_interviewer.clear_thread)
191
-
 
192
  video = gr.Video(sources='webcam', include_audio=True)
193
  video.stop_recording(fn=mock_interviewer.chat_with_video,
194
  inputs=[video, job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
195
  outputs=[chat_interface.chatbot],
196
- api_name=False).then(lambda:None, None, video, queue=False)
197
 
198
  if __name__ == '__main__':
199
  demo.launch()
 
1
  import os
2
+ import random
3
  from dataclasses import dataclass
4
  from time import sleep
5
+ from typing import Dict, List, Generator, Optional
6
 
7
+ import cv2
8
  import gradio as gr
9
  from openai import OpenAI
10
  from dotenv import load_dotenv
 
59
  model='whisper-1',
60
  file=file,
61
  )
62
+ video_frame_file_ids = self._extract_frames(video)
63
  os.remove(video)
64
  config = Config(job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count)
65
+ response = self._chat(transcriptions.text, config, video_frame_file_ids)
66
  return [(transcriptions.text, response)]
67
 
68
  def clear_thread(self) -> None:
69
  print('Initializing new thread')
70
  self._thread = self._client.beta.threads.create()
71
+
72
+ def _extract_frames(self, video_path: str) -> List[str]:
73
+ video = cv2.VideoCapture(video_path)
74
+ num_frames = int(video.get(cv2.CAP_PROP_FRAME_COUNT))
75
+
76
+ counter = 0
77
+ for i in random.sample(range(num_frames), 10):
78
+ video.set(cv2.CAP_PROP_FRAME_COUNT, i)
79
+ success, frame = video.read()
80
+ if not success:
81
+ print('Error in video frame extraction')
82
+ break
83
+ cv2.imwrite(f'{counter}.jpg', frame)
84
+ counter += 1
85
+
86
+ video.release()
87
+
88
+ file_ids = []
89
+ for i in range(counter):
90
+ with open(f'{i}.jpg', 'rb') as image:
91
+ file = self._client.files.create(file=image, purpose='assistants')
92
+ file_ids.append(file.id)
93
+ os.remove(f'{i}.jpg')
94
+
95
+ return file_ids
96
 
97
+ def _chat(self, message: str, config: Config, video_frame_file_ids: List[str] = list()) -> str:
98
  print('Started chat')
99
  assistant_id = self._init_assistant(config)
100
+ return self._send_message(message, assistant_id, video_frame_file_ids)
101
 
102
+ def _send_message(self, message: str, assistant_id: str, video_frame_file_ids: List[str]) -> str:
103
+ self._client.beta.threads.messages.create(thread_id=self._thread.id, role='user', content=message, file_ids=video_frame_file_ids)
104
  print('Message created')
105
  run = self._client.beta.threads.runs.create(thread_id=self._thread.id, assistant_id=assistant_id)
106
  print('Run created')
 
119
  response = messages.data[0].content[0].text.value
120
  print(f'Assistant response: {response}')
121
  return response
122
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
123
  def _init_assistant(self, config: Config) -> str:
124
  cache_key = config
125
  if cache_key in self._assistant_id_cache:
 
127
  return self._assistant_id_cache.get(cache_key)
128
  else:
129
  print(f'Initializing new assistant for key {cache_key}')
 
 
130
  assistant = self._client.beta.assistants.create(
131
  name='Mock Interviewer',
132
  instructions=self._generate_assistant_instructions(config),
133
+ model='gpt-4-0125-preview')
 
 
 
 
 
 
134
 
135
  self._assistant_id_cache[cache_key] = assistant.id
136
  return assistant.id
137
 
138
  def _generate_assistant_instructions(self, config: Config) -> str:
139
  if config.job_role and config.company:
140
+ purpose = f'You are AiMI, an AI mock interviewer for {config.job_role} roles at {config.company}.'
141
  elif config.job_role:
142
+ purpose = f'You are AiMI, an AI mock interviewer for {config.job_role} roles.'
143
  elif config.company:
144
+ purpose = f'You are AiMI, an AI mock interviewer for roles at {config.company}.'
145
  else:
146
+ purpose = 'You are AiMI, an AI mock interviewer.'
147
 
148
  if config.job_description:
149
  specifics = f'Tailor your questions based on the following job posting: {config.job_description}.'
150
  else:
151
  specifics = ''
152
 
153
+ return f'''
154
+ {purpose} Please greet the candidate and begin the mock interview when the candidate sends you the first message. {specifics} Ask {config.behavioral_count} number of behavioral questions, {config.technical_count} number of technical questions, {config.situational_count} number of situational questions, and {config.case_count} number of case-like questions, one question at a time.
155
+
156
+ After the candidate gives a response, evaluate the response of the candidate by addressing the candidate as if you were giving feedback to them (i.e. address them as you). Keep in mind what your company values in candidates (if you have been assigned a company). Provide a detailed analysis of the candidate's response based on the question type. In your feedback, comment on 1) avoiding filler words and non-words such as um or like, 2) avoiding jargon, and 3) flow (ideas flow logically with clear transitions between main ideas).
157
+
158
+ The candidate may have included frames from a video recording of their response. If so, please analyze the provided images from a mock interview setting, focusing on the following key aspects to evaluate the subject's presentation and the interview environment. Provide recommendations for improvement (limit observations to be brief). Focus on these nonverbal criteria: Facial Expressions: Assess the subject's facial expressions, considering if they convey confidence, engagement, and professionalism. Offer insights into how facial expressions could impact the interviewer's perception. Energy: If they appear energetic (conveying energy to engage viewers). Please provide detailed feedback on each aspect, including what is done well and what could be enhanced to improve the subject's presentation and the overall interview setting. Also comment on the following briefly only if it really needs improvement. It is not necessary to comment on the following, only if it needs improvement: Lighting: Describe the quality and direction of the lighting in the image. Note whether the subject is well-lit, if there are any harsh shadows on the face, and if the background is appropriately illuminated. Apparel: Comment on the appropriateness of the subject's attire for a professional interview. Mention the colors, fit, and formality of the clothing, and suggest any adjustments if needed. Speaking Environment/Background: Analyze the speaking environment and background for any distractions or elements that could detract from the focus on the subject. Recommend changes to create a more neutral and professional background. Limit your complete comments on the candidate's video to 100 words.
159
+
160
+ Finally, rate the complete response (content and video) on a scale from 1 to 10, where 1 is inadequate and 10 is exceptional.
161
+ '''
162
 
163
  mock_interviewer = MockInterviewer()
164
 
 
183
  placeholder='Key job responsibilities, basic qualifications, preferred qualifications, about the company, etc.'
184
  )
185
  with gr.Accordion("Question Preferences", open=False):
 
 
186
  behavioral_count = gr.Slider(label="Behavioral", maximum=10, value=1, step=1)
187
  technical_count = gr.Slider(label="Technical", maximum=10, value=1, step=1)
188
  situational_count = gr.Slider(label="Situational", maximum=10, value=1, step=1)
 
191
  with gr.Column(variant='panel', scale=6):
192
  chat_interface = gr.ChatInterface(
193
  fn=mock_interviewer.chat_with_text,
 
 
194
  additional_inputs=[job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
195
  retry_btn=None,
196
  undo_btn=None)
197
 
198
+ chat_interface.chatbot.value = [(None, "Hi! I'm AiMI, your AI Mock Interviewer. You can begin by clicking record and introducing yourself!")]
199
+ chat_interface.chatbot.height = '70vh'
200
  chat_interface.load(mock_interviewer.clear_thread)
201
  chat_interface.clear_btn.click(mock_interviewer.clear_thread)
202
+
203
+ with gr.Column(variant='panel', scale=1):
204
  video = gr.Video(sources='webcam', include_audio=True)
205
  video.stop_recording(fn=mock_interviewer.chat_with_video,
206
  inputs=[video, job_role, company, job_description, behavioral_count, technical_count, situational_count, case_count],
207
  outputs=[chat_interface.chatbot],
208
+ api_name=False).then(lambda: None, None, video, queue=False)
209
 
210
  if __name__ == '__main__':
211
  demo.launch()
requirements.txt CHANGED
@@ -1,3 +1,4 @@
1
  openai==1.16.2
2
  gradio==4.25.0
3
- python-dotenv==1.0.1
 
 
1
  openai==1.16.2
2
  gradio==4.25.0
3
+ python-dotenv==1.0.1
4
+ opencv-python==4.9.0.80