Restructured graders input and candidate messages

tests/candidate.py

@@ -67,7 +67,7 @@ def complete_interview(interview_type, exp_name, requirements="", difficulty="",
         )
         response_json = json.loads(response.choices[0].message.content)
 
-        code = response_json.get("code", "")
+        code = response_json.get("code_and_notes", "")
         candidate_message = response_json.get("message", "")
 
         if not code and not candidate_message:
@@ -77,7 +77,7 @@ def complete_interview(interview_type, exp_name, requirements="", difficulty="",
         messages_candidate.append({"role": "assistant", "content": response.choices[0].message.content})
 
         if code:
-            interview_data["transcript"].append(f"CANDIDATE CODE: {code}")
+            interview_data["transcript"].append(f"CANDIDATE CODE AND NOTES: {code}")
         elif candidate_message:
             interview_data["transcript"].append(f"CANDIDATE MESSAGE: {candidate_message}")
 

tests/grader.py

@@ -11,10 +11,21 @@ def grade(json_file_path, model="gpt-4-turbo"):
     with open(json_file_path) as file:
         interview_data = json.load(file)
 
+    interview_summary_list = []
+    interview_summary_list.append(f"Interview type: {interview_data['inputs']['interview_type']}")
+    interview_summary_list.append(f"Interview difficulty: {interview_data['inputs']['difficulty']}")
+    interview_summary_list.append(f"Interview topic: {interview_data['inputs']['topic']}")
+    if interview_data["inputs"]["requirements"] != "":
+        interview_summary_list.append(f"Interview requirements: {interview_data['inputs']['requirements']}")
+    interview_summary_list.append(f"Problem statement proposed by interviewer: {interview_data['problem_statement']}")
+    interview_summary_list.append(f"\nTranscript of the whole interview below:")
+    interview_summary_list += interview_data["transcript"]
+    interview_summary_list.append(f"\nTHE MAIN PART OF THE INTERVIEW ENDED HERE.")
+    interview_summary_list.append(f"Feedback provided by interviewer: {interview_data['feedback']}")
+
     messages = [
         {"role": "system", "content": grader_prompt},
-        {"role": "user", "content": f"Interview data: {interview_data}"},
-        {"role": "user", "content": "Please evaluate the interview."},
+        {"role": "user", "content": f"Please evaluate the interviewer based on the following data: \n {'\n'.join(interview_summary_list)}"},
     ]
 
     response = client.chat.completions.create(model=model, messages=messages, temperature=1, response_format={"type": "json_object"})

tests/testing_prompts.py

@@ -4,12 +4,13 @@ You should solve the problem in a very small and incremental steps and explain y
 Don't rush to give the whole solution at once.
 Sometime make mistakes in logic, computations, or syntax. Pretend you are stuck and ask for help.
 Follow interviewer (user) instructions and answer their questions.
+If you see that you and interviewer are repeating yourselves just move on to the next point.
 You can ask for clarification if you don't understand something.
-Each your answer should be a json with 4 keys: "finished", "question", "message" and "code".
+Each your answer should be a json with 4 keys: "finished", "question", "message" and "code_and_notes".
 "finished" is a boolean, it is True if the user told you that the interview is finished, otherwise False.
 "question" is a boolean, it is True if the last user message contains a question, otherwise False.
 "message" is a string, it is a message you want to tell the interviewer. Message should never be empty.
-"code" is a string, it is the current version of your notes (it can be code, query, pseudocode, answer structure, formulas, calculations, schemas, examples, test cases, etc.), if it didn't change from the last message return an empty string. Try to actively use this field, it is very important.
+"code_and_notes" is a string, it is the current version of your notes (it can be code, query, pseudocode, answer structure, formulas, calculations, schemas, examples, test cases, etc.), if it didn't change from the last message return an empty string. Try to actively use this field, it is very important.
 """
 
 
@@ -43,6 +44,8 @@ You should evaluate the following aspects and return a JSON with these keys:
   "interviewer_leaks": "The interviewer didn't leak any hidden notes to candidate during the main part of the interview.",
   "interviewer_empty": "The interviewer didn't send any empty messages.",
   "interviewer_notes": "The interviewer made reasonable notes catching candidates mistakes and important facts.",
+  "interviewer_stuck": "The interview stuck at some point in repeating cycle of same questions and answers.",
+  "interviewer_end": "The interview ended interview after candidate answer all questions (vs. interview ended abruptly).",
 
   "feedback_quality": "The feedback was constructive and offered actionable insights.",
   "feedback_overview": "The feedback contains the recap of main mistakes and good ideas of the candidate.",