Fixed tests and candidate simulation

pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+log_cli = true
+log_cli_level = INFO
+log_cli_format = %(asctime)s [%(levelname)s] %(message)s
+log_cli_date_format = %Y-%m-%d %H:%M:%S

tests/candidate.py

@@ -87,40 +87,48 @@ def complete_interview(
     previous_code = ""
 
     if max_messages is None:
-        max_messages = 30 if mode == "normal" else 5
+        max_messages = 25 if mode == "normal" else 5
 
     for _ in range(max_messages):
+        code = ""
         if mode == "empty":
-            response_content = ""
+            candidate_message = ""
         elif mode == "gibberish":
-            response_content = "".join(random.choices(string.ascii_letters + string.digits, k=50))
+            candidate_message = "".join(random.choices(string.ascii_letters + string.digits, k=50))
         elif mode == "repeat":
-            response_content = chat_display[-1][1]
+            candidate_message = chat_display[-1][1]
         else:
             response = client.chat.completions.create(
                 model=model, messages=messages_candidate, temperature=1, response_format={"type": "json_object"}, stream=False
             )
             try:
                 response_json = json.loads(response.choices[0].message.content)
-                response_content = response_json.get("message", "")
+                candidate_message = response_json.get("message", "")
+                code = response_json.get("code_and_notes", "")
+                finished = response_json.get("finished", False)
+                question = response_json.get("question", False)
+
+                if finished and not question and not code:
+                    break
             except:
                 continue
 
-        candidate_message = response_content
-
-        if not candidate_message and mode != "empty":
-            print("No message in response")
+        if not candidate_message and not code and mode != "empty":
+            print("No message or code in response")
             continue
 
-        messages_candidate.append({"role": "assistant", "content": candidate_message})
-
-        interview_data["transcript"].append(f"CANDIDATE MESSAGE: {candidate_message}")
+        if candidate_message:
+            messages_candidate.append({"role": "assistant", "content": candidate_message})
+            interview_data["transcript"].append(f"CANDIDATE MESSAGE: {candidate_message}")
+        if code:
+            interview_data["transcript"].append(f"CANDIDATE CODE AND NOTES: {code}")
+            messages_candidate.append({"role": "assistant", "content": code})
 
         chat_display.append([candidate_message, None])
 
         send_time = time.time()
         for messages_interviewer, chat_display, previous_code, _ in send_request(
-            candidate_message, previous_code, messages_interviewer, chat_display, llm, tts=None, silent=True
+            code, previous_code, messages_interviewer, chat_display, llm, tts=None, silent=True
         ):
             pass
 

tests/test_e2e.py

@@ -2,6 +2,7 @@ from tests.candidate import complete_interview
 from tests.grader import grade
 from concurrent.futures import ThreadPoolExecutor
 import random
+import logging
 from typing import List
 
 
@@ -14,7 +15,9 @@ def complete_and_grade_interview(interview_type: str, mode: str = "normal", min_
     :return: Overall score of the interview.
     """
     file_path, _ = complete_interview(interview_type, "test", model="gpt-4o-mini", mode=mode)
-    feedback = grade(file_path, model="gpt-4-turbo")
+    feedback = grade(file_path, model="gpt-4o")
+
+    logging.info(f"Interview type: {interview_type}, mode: {mode}, score: {feedback['overall_score']}")
     assert feedback["overall_score"] > min_score
     return feedback["overall_score"]
 
@@ -31,13 +34,13 @@ def test_complete_interview() -> None:
         futures = [executor.submit(complete_and_grade_interview, it) for it in interview_types]
 
         # Test edge cases: empty, gibberish, repeat for one random interview type each
-        # The test are placeholders for not, I will increase thresholds later
-        futures.append(executor.submit(complete_and_grade_interview, random.choice(interview_types), mode="empty", min_score=0.0))
-        futures.append(executor.submit(complete_and_grade_interview, random.choice(interview_types), mode="gibberish", min_score=0.0))
-        futures.append(executor.submit(complete_and_grade_interview, random.choice(interview_types), mode="repeat", min_score=0.0))
+        futures.append(executor.submit(complete_and_grade_interview, random.choice(interview_types), mode="empty"))
+        futures.append(executor.submit(complete_and_grade_interview, random.choice(interview_types), mode="gibberish"))
+        futures.append(executor.submit(complete_and_grade_interview, random.choice(interview_types), mode="repeat"))
 
         for future in futures:
             score = future.result()
             scores.append(score)
 
-    assert sum(scores) / len(scores) > 0.6
+    logging.info(f"Average score: {sum(scores) / len(scores)}")
+    assert sum(scores) / len(scores) > 0.7