cleanup

requirements.txt

@@ -11,5 +11,5 @@ python-dateutil==2.8.2
 requests==2.28.2
 tqdm==4.65.0
 git+https://github.com/EleutherAI/lm-evaluation-harness.git@a79a7c33d04d7090b94256d2fcd631c5e6fd8631#egg=lm-eval
-lamini
+lamini==2.1.4a3
 openai

src/backend/harness_evaluator.py

@@ -28,5 +28,6 @@ class HarnessEvaluator:
             batch_size=self.batch_size,
             num_fewshot=0,
             limit=MAX_EXAMPLES,
+            random_seed=42
         )
         return results

src/backend/lamini_eval/evaluators/icd_evaluator.py

@@ -118,8 +118,8 @@ class ICDAnswerGenerator(GenerationNode):
         )
         prompt += "You are answering questions about the ICD 11 standard. "
         prompt += "Include the most relevant ICD11 code in your answer. "
-        prompt += "Every code starts with a letter, followed by a two digit number. "
-        prompt += "In the ICD-11, the next level of the hierarchy is indicated in the code by a dot and a single number or letter from A to Z."
+        # prompt += "Every code starts with a letter, followed by a two digit number. "
+        # prompt += "In the ICD-11, the next level of the hierarchy is indicated in the code by a dot and a single number or letter from A to Z."
         prompt += "A customer asks\n"
         prompt += chunk.data["question"]
         prompt += " [/INST]"

src/backend/lamini_eval/models/gpt4_model.py

@@ -48,7 +48,8 @@ class GPT4Model(LM):
             obj = TruthfulQAEvaluator()
             prompt = obj.get_prompt(question)
             try:
-                response = self.__call__(prompt=prompt, output_type=str)
+                result = self.__call__(prompt=prompt, output_type={"answer": "str"})
+                response = result['answer']
             except Exception as e:
                 print("Error fetching gpt4 response: ", e)
                 # select random answer
@@ -59,8 +60,8 @@ class GPT4Model(LM):
             prompt = obj.get_prompt(question)
             try:
                 op_type = {"explanation": "str", "answer": "str"}
-                answer = self.__call__(prompt=prompt, output_type=op_type)
-                response = f"({answer['answer']})"
+                result = self.__call__(prompt=prompt, output_type=op_type)
+                response = f"({result['answer']})"
             except Exception as e:
                 print("Error fetching gpt4 response: ", e)
                 # select random answer

src/backend/run_eval_suite.py

@@ -23,7 +23,6 @@ async def run_evaluation(eval_request: EvalRequest, task_names, num_fewshot, bat
                                        batch_size, device, no_cache, limit, write_out=True,
                                        output_base_path='logs')
     lamini_results = await custom_evaluator.evaluate()
-    task_names = [task_names[0]]
 
     print(f"Selected Harness Tasks: {task_names}")
     harness_evaluator = HarnessEvaluator(eval_request.model, eval_request.revision, eval_request.precision,