update scripts

src/backend/model_operations.py

@@ -43,7 +43,7 @@ logging.basicConfig(level=logging.INFO,
 # Load spacy model for word tokenization
 nlp = spacy.load("en_core_web_sm")
 nlp1 = spacy.load("en_core_web_trf")
-os.environ["HUGGINGFACE_API_KEY"] =  envs.TOKEN
+# os.environ["HUGGINGFACE_API_KEY"] =  envs.TOKEN
 
 def load_evaluation_model(model_path):
     """Load the evaluation model from the given path
@@ -173,7 +173,7 @@ class SummaryGenerator:
                     # print(ID, q_ID, prompt_value)
                     system_prompt = envs.SYSTEM_PROMPT 
                     _user_prompt = prompt_value
-                    for ii in range(10):
+                    for ii in range(1):
                     # user_prompt = f"{envs.USER_PROMPT}\nPassage:\n{_source}"
                         while True:
                             try:
@@ -405,20 +405,27 @@ class SummaryGenerator:
                 #     max_tokens=1024,
                 #     api_base= "https://api-inference.huggingface.co/models/" + self.model_id,
                 # )
-                self.model_id = 'command-r-plus' if 'command' in self.model_id else self.model_id
-                response = litellm.completion(
-                            model="huggingface/" + self.model_id,
-                            # mistralai/Mistral-7B-Instruct-v0.1",
-                            messages=[{"role": "system", "content": system_prompt},
-                                {"role": "user", "content": user_prompt}],
-                            #temperature=0.0,
-                            max_tokens=1024,
-                            api_base="https://api-inference.huggingface.co/models/" + self.model_id)
-                print("模型返回结果",response)
-                print("模型返回结果结束")
-                # exit()
-                result = response['choices'][0]['message']['content']
-                print(result)
+                # self.model_id = 'command-r-plus' if 'command' in self.model_id else self.model_id
+                # response = litellm.completion(
+                #             model="huggingface/" + self.model_id,
+                #             # mistralai/Mistral-7B-Instruct-v0.1",
+                #             messages=[{"role": "system", "content": system_prompt},
+                #                 {"role": "user", "content": user_prompt}],
+                #             #temperature=0.0,
+                #             max_tokens=1024,
+                #             api_base="https://api-inference.huggingface.co/models/" + self.model_id)
+                # print("模型返回结果",response)
+                # print("模型返回结果结束")
+                # # exit()
+                # result = response['choices'][0]['message']['content']
+                # print(result)
+                from huggingface_hub import InferenceClient
+
+                client = InferenceClient(self.model_id,api_key=envs.TOKEN)
+                messages = [{"role": "system", "content": system_prompt},{"role": "user", "content": user_prompt}]
+                outputs = client.chat_completion(messages, max_tokens=50)
+                result = outputs['choices'][0]['message']['content']
+
                 return result
                 # exit()
             except: # fail to call api. run it locally.
@@ -544,6 +551,11 @@ class EvaluationModel:
                 output.append("Other")
                 continue
             rs = summaries_df["Response"][i].strip().lower()
+            sentences = rs.split('\n')
+            sentences = [sentence.split(':', 1)[-1].strip() if ':' in sentence else sentence
+                         for sentence in sentences]
+            rs = [sentence.strip() for sentence in sentences if sentence.strip()]
+
             '''Exp1'''
             if summaries_df["Experiment"][i] == "E1":
                 print("E1", rs)
@@ -864,13 +876,22 @@ class EvaluationModel:
                     output.append("Other")
 
                 '''Exp4'''
-            
+
             elif summaries_df["Experiment"][i] == "E4":
-                # rs = summaries_df["Response"][i].strip()
-                meaning_word = rs.split(";")[4].replace(" ",'')
+                try:
+                    meaning_word = rs.split(";")[4].replace(" ", '')
+                except IndexError:
+                    output.append("Other")
+                    continue
+                except Exception as e:
+                    print(f"Unexpected error: {e}")
+                    output.append("Other")
+                    continue
+
                 target = summaries_df["Factor 2"][i].strip().lower()
                 pair = target + "_" + meaning_word
                 print("E4:", pair)
+
                 if pair in wordpair2code.keys():
                     output.append(wordpair2code[pair])
                 else:
@@ -1068,7 +1089,7 @@ class EvaluationModel:
         float: The average JS divergence across all common Question_IDs.
         """
         # Load the datasets
-        human_df = pd.read_excel(file_path_1)
+        human_df = pd.read_csv(file_path_1, encoding='ISO-8859-1')
         llm_df = pd.read_csv(file_path_2)
 
         def create_e5_entries(df):
@@ -1146,7 +1167,7 @@ class EvaluationModel:
         return avg_js_divergence
     
 
-    def evaluate_humanlike(self, summaries_df, human_data_path, result_save_path):
+    def evaluate_humanlike(self, summaries_df: object, human_data_path: object, result_save_path: object) -> object:
         '''
         evaluate humanlike score
         1. code the result
@@ -1156,8 +1177,8 @@ class EvaluationModel:
         '''coding human data'''
         # self.huamn_df = pd.read_csv(human_data_path)
         # self.data = self.code_results(self.huamn_df)
-        save_path = human_data_path.replace('.csv','_coding.csv')
-        human_save_path =  "./src/datasets/coding_human.xlsx"
+        #save_path = human_data_path.replace('.csv','_coding.csv')
+        #human_save_path =  "./src/datasets/coding_human.xlsx"
         # if save_path is not None:
         #     print(f'Save human coding results to {save_path}')
         #     fpath = Path(save_path)
@@ -1175,7 +1196,7 @@ class EvaluationModel:
             self.llm_df.to_csv(fpath)
         # file_path_1 = '/Users/simon/Downloads/coding_human.xlsx'
         # file_path_2 = '/Users/simon/Downloads/Meta-Llama-3.1-70B-Instruct_coding.csv'
-        avg_js_divergence = self.calculate_js_divergence("./src/datasets/coding_human.xlsx", save_path)
+        avg_js_divergence = self.calculate_js_divergence(human_data_path, save_path)
 
         return avg_js_divergence
 

src/backend/util.py

@@ -61,7 +61,7 @@ def format_results(model_name: str, revision: str, precision: str,
         },
         "results": {
             "hallucination_rate": {
-                "hallucination_rate": round(hallucination_rate,1)
+                "hallucination_rate": round(hallucination_rate,3)
             },
             "factual_consistency_rate": {
                 "factual_consistency_rate": round(factual_consistency_rate,1)

src/envs.py

@@ -36,7 +36,7 @@ API = HfApi(token=TOKEN)
 DATASET_PATH = "./src/datasets/Material_Llama2_0603.xlsx" #experiment data
 PROMPT_PATH = "./src/datasets/prompt.xlsx" #prompt for each experiment
 HEM_PATH = 'vectara/hallucination_evaluation_model'
-HUMAN_DATA = "./src/datasets/human_data.csv" #experiment data
+HUMAN_DATA = "./src/datasets/human_data_coding.csv" #experiment data
 ITEM_4_DATA = "./src/datasets/associataion_dataset.csv" #database
 ITEM_5_DATA = "./src/datasets/Items_5.csv" #experiment 5 need verb words