test_b resultls

.gitattributes

@@ -69,3 +69,4 @@ llama-factory/data/Icon
 llama-factory/data/alpaca_mgtv_p1_en.json filter=lfs diff=lfs merge=lfs -text
 llama-factory/data/alpaca_mgtv_p2_en.json filter=lfs diff=lfs merge=lfs -text
 datasets/mgtv/test_b.csv filter=lfs diff=lfs merge=lfs -text
+results/test_b-results.csv filter=lfs diff=lfs merge=lfs -text

llm_toolkit/eval_logical_reasoning.py

@@ -24,6 +24,7 @@ data_path = os.getenv("LOGICAL_REASONING_DATA_PATH")
 results_path = os.getenv("LOGICAL_REASONING_RESULTS_PATH")
 use_english_datasets = os.getenv("USE_ENGLISH_DATASETS") == "true"
 using_p1 = os.getenv("USING_P1_PROMPT_TEMPLATE") == "true"
+test_data = os.getenv("TEST_DATA", None)
 using_llama_factory = os.getenv("USING_LLAMA_FACTORY") == "true"
 max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", 16))
 repetition_penalty = float(os.getenv("REPETITION_PENALTY", 1.0))
@@ -65,6 +66,7 @@ datasets = load_logical_reasoning_dataset(
     tokenizer=tokenizer,
     chinese_prompt=not use_english_datasets,
     using_p1=using_p1,
+    test_data=test_data,
 )
 
 if len(sys.argv) > 1:
@@ -94,12 +96,17 @@ if adapter_name_or_path is not None:
     model_name += "/" + adapter_name_or_path.split("/")[-1]
 
 save_results(
-    f"{model_name}_{dtype}{'_4bit' if load_in_4bit else ''}{'_lf' if using_llama_factory else ''}",
+    (
+        "answer"
+        if test_data
+        else f"{model_name}_{dtype}{'_4bit' if load_in_4bit else ''}{'_lf' if using_llama_factory else ''}"
+    ),
     results_path,
     datasets["test"],
     predictions,
     debug=True,
 )
 
-metrics = calc_metrics(datasets["test"]["label"], predictions, debug=True)
-print(metrics)
+if not test_data:
+    metrics = calc_metrics(datasets["test"]["label"], predictions, debug=True)
+    print(metrics)

llm_toolkit/logical_reasoning_utils.py

@@ -167,11 +167,11 @@ def save_results(model_name, results_path, dataset, predictions, debug=False):
 
 
 def load_logical_reasoning_dataset(
-    data_path, tokenizer=None, using_p1=True, chinese_prompt=True
+    data_path, tokenizer=None, using_p1=True, chinese_prompt=True, test_data=None
 ):
     postfix = "" if chinese_prompt else "_en"
     train_data_file = data_path + f"/train{postfix}.csv"
-    test_data_file = data_path + f"/dev{postfix}.csv"
+    test_data_file = data_path + f"/{test_data if test_data else 'dev'}{postfix}.csv"
 
     print("loading train/test data files")
     datasets = load_dataset(
@@ -211,7 +211,7 @@ def load_logical_reasoning_dataset(
                     messages, tokenize=False, add_generation_prompt=True
                 )
                 prompts.append(prompt)
-                texts.append(prompt + output + tokenizer.eos_token)
+                texts.append(prompt + output + tokenizer.eos_token if output else "")
             return {"train_text": texts, "prompt": prompts}
 
         datasets = datasets.map(

results/test_b-results.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8efee9e847b45b8d1ccf187a7e214e73cfbd368840b1d699af6b228eca3b22a
+size 23116718

scripts/test-mgtv.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+BASEDIR=`pwd`
+
+nvidia-smi
+uname -a
+cat /etc/os-release
+lscpu
+grep MemTotal /proc/meminfo
+
+export LOAD_IN_4BIT=false
+
+export MODEL_NAME=internlm/internlm2_5-7b-chat-1m
+export ADAPTER_NAME_OR_PATH=inflaton-ai/InternLM_2_5-7b_LoRA-Adapter
+export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
+
+export TEST_DATA=test_b
+export LOGICAL_REASONING_RESULTS_PATH=results/$TEST_DATA-results.csv
+
+echo "Eval $MODEL_NAME with $ADAPTER_NAME_OR_PATH"
+python llm_toolkit/eval_logical_reasoning.py