h100 results

.env.example

@@ -1,15 +1,11 @@
+MODEL_NAME=internlm/internlm2_5-7b-chat-1m
+
 OPENAI_API_KEY=
 HF_TOKEN=
+WANDB_API_KEY=
 
-MODEL_NAME=Qwen2/Qwen2-1.5B-Instruct
-
-LOAD_IN_4BIT=true
-NUM_TRAIN_EPOCHS=10
-
-DATA_PATH=datasets/mac/mac.tsv
-RESULTS_PATH=results/mac-results_lf.csv
+LOAD_IN_4BIT=false
+NUM_TRAIN_EPOCHS=3
 
-EVAL_BASE_MODEL=true
-EVAL_FINE_TUNED=true
-SAVE_FINE_TUNED=true
-DO_FINE_TUNING=true
+LOGICAL_REASONING_DATA_PATH=datasets/mgtv
+LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_l40.csv

.gitattributes

@@ -52,3 +52,6 @@ results/mgtv-results_t4_r3.csv filter=lfs diff=lfs merge=lfs -text
 llama-factory/data/alpaca_mac.json filter=lfs diff=lfs merge=lfs -text
 llama-factory/data/alpaca_mgtv_p2.json filter=lfs diff=lfs merge=lfs -text
 llama-factory/data/dataset_info.json filter=lfs diff=lfs merge=lfs -text
+results/mgtv-results_colab_p2.csv filter=lfs diff=lfs merge=lfs -text
+results/mgtv-results_h100.csv filter=lfs diff=lfs merge=lfs -text
+results/mgtv-results_bf16.csv filter=lfs diff=lfs merge=lfs -text

llm_toolkit/eval_logical_reasoning_all_epochs.py

@@ -0,0 +1,90 @@
+import os
+import sys
+import subprocess
+from dotenv import find_dotenv, load_dotenv
+
+
+def evaluate_model_all_epochs_v2(
+    model_name,
+    adapter_path_base=None,
+    start_epoch=0,
+    load_in_4bit=True,
+    num_of_entries=-1,
+    result_file=None,
+):
+    new_env = os.environ.copy()
+    new_env["MODEL_NAME"] = model_name
+    model = model_name.split("/")[-1]
+
+    new_env["LOAD_IN_4BIT"] = "true" if load_in_4bit else "false"
+    if result_file is not None:
+        new_env["LOGICAL_REASONING_RESULTS_PATH"] = result_file
+
+    if adapter_path_base is None:
+        num_train_epochs = 0
+        print(f"No adapter path provided. Running with base model:{model_name}")
+    else:
+        # find subdirectories in adapter_path_base
+        # and sort them by epoch number
+        subdirs = [
+            d
+            for d in os.listdir(adapter_path_base)
+            if os.path.isdir(os.path.join(adapter_path_base, d))
+        ]
+
+        subdirs = sorted(subdirs, key=lambda x: int(x.split("-")[-1]))
+        num_train_epochs = len(subdirs)
+        print(f"found {num_train_epochs} checkpoints: {subdirs}")
+
+    for i in range(start_epoch, num_train_epochs + 1):
+        print(f"Epoch {i}")
+        if i == 0:
+            os.unsetenv("ADAPTER_NAME_OR_PATH")
+        else:
+            adapter_path = subdirs[i - 1]
+            new_env["ADAPTER_NAME_OR_PATH"] = adapter_path
+
+        print(f"adapter path: {new_env.get('ADAPTER_NAME_OR_PATH')}")
+
+        log_file = "./logs/{}_epoch_{}.txt".format(model, i)
+        with open(log_file, "w") as f_obj:
+            subprocess.run(
+                f"python llm_toolkit/eval_logical_reasoning.py {num_of_entries}",
+                shell=True,
+                env=new_env,
+                stdout=f_obj,
+                text=True,
+            )
+
+
+if __name__ == "__main__":
+    found_dotenv = find_dotenv(".env")
+
+    if len(found_dotenv) == 0:
+        found_dotenv = find_dotenv(".env.example")
+    print(f"loading env vars from: {found_dotenv}")
+    load_dotenv(found_dotenv, override=False)
+
+    workding_dir = os.path.dirname(found_dotenv)
+    os.chdir(workding_dir)
+    sys.path.append(workding_dir)
+    print("workding dir:", workding_dir)
+    print(f"adding {workding_dir} to sys.path")
+    sys.path.append(workding_dir)
+
+    model_name = os.getenv("MODEL_NAME")
+    adapter_path_base = os.getenv("ADAPTER_PATH_BASE")
+    start_epoch = int(os.getenv("START_EPOCH", 0))
+    load_in_4bit = os.getenv("LOAD_IN_4BIT", "true").lower() == "true"
+    result_file = os.getenv("LOGICAL_REASONING_RESULTS_PATH", None)
+
+    num_of_entries = int(sys.argv[1]) if len(sys.argv) > 1 else -1
+
+    evaluate_model_all_epochs_v2(
+        model_name,
+        adapter_path_base=adapter_path_base,
+        start_epoch=start_epoch,
+        load_in_4bit=load_in_4bit,
+        num_of_entries=num_of_entries,
+        result_file=result_file,
+    )

results/mgtv-results_h100.csv

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12a51462b06704ff4867c91a3c7a305371c706ceefe02be5473f27f7f612f4f9
+size 2812796

scripts/eval-mgtv-internlm.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+BASEDIR=`pwd`
+
+nvidia-smi
+uname -a
+cat /etc/os-release
+lscpu
+grep MemTotal /proc/meminfo
+
+#pip install -r requirements.txt
+#cd ../LLaMA-Factory && pip install -e .[torch,bitsandbytes]
+
+export MODEL_NAME=internlm/internlm2_5-7b-chat-1m
+export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
+export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full.csv
+export ADAPTER_PATH_BASE=llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full
+
+echo "Eval $MODEL_NAME with $ADAPTER_PATH_BASE"
+python llm_toolkit/eval_logical_reasoning_all_epochs.py

scripts/eval-mgtv.sh

@@ -0,0 +1 @@
+eval-mgtv-internlm.sh

scripts/tune-mgtv-internlm.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+BASEDIR=`pwd`
+
+nvidia-smi
+uname -a
+cat /etc/os-release
+lscpu
+grep MemTotal /proc/meminfo
+
+#pip install -r requirements.txt
+#cd ../LLaMA-Factory && pip install -e .[torch,bitsandbytes]
+
+export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
+
+export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p1_full.csv
+export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p1_full.yaml
+#echo "Tuning with $CONFIG_FILE"
+#$BASEDIR/scripts/tune-lf.sh $CONFIG_FILE
+
+export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full.csv
+export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full.yaml
+echo "Tuning with $CONFIG_FILE"
+$BASEDIR/scripts/tune-lf.sh $CONFIG_FILE

scripts/tune-mgtv.sh

@@ -1,29 +0,0 @@
-#!/bin/sh
-
-BASEDIR=$(dirname "$0")
-cd $BASEDIR/..
-echo Current Directory:
-pwd
-
-BASEDIR=`pwd`
-
-nvidia-smi
-uname -a
-cat /etc/os-release
-lscpu
-grep MemTotal /proc/meminfo
-
-#pip install -r requirements.txt
-#cd ../LLaMA-Factory && pip install -e .[torch,bitsandbytes]
-
-export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
-
-export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p1_full.csv
-export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p1_full.yaml
-#echo "Tuning with $CONFIG_FILE"
-#$BASEDIR/scripts/tune-lf.sh $CONFIG_FILE
-
-export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full.csv
-export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full.yaml
-cho "Tuning with $CONFIG_FILE"
-$BASEDIR/scripts/tune-lf.sh $CONFIG_FILE

scripts/tune-mgtv.sh

@@ -0,0 +1 @@
+tune-mgtv-internlm.sh