final rank: 42

llm_toolkit/tune_logical_reasoning.py

@@ -140,7 +140,7 @@ tokenizer.save_pretrained(local_model)
 
 print("Evaluating fine-tuned model: " + model_name)
 FastLanguageModel.for_inference(model)  # Enable native 2x faster inference
-predictions = eval_model(model, tokenizer, datasets["test"])
+predictions = eval_model(model, tokenizer, dataset["test"])
 
 gpu_stats = torch.cuda.get_device_properties(0)
 start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
@@ -151,12 +151,12 @@ print(f"{start_gpu_memory} GB of memory reserved.")
 save_results(
     model_name + "(unsloth_finetuned)",
     results_path,
-    datasets["test"],
+    dataset["test"],
     predictions,
     debug=True,
 )
 
-metrics = calc_metrics(datasets["test"]["label"], predictions, debug=True)
+metrics = calc_metrics(dataset["test"]["label"], predictions, debug=True)
 print(metrics)
 
 model.push_to_hub(hub_model, token=token)  # Online saving

results/mgtv-results_colab_p2_gemma2.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:08ba752e1739c0ec2034f0eeffa263eee70908b4ff436e6ed0c875c3332aec64
-size 2797399
+oid sha256:157c7b5b2fba5a2f3874a202d9c210e7d4af22f52adce5a0f14309df17e064ba
+size 2797369

results/mgtv-results_p2_r3_full_metrics.csv

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:113b15494641f116be89c92a9e46880a00c842956375bb6e7b079d17c221f7d6
-size 1538
+oid sha256:4010fd69e8ef81a6e6256e4733a49a760d567f010ebe9c3a36ac976564e53928
+size 408

scripts/eval-mgtv-gemma2_9b.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+BASEDIR=`pwd`
+
+nvidia-smi
+uname -a
+cat /etc/os-release
+lscpu
+grep MemTotal /proc/meminfo
+
+# pip install unsloth[colab-new] file:///common2/dh.huang.2023/code/unsloth
+# pip install --no-deps "xformers<0.0.27" "trl<0.9.0" peft accelerate bitsandbytes
+
+export LOAD_IN_4BIT=false
+export USING_LLAMA_FACTORY=false
+
+export MODEL_NAME=google/gemma-2-9b-it
+export USING_P1_PROMPT_TEMPLATE=false
+export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
+
+export MODEL_PREFIX=gemma2-9b_lora_sft_bf16
+export LOGICAL_REASONING_RESULTS_PATH=results/$MODEL_PREFIX-p2.csv
+export ADAPTER_PATH_BASE=unsloth/gemma-2-9b-it
+
+echo "Eval $MODEL_NAME with $ADAPTER_PATH_BASE"
+python llm_toolkit/eval_logical_reasoning_all_epochs.py