ready for few shots eval

llm_toolkit/eval.py

@@ -1,67 +0,0 @@
-import os
-import sys
-import torch
-from dotenv import find_dotenv, load_dotenv
-
-found_dotenv = find_dotenv(".env")
-
-if len(found_dotenv) == 0:
-    found_dotenv = find_dotenv(".env.example")
-print(f"loading env vars from: {found_dotenv}")
-load_dotenv(found_dotenv, override=False)
-
-path = os.path.dirname(found_dotenv)
-print(f"Adding {path} to sys.path")
-sys.path.append(path)
-
-from llm_toolkit.translation_engine import *
-from llm_toolkit.translation_utils import *
-
-model_name = os.getenv("MODEL_NAME")
-adapter_name_or_path = os.getenv("ADAPTER_NAME_OR_PATH")
-load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true"
-data_path = os.getenv("DATA_PATH")
-results_path = os.getenv("RESULTS_PATH")
-
-print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)
-
-gpu_stats = torch.cuda.get_device_properties(0)
-start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
-max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
-print(f"(1) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
-print(f"{start_gpu_memory} GB of memory reserved.")
-
-model, tokenizer = load_model(
-    model_name, load_in_4bit=load_in_4bit, adapter_name_or_path=adapter_name_or_path
-)
-
-gpu_stats = torch.cuda.get_device_properties(0)
-start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
-max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
-print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
-print(f"{start_gpu_memory} GB of memory reserved.")
-
-datasets = load_translation_dataset(data_path, tokenizer)
-
-print("Evaluating model: " + model_name)
-predictions = eval_model(model, tokenizer, datasets["test"])
-
-gpu_stats = torch.cuda.get_device_properties(0)
-start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
-max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
-print(f"(3) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
-print(f"{start_gpu_memory} GB of memory reserved.")
-
-if adapter_name_or_path is not None:
-    model_name += "_" + adapter_name_or_path.split("/")[-1]
-
-save_results(
-    model_name,
-    results_path,
-    datasets["test"],
-    predictions,
-    debug=True,
-)
-
-metrics = calc_metrics(datasets["test"]["english"], predictions, debug=True)
-print(metrics)

llm_toolkit/eval_lf.py

@@ -1,110 +0,0 @@
-import os
-import sys
-import torch
-from dotenv import find_dotenv, load_dotenv
-from llamafactory.chat import ChatModel
-from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
-
-found_dotenv = find_dotenv(".env")
-
-if len(found_dotenv) == 0:
-    found_dotenv = find_dotenv(".env.example")
-print(f"loading env vars from: {found_dotenv}")
-load_dotenv(found_dotenv, override=False)
-
-path = os.path.dirname(found_dotenv)
-print(f"Adding {path} to sys.path")
-sys.path.append(path)
-
-from llm_toolkit.translation_utils import *
-
-model_name = os.getenv("MODEL_NAME")
-adapter_name_or_path = os.getenv("ADAPTER_NAME_OR_PATH")
-load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true"
-data_path = os.getenv("DATA_PATH")
-results_path = os.getenv("RESULTS_PATH")
-
-print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)
-
-
-def load_model(
-    model_name,
-    max_seq_length=2048,
-    dtype=torch.bfloat16,
-    load_in_4bit=False,
-    adapter_name_or_path=None,
-):
-    print(f"loading model: {model_name}")
-
-    if adapter_name_or_path:
-        template = "llama3" if "llama-3" in model_name.lower() else "chatml"
-
-        args = dict(
-            model_name_or_path=model_name,
-            adapter_name_or_path=adapter_name_or_path,  # load the saved LoRA adapters
-            template=template,  # same to the one in training
-            finetuning_type="lora",  # same to the one in training
-            quantization_bit=4 if load_in_4bit else None,  # load 4-bit quantized model
-        )
-        chat_model = ChatModel(args)
-        return chat_model.engine.model, chat_model.engine.tokenizer
-
-    tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
-    bnb_config = BitsAndBytesConfig(
-        load_in_4bit=load_in_4bit,
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_use_double_quant=False,
-        bnb_4bit_compute_dtype=dtype,
-    )
-
-    model = AutoModelForCausalLM.from_pretrained(
-        model_name,
-        quantization_config=bnb_config,
-        torch_dtype=dtype,
-        trust_remote_code=True,
-        device_map="auto",
-    )
-
-    return model, tokenizer
-
-
-gpu_stats = torch.cuda.get_device_properties(0)
-start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
-max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
-print(f"(1) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
-print(f"{start_gpu_memory} GB of memory reserved.")
-
-model, tokenizer = load_model(
-    model_name, load_in_4bit=load_in_4bit, adapter_name_or_path=adapter_name_or_path
-)
-
-gpu_stats = torch.cuda.get_device_properties(0)
-start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
-max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
-print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
-print(f"{start_gpu_memory} GB of memory reserved.")
-
-datasets = load_translation_dataset(data_path, tokenizer)
-
-print("Evaluating model: " + model_name)
-predictions = eval_model(model, tokenizer, datasets["test"])
-
-gpu_stats = torch.cuda.get_device_properties(0)
-start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
-max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
-print(f"(3) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
-print(f"{start_gpu_memory} GB of memory reserved.")
-
-if adapter_name_or_path is not None:
-    model_name += "_" + adapter_name_or_path.split("/")[-1]
-
-save_results(
-    model_name,
-    results_path,
-    datasets["test"],
-    predictions,
-    debug=True,
-)
-
-metrics = calc_metrics(datasets["test"]["english"], predictions, debug=True)
-print(metrics)

llm_toolkit/eval_shots.py

@@ -0,0 +1,166 @@
+import os
+import sys
+import torch
+from dotenv import find_dotenv, load_dotenv
+
+found_dotenv = find_dotenv(".env")
+
+if len(found_dotenv) == 0:
+    found_dotenv = find_dotenv(".env.example")
+print(f"loading env vars from: {found_dotenv}")
+load_dotenv(found_dotenv, override=False)
+
+path = os.path.dirname(found_dotenv)
+print(f"Adding {path} to sys.path")
+sys.path.append(path)
+
+from llm_toolkit.llm_utils import *
+from llm_toolkit.logical_reasoning_utils import *
+
+device = check_gpu()
+is_cuda = torch.cuda.is_available()
+
+model_name = os.getenv("MODEL_NAME")
+adapter_name_or_path = os.getenv("ADAPTER_NAME_OR_PATH")
+load_in_4bit = os.getenv("LOAD_IN_4BIT") == "true"
+data_path = os.getenv("LOGICAL_REASONING_DATA_PATH")
+results_path = os.getenv("LOGICAL_REASONING_RESULTS_PATH")
+use_english_datasets = os.getenv("USE_ENGLISH_DATASETS") == "true"
+batch_size = int(os.getenv("BATCH_SIZE", 1))
+using_llama_factory = os.getenv("USING_LLAMA_FACTORY") == "true"
+max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", 2048))
+start_num_shots = int(os.getenv("START_NUM_SHOTS", 0))
+
+print(
+    model_name,
+    adapter_name_or_path,
+    load_in_4bit,
+    data_path,
+    results_path,
+    max_new_tokens,
+    batch_size,
+)
+
+dtype = (
+    torch.float32
+    if os.getenv("USE_FLOAT32_FOR_INFERENCE") == "true"
+    else (
+        torch.bfloat16
+        if os.getenv("USE_BF16_FOR_INFERENCE") == "true"
+        else torch.float16
+    )
+)
+
+if is_cuda:
+    torch.cuda.empty_cache()
+    gpu_stats = torch.cuda.get_device_properties(0)
+    start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+    max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
+    print(f"(0) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
+    print(f"{start_gpu_memory} GB of memory reserved.")
+
+    torch.cuda.empty_cache()
+
+model, tokenizer = load_model(
+    model_name,
+    load_in_4bit=load_in_4bit,
+    adapter_name_or_path=adapter_name_or_path,
+    using_llama_factory=using_llama_factory,
+    dtype=dtype,
+)
+
+if is_cuda:
+    gpu_stats = torch.cuda.get_device_properties(0)
+    start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+    max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
+    print(f"(2) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
+    print(f"{start_gpu_memory} GB of memory reserved.")
+
+
+def on_num_shots_step_completed(model_name, dataset, predictions):
+    save_results(
+        model_name,
+        results_path,
+        dataset,
+        predictions,
+    )
+
+    metrics = calc_metrics(dataset["label"], predictions, debug=True)
+    print(f"{model_name} metrics: {metrics}")
+
+
+if adapter_name_or_path is not None:
+    model_name += "/" + adapter_name_or_path.split("/")[-1]
+
+
+def evaluate_model_with_num_shots(
+    model,
+    tokenizer,
+    model_name,
+    data_path,
+    start_num_shots=0,
+    range_num_shots=[10],
+    batch_size=1,
+    max_new_tokens=2048,
+    device="cuda",
+):
+    print(f"Evaluating model: {model_name} on {device}")
+
+    for num_shots in range_num_shots:
+        if num_shots < start_num_shots:
+            continue
+
+        print(f"*** Evaluating with num_shots: {num_shots}")
+
+        datasets = load_logical_reasoning_dataset(
+            data_path,
+            tokenizer=tokenizer,
+            chinese_prompt=not use_english_datasets,
+            using_p1=False,
+        )
+        if len(sys.argv) > 1:
+            num = int(sys.argv[1])
+            if num > 0:
+                print(f"--- evaluating {num} entries")
+                datasets["test"] = datasets["test"].select(range(num))
+
+        print_row_details(datasets["test"].to_pandas(), indices=[0, -1])
+
+        predictions = eval_model(
+            model,
+            tokenizer,
+            datasets["test"],
+            device=device,
+            batch_size=batch_size,
+            max_new_tokens=max_new_tokens,
+        )
+
+        model_name_with_rp = f"{model_name}/shots-{num_shots:02d}"
+
+        try:
+            on_num_shots_step_completed(
+                model_name_with_rp,
+                datasets["test"],
+                predictions,
+            )
+        except Exception as e:
+            print(e)
+
+
+evaluate_model_with_num_shots(
+    model,
+    tokenizer,
+    model_name,
+    data_path,
+    batch_size=batch_size,
+    max_new_tokens=max_new_tokens,
+    device=device,
+    start_num_shots=start_num_shots,
+)
+
+if is_cuda:
+    gpu_stats = torch.cuda.get_device_properties(0)
+    start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+    max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
+    print(f"(3) GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
+    print(f"{start_gpu_memory} GB of memory reserved.")

llm_toolkit/logical_reasoning_utils.py

@@ -294,7 +294,7 @@ def load_logical_reasoning_dataset(
 
             model_name = os.getenv("MODEL_NAME")
 
-            if "mistral" in model_name.lower() or "gemma" in model_name.lower():
+            if "gemma" in model_name.lower():
                 messages = messages[1:]
 
             texts = []

scripts/eval-mgtv-shots.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+BASEDIR=`pwd`
+
+nvidia-smi
+uname -a
+cat /etc/os-release
+lscpu
+grep MemTotal /proc/meminfo
+
+$BASEDIR/scripts/eval-shots.sh shenzhi-wang Mistral-7B-v0.3-Chinese-Chat
+
+$BASEDIR/scripts/eval-shots.sh internlm internlm2_5-7b-chat
+
+$BASEDIR/scripts/eval-shots.sh internlm internlm2_5-7b-chat-1m
+
+$BASEDIR/scripts/eval-shots.sh Qwen Qwen2-7B-Instruct
+
+$BASEDIR/scripts/eval-shots.sh shenzhi-wang Llama3.1-8B-Chinese-Chat

scripts/eval-shots.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+BASEDIR=$(dirname "$0")
+cd $BASEDIR/..
+echo Current Directory:
+pwd
+
+export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
+export RESIZE_TOKEN_EMBEDDINGS=true
+export USING_LLAMA_FACTORY=true
+export USING_P1_PROMPT_TEMPLATE=false
+export LOAD_IN_4BIT=false
+
+export ORG_NAME=$1
+export MODEL=$2
+export MODEL_NAME=$ORG_NAME/$MODEL
+
+export LOGICAL_REASONING_RESULTS_PATH=data/${MODEL}_results.csv
+
+echo Evaluating $MODEL_NAME with few-shot learning
+python llm_toolkit/eval_shots.py