eval.sh

# Install required package
pip install antlr4-python3-runtime==4.11 immutabledict langdetect

python -c "import nltk; nltk.download('punkt')"

MODEL_PATHS=(
 meta-llama/Meta-Llama-3.1-8B-Instruct
)

for MODEL_PATH in "${MODEL_PATHS[@]}"; do
  MODEL_NAME=$(basename "$MODEL_PATH")
  MODEL_DIR="./results/$MODEL_NAME"
  mkdir -p "$MODEL_DIR"
 
  MODEL_ARGS="trust_remote_code=True,pretrained=$MODEL_PATH,dtype=bfloat16"
  
  BASE_COMMAND="accelerate launch -m lm_eval --model hf --model_args $MODEL_ARGS --batch_size 4 --fewshot_as_multiturn --apply_chat_template"
  
  # IFEval
  $BASE_COMMAND --tasks leaderboard_ifeval --fewshot_as_multiturn --output_path "$MODEL_DIR/ifeval"

  # BBH (Big-Bench Hard)
  $BASE_COMMAND --tasks leaderboard_bbh --num_fewshot 3 --fewshot_as_multiturn --output_path "$MODEL_DIR/bbh"

  # GPQA
  $BASE_COMMAND --tasks leaderboard_gpqa --fewshot_as_multiturn --output_path "$MODEL_DIR/gpqa"

  # MMLU-Pro
  $BASE_COMMAND --tasks leaderboard_mmlu_pro --num_fewshot 5 --fewshot_as_multiturn --output_path "$MODEL_DIR/mmlu_pro"

  # TruthfulQA
  $BASE_COMMAND --tasks truthfulqa_mc2 --fewshot_as_multiturn --output_path "$MODEL_DIR/truthfulqa"

  
done