huihui-ai commited on
Commit
d1f77df
1 Parent(s): fa62f7c

Upload eval.sh

Browse files
Files changed (1) hide show
  1. eval.sh +36 -0
eval.sh ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Install required package
2
+ pip install antlr4-python3-runtime==4.11 immutabledict langdetect nltk lm_eval
3
+
4
+
5
+ python -c "import nltk; nltk.download('punkt')"
6
+
7
+ MODEL_PATHS=(
8
+ huihui-ai/Meta-Llama-3.1-8B-Instruct-abliterated
9
+ )
10
+
11
+ for MODEL_PATH in "${MODEL_PATHS[@]}"; do
12
+ MODEL_NAME=$(basename "$MODEL_PATH")
13
+ MODEL_DIR="./results/$MODEL_NAME"
14
+ mkdir -p "$MODEL_DIR"
15
+
16
+ MODEL_ARGS="trust_remote_code=True,pretrained=$MODEL_PATH,dtype=bfloat16"
17
+
18
+ BASE_COMMAND="accelerate launch -m lm_eval --model hf --model_args $MODEL_ARGS --batch_size 4 --fewshot_as_multiturn --apply_chat_template"
19
+
20
+ # IFEval
21
+ $BASE_COMMAND --tasks leaderboard_ifeval --fewshot_as_multiturn --output_path "$MODEL_DIR/ifeval"
22
+
23
+ # BBH (Big-Bench Hard)
24
+ $BASE_COMMAND --tasks leaderboard_bbh --num_fewshot 3 --fewshot_as_multiturn --output_path "$MODEL_DIR/bbh"
25
+
26
+ # GPQA
27
+ $BASE_COMMAND --tasks leaderboard_gpqa --fewshot_as_multiturn --output_path "$MODEL_DIR/gpqa"
28
+
29
+ # MMLU-Pro
30
+ $BASE_COMMAND --tasks leaderboard_mmlu_pro --num_fewshot 5 --fewshot_as_multiturn --output_path "$MODEL_DIR/mmlu_pro"
31
+
32
+ # TruthfulQA
33
+ $BASE_COMMAND --tasks truthfulqa_mc2 --fewshot_as_multiturn --output_path "$MODEL_DIR/truthfulqa"
34
+
35
+
36
+ done