ehartford commited on
Commit
8a7827c
1 Parent(s): 50c0072

Upload eval.sh with huggingface_hub

Browse files
Files changed (1) hide show
  1. eval.sh +44 -0
eval.sh ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ MODEL_PATH="cognitivecomputations/dolphin-2.8-mistral-7b-v02"
4
+ MODEL_NAME="dolphin-2.8-mistral-7b-v02"
5
+ RESULTS_PATH="/workspace/results/$MODEL_NAME"
6
+ mkdir -p "$RESULTS_PATH"
7
+
8
+ MODEL_ARGS="pretrained=$MODEL_PATH,dtype=auto"
9
+
10
+ tasks=(
11
+ "truthfulqa"
12
+ "winogrande"
13
+ "gsm8k"
14
+ "hellaswag"
15
+ "arc_challenge"
16
+ "mmlu"
17
+ )
18
+
19
+ # Function to get the number of fewshot for a given task
20
+ get_num_fewshot() {
21
+ case "$1" in
22
+ "mmlu") echo 5 ;;
23
+ "truthfulqa") echo 0 ;;
24
+ "gsm8k") echo 5 ;;
25
+ "hellaswag") echo 10 ;;
26
+ "arc_challenge") echo 25 ;;
27
+ "winogrande") echo 5 ;;
28
+ *) echo 0 ;;
29
+ esac
30
+ }
31
+
32
+ for TASK in "${tasks[@]}"; do
33
+ lm_eval --model hf --model_args "$MODEL_ARGS" --task="$TASK" --num_fewshot "$(get_num_fewshot "$TASK")" --device cuda:0 --batch_size 8 --output_path "$RESULTS_PATH/$TASK.json"
34
+ # lm_eval --model vllm --model_args "$MODEL_ARGS" --task="$TASK" --num_fewshot "$(get_num_fewshot "$TASK")" --batch_size 8 --output_path "$RESULTS_PATH/$TASK.json"
35
+
36
+ done
37
+
38
+
39
+ jq -s '[.[]]' $RESULTS_PATH/*.json > $RESULTS_PATH/eval_results.json
40
+
41
+ huggingface-cli upload cognitivecomputations/$MODEL_NAME $RESULTS_PATH/eval_results.json
42
+ huggingface-cli upload cognitivecomputations/$MODEL_NAME eval.sh
43
+
44
+ # docker run -it --network=host --group-add=video --ipc=host --cap-add=SYS_PTRACE --security-opt seccomp=unconfined --device /dev/kfd --device /dev/dri -v /workspace/models/dolphin-phi-kensho:/app/model embeddedllminfo/vllm-rocm bash