evaluation/infinite_bench_eval/test_vllm.sh

#########################################################################
# File Name: 1.comb.tstout.ref.sh
# Author: Xianchao Wu
# mail: xianchaow@nvidia.com
# Created Time: Tue Jun 11 08:14:18 2024
#########################################################################
#!/bin/bash

datetime=$(date +%Y%m%d)
echo ${datetime}

model_path="" # Your_model_path https://huggingface.co/nvidia/Llama3-ChatQA-2-70B/
indir=${model_path}/outputs
data_home="" # https://huggingface.co/nvidia/Llama3-ChatQA-2-70B/tree/main/data

function longbook_eng_eval(){
	data_name=$1
	pxout_txt=$2
	pxref_json=$3	

	task_name="${data_name}_eng"

	pxout_ref_json="${pxout_txt}.${datetime}.json"

	python3 prepare_json_from_pxout.py --task ${task_name} \
		--pxout_txt ${pxout_txt} \
		--pxref_json ${pxref_json} \
		--pxout_ref_json ${pxout_ref_json} \
		--sep_by_assistant

	if [[ $task_name =~ "longbook_choice" ]]
	then
		python3 compute_scores_2sets.py \
			--task ${task_name} \
			--pxout_ref_json ${pxout_ref_json} \
			--model_name pxlong \
			--use_zero_scrolls
	else
          	python3 compute_scores_2sets.py \
       		    --task ${task_name} \
       		    --pxout_ref_json ${pxout_ref_json} \
       		    --model_name pxlong
	fi
}

for afile in `ls $indir/long*.txt`
do
	echo $afile
	for data_name in "longbook_qa" "longbook_choice" "longbook_sum" "longdialogue_qa"
	do
		task_name="${data_name}_eng"
		if [[ $afile =~ $task_name ]]
		then
			# TODO
			echo "do $task_name for $afile"
			pxref_json="${data_home}/${task_name}_gpt4_same/test.json"
			echo ${data_name} ${afile} ${pxref_json}
			longbook_eng_eval ${data_name} ${afile} ${pxref_json}
			echo "--------"
		fi
		#break
	done
	#break
done