eval_all_gen.sh

#!/bin/bash
#SBATCH --exclude=nid005159
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=4
#SBATCH --mem=256G
#SBATCH -p small-g
#SBATCH -t 2-0:00:00
#SBATCH --gpus-per-node=mi250:0
#SBATCH --exclusive=user
#SBATCH --hint=nomultithread
#SBATCH --account=project_462000119
#SBATCH -o logs/%j.out
#SBATCH -e logs/%j.err

# if run without sbatch, invoke here
if [ -z $SLURM_JOB_ID ]; then
    mkdir -p logs
    sbatch "$0"
    exit
fi

set -euo pipefail

# symlink logs/latest_eval.out and logs/latest_eval.err
ln -f -s $SLURM_JOB_ID.out logs/latest_eval.out
ln -f -s $SLURM_JOB_ID.err logs/latest_eval.err

source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate

echo "START TIME: $(date)"

# defining the right environment variables
export HF_DATASETS_OFFLINE=1
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache

# Converted transformer checkpoint
# cd /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/bigscience/lm-evaluation-harness

# Data
CONFIGS=(
copa,"best_option",validation
copa,"choose",validation
copa,"i_am_hesitating",validation
copa,"cause_effect",validation
copa,"plausible_alternatives",validation
superglue_rte,"MNLI crowdsource",validation
superglue_rte,"GPT-3 style",validation
superglue_rte,"does it follow that",validation
superglue_rte,"should assume",validation
superglue_rte,"guaranteed true",validation
anli_r1,"guaranteed/possible/impossible",dev_r1
anli_r1,"MNLI crowdsource",dev_r1
anli_r1,"GPT-3 style",dev_r1
anli_r1,"justified in saying",dev_r1
anli_r1,"can we infer",dev_r1
anli_r2,"guaranteed/possible/impossible",dev_r2
anli_r2,"MNLI crowdsource",dev_r2
anli_r2,"GPT-3 style",dev_r2
anli_r2,"justified in saying",dev_r2
anli_r2,"can we infer",dev_r2
anli_r3,"guaranteed/possible/impossible",dev_r3
anli_r3,"MNLI crowdsource",dev_r3
anli_r3,"GPT-3 style",dev_r3
anli_r3,"justified in saying",dev_r3
anli_r3,"can we infer",dev_r3
cb,"guaranteed/possible/impossible",validation
cb,"MNLI crowdsource",validation
cb,"GPT-3 style",validation
cb,"justified in saying",validation
cb,"can we infer",validation
winogrande,"underscore refer to",validation
winogrande,"Replace",validation
winogrande,"stand for",validation
winogrande,"does underscore refer to",validation
winogrande,"True or False",validation
story_cloze_2016,"Story Continuation and Options",validation
story_cloze_2016,"Answer Given options",validation
story_cloze_2016,"Novel Correct Ending",validation
story_cloze_2016,"Generate Ending",validation
story_cloze_2016,"Choose Story Ending",validation
boolq,"after_reading",validation
boolq,"GPT-3 Style",validation
boolq,"yes_no_question",validation
boolq,"exercise",validation
boolq,"valid_binary",validation
arc_easy,"pick_the_most_correct_option",test
arc_easy,"qa_options",test
arc_easy,"i_am_hesitating",test
arc_easy,"multiple_choice",test
arc_easy,"heres_a_problem",test
arc_challenge,"pick_the_most_correct_option",test
arc_challenge,"qa_options",test
arc_challenge,"i_am_hesitating",test
arc_challenge,"multiple_choice",test
arc_challenge,"heres_a_problem",test
sciq,"Direct Question (Closed Book)",test
sciq,"Multiple Choice (Closed Book)",test
sciq,"Multiple Choice Question First",test
sciq,"Multiple Choice",test
sciq,"Direct Question",test
piqa,"what_is_the_correct_ending"
piqa,"pick_correct_choice_index"
piqa,"Correct the solution"
piqa,"choose the most appropriate solution"
piqa,"no prompt needed"
GEM/wiki_lingua_en,"tldr_en"
GEM/wiki_lingua_en,"article_summary_en"
GEM/wiki_lingua_en,"summarize_above_en"
GEM/wiki_lingua_en,"rephrase_en"
GEM/wiki_lingua_en,"write_abstract_en"
gem_xsum,"article_DOC_summary"
gem_xsum,"summarize_DOC"
gem_xsum,"summarize_this_DOC_summary"
gem_xsum,"DOC_tldr"
gem_xsum,"DOC_boils_down_to_simple_idea_that"
GEM/web_nlg_en,"PALM_prompt"
GEM/web_nlg_en,"explicit-graph-description2"
GEM/web_nlg_en,"non-explicit-description"
GEM/web_nlg_en,"very-explicit-description"
GEM/web_nlg_en,"implicit-graph-description"
e2e_nlg_cleaned,"generate_text_restaurant"
e2e_nlg_cleaned,"coherent_text"
e2e_nlg_cleaned,"create_text_for_me"
e2e_nlg_cleaned,"generate_gramatically_correct_text"
e2e_nlg_cleaned,"text"
)

CONFIGS=(
GEM/wiki_lingua_en,"tldr_en"
gem_xsum,"article_DOC_summary"
GEM/web_nlg_en,"PALM_prompt"
e2e_nlg_cleaned,"generate_text_restaurant"
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b12bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b17bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b21bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b28bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b42bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4-repetitions/4b284b84bc4/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b12boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b17boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b21boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b28boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b1b25oscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b4boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b9boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b11boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b14boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b18boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b28boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-oscar-repetitions/2b855b55boscar/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/transformers
)

CKPTSX=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b25b/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b35b/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b44b/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b58b/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-c4-repetitions/8b7178b88b/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b44b/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-8b7-178b-oscar-repetitions/8b7178b88b/transformers
)


CKPTSX=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b42boscar/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-oscar-repetitions/4b284b84boscar/transformers
)

CKPTSX=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed1/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed2/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed3/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-4b2-84b-c4seeds/4b284b84bc4seed4/transformers
)

CKPTSX=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed1/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed2/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4seeds/2b855b55bc4seed3/transformers
)
CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
)


CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b1b25c4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b4bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b9bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b11bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b14bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b18bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b28bc4/transformers
/pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/lm1-2b8-55b-c4-repetitions/2b855b55bc4/transformers
)

CKPTS=(
/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/lm3-2b8-55b-c4/transformers
)

FEWSHOT_CONFIGS=(
0
1
2
3
4
5
)

TOKENIZER=/pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/gpt2


# Iterate through all possible combinations of data config, model ckpt & fewshot config and run the jobs
for ((i=0; i<${#CKPTS[@]}; i++)); do
    for ((j=0; j<${#FEWSHOT_CONFIGS[@]}; j++)); do
        for ((k=0; k<${#CONFIGS[@]}; k++)); do
            #echo "sbatch --export=CKPT=${CKPTS[$i]},FEWSHOT_CONFIG=${FEWSHOT_CONFIGS[$j]},DATASET=${DATASETS[$k]} eval.sh"
            
            DATA_CONFIG=${CONFIGS[$k]}
            IFS=',' read dataset_name template_name x <<< "${DATA_CONFIG}"
            MODEL_CKPT=${CKPTS[$i]}
            MODEL_CKPT_NO_TRF=${MODEL_CKPT%/*}
            MODEL_NAME=${MODEL_CKPT_NO_TRF##*/}
            OUTPUT_PATH=$MODEL_CKPT_NO_TRF/evaluation/generation_sss_denoiser
            mkdir -p $OUTPUT_PATH
            OUTPUT_NAME=$MODEL_NAME\_$dataset_name\_$template_name\_${FEWSHOT_CONFIGS[$j]}

            eval_script="./eval_$i-$j-$k.slurm"
            cat <<EOT > $eval_script
#!/bin/bash
#SBATCH --exclude=nid005159
#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --cpus-per-task=8
#SBATCH --mem=256G
#SBATCH -p small-g
#SBATCH -t 2-0:00:00
#SBATCH --gpus-per-node=mi250:1
#SBATCH --exclusive=user
#SBATCH --hint=nomultithread
#SBATCH --account=project_462000119
#SBATCH -o logs/%j.out
#SBATCH -e logs/%j.err
set -euo pipefail
# symlink logs/latest_eval.out and logs/latest_eval.err
ln -f -s "\$SLURM_JOB_ID.out" logs/latest_eval.out
ln -f -s "\$SLURM_JOB_ID.err" logs/latest_eval.err
source /pfs/lustrep4/scratch/project_462000119/muennighoff/nov-2022-bettercom/venv/bin/activate
echo "START TIME: $(date)"
# defining the right environment variables
export HF_DATASETS_OFFLINE=1
export HF_DATASETS_CACHE=/scratch/project_462000119/ds_cache
# Converted transformer checkpoint
cd /pfs/lustrep4/scratch/project_462000119/muennighoff/dec-2022-ul2/bigscience/lm-evaluation-harness
python main.py \
    --model_api_name 'hf-causal' \
    --model_args pretrained=${CKPTS[$i]},use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
    --device cuda \
    --batch_size 16 \
    --no_tracking \
    --task_name $dataset_name \
    --template_names "$template_name" \
    --bootstrap_iters 10 \
    --limit 3000 \
    --num_fewshot ${FEWSHOT_CONFIGS[$j]} \
    --output_dir $OUTPUT_PATH \
    --output_path "$OUTPUT_NAME"
python main.py \
    --model_api_name 'hf-causal' \
    --model_args pretrained=${CKPTS[$i]},use_accelerate=True,tokenizer=$TOKENIZER,dtype=bfloat16 \
    --device cuda \
    --batch_size 8 \
    --no_tracking \
    --task_name $dataset_name \
    --template_names "$template_name" \
    --bootstrap_iters 10 \
    --limit 3000 \
    --num_fewshot ${FEWSHOT_CONFIGS[$j]} \
    --output_dir $OUTPUT_PATH \
    --output_path "$OUTPUT_NAME"
echo "END TIME: $(date)"
EOT
            # Submit the job
            sbatch $eval_script
            # Sleep for a bit to avoid hitting the job submission limit
            sleep 0.1
        done
    done
done


echo "END TIME: $(date)"