p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/alfred/berlin/lm_harness_eval/convert_eval_spawner.sh (shellscript)
#!/usr/bin/env bash


experiment_names=(
"qwen3_600m_full_finetune_crowd_code_dataset_seqlen_16k_token_batch_size_128k/2025-11-24_18-17-40"
# "qwen3_600m_full_finetune_crowd_code_dataset_seqlen_4k_token_batch_size_128k/2025-11-24_18-06-34"
# "qwen3_600m_full_finetune_crowd_code_dataset_seqlen_8k_token_batch_size_128k/2025-11-24_18-09-07"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_16k_token_batch_size_128k/2025-11-24_18-30-35"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_32k_token_batch_size_128k/2025-11-24_18-30-06"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_4k_token_batch_size_128k/2025-11-24_18-21-48"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_8k_token_batch_size_128k/2025-11-24_18-30-26"
)

for experiment_name in "${experiment_names[@]}"; do
    EXPERIMENT_NAME=${experiment_name} \
    sbatch \
    --job-name="convert_and_eval_${experiment_name}" \
    -qos low \
    slurm/dev/alfred/berlin/lm_harness_eval/convert_eval_spawner.sbatch

done
[tab] slurm/dev/alfred/berlin/lm_harness_eval/convert_eval.sh (shellscript)
#!/usr/bin/env bash

source .venv/bin/activate
module load CUDA/12.8


echo "================================================"
echo "Finding latest checkpoint ..."
echo "================================================"

experiment_name="qwen3_600m_full_finetune_crowd_code_dataset_seqlen_16k_token_batch_size_128k/2025-11-24_18-17-40"

workspace="/fast/project/HFMI_SynergyUnit/tab_model/data/outputs"
workspace_hf="${workspace}/hf"

latest_ckpt=$(find "${workspace}/${experiment_name}/checkpoints" -maxdepth 1 -type d -regex ".*\(last\|latest\).*" | sort | tail -n 1)

echo "Latest checkpoint: ${latest_ckpt}"

# exit if latest_ckpt is not found
if [ -z "${latest_ckpt}" ]; then
    echo "Latest checkpoint not found"
    exit 1
fi

nemo_ckpt_path="${latest_ckpt}"
hf_ckpt_target_path="${workspace_hf}/${experiment_name}"
mkdir -p "$(dirname "${hf_ckpt_target_path}")"

hf_ckpt_path=$(find "${hf_ckpt_target_path}/checkpoints" -maxdepth 1 -type d -regex ".*\(last\|latest\).*" | sort | tail -n 1)

echo ""
echo "================================================"
echo "Converting model to HF format ..."
echo "================================================"

# Export model
echo "Nemo model path: ${nemo_ckpt_path}"
echo "HF model target output path: ${hf_ckpt_target_path}"
echo "HF model checkpoint path: ${hf_ckpt_path}"

nemo llm export --yes \
    path="${nemo_ckpt_path}" \
    target="hf" \
    output_path="${hf_ckpt_target_path}"

echo ""
echo "================================================"
echo "Evaluating model ..."
echo "================================================"


mem_fraction_static=0.95
batch_size=32

wandb_name="${experiment_name}"

lm_eval --model sglang \
    --model_args pretrained=${hf_ckpt_path},mem_fraction_static=${mem_fraction_static} \
    --tasks gsm8k \
    --batch_size ${batch_size} \
    --wandb_args project=llm-coding-agent,name=${wandb_name} \
    --limit 100 \
[tab] slurm/jobs/alfred/berlin/coding_agent/lm_eval/convert_eval_spawner_if_eval.sh (shellscript)
#!/usr/bin/env bash


experiment_names=(
"qwen3_600m_full_finetune_crowd_code_dataset_seqlen_16k_token_batch_size_128k/2025-11-24_18-17-40"
"qwen3_600m_full_finetune_crowd_code_dataset_seqlen_4k_token_batch_size_128k/2025-11-24_18-06-34"
"qwen3_600m_full_finetune_crowd_code_dataset_seqlen_8k_token_batch_size_128k/2025-11-24_18-09-07"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_16k_token_batch_size_128k/2025-11-24_18-30-35"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_32k_token_batch_size_128k/2025-11-24_18-30-06"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_4k_token_batch_size_128k/2025-11-24_18-21-48"
# "qwen3_600m_lora_finetune_crowd_code_dataset_seqlen_8k_token_batch_size_128k/2025-11-24_18-30-26"
)

eval_tasks="ifeval"

for experiment_name in "${experiment_names[@]}"; do
    EXPERIMENT_NAME=${experiment_name} \
    EVAL_TASKS=${eval_tasks} \
    sbatch \
    --job-name="${eval_tasks}_${experiment_name}" \
    -q low \
    slurm/jobs/alfred/berlin/coding_agent/lm_eval/convert_eval_spawner.sbatch
done