p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/sbatch_files/sbatch_qwen_1.7b_lora_bs16.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0600-0699]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/qwen3-0.6b-sft-torch-crowd-code/%x_%j.log
#SBATCH --job-name=qwen_1.7b_lora_bs_16


trap 'echo "Job killed by signal"' USR1

cat $0

module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

source .venv/bin/activate

srun sh slurm/jobs/mihir/juelich/crowd-pilot/1-wave-anta-santa/qwen_1.7b_lora_bs_16.sh
[tab] slurm/dev/mihir/berlin/crowd_pilot/install_sglang.sh (shellscript)
#! /bin/bash

# NOTE: Make sure that the project is set correctly using jutil
mkdir $PROJECT/mihir/
export CARGO_HOME=$PROJECT/mihir/.cargo
export RUSTUP_HOME=$PROJECT/mihir/.rustup

source .venv/bin/activate

# NOTE: A local rust installation already exists in the project root
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
uv pip install "sglang" --prerelease=allow
[tab] slurm/dev/alfred/juwels/evals/evaluate/evaluate_qwen_no_launch_server.sh (shellscript)
#!/usr/bin/env bash

source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"


# Generation
generator_host="jwb0065i"
generator_port=30000
generator_model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"
context_length=40960
temperature=0.7
num_samples=5

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
generations_file="data/eval/output/handcrafted_nls_debug_temp_code_think_generations.jsonl"
evaluations_file="data/eval/output/handcrafted_nls_debug_temp_code_think_evaluations.jsonl"

# Evaluation
judge_host="jwb0001i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Thinking-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_30B_code_think"
wandb_tags="qwen3_30b debug"
wandb_eval_type="handcrafted_nls_debug"


# python src/evaluation/sglang_generations.py \
#   --test_cases_file $test_cases_file \
#   --generations_file $generations_file \
#   --model_path $generator_model_path \
#   --context_length=$context_length \
#   --server_host $generator_host \
#   --server_port $generator_port \
#   --temperature $temperature \
#   --num_samples $num_samples \
#   --no_launch_server


python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --limit 2 \
  --concurrency 1 \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/alfred/juwels/evals/lora_runs/eval_qwen_1.7B.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=0:20:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=eval_lora_runs_qwen_1.7B_ckpt_13032800


source .venv/bin/activate

module load Stages/2026 CUDA/13

cat "$0"


# Generation
generator_host="0.0.0.0"
generator_port=30000
generator_model_path="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/snapshots/70d244cc86ccca08cf5af4e1e306ecf908b1ad5e"
lora_path="/p/home/jusers/nguyen31/juwels/envcomp/crowd-pilot/tab-model-eval/checkpoints/lora_run/13032800/checkpoints/iter_0010000"
context_length=40960
temperature=0.7
num_samples=10
top_p=0.8
top_k=20
min_p=0.0

test_cases_file="data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"
output_dir="data/eval/output/lora_runs"
generations_file="${output_dir}/qwen_1.7B_ckpt_13032800_generations.jsonl"
evaluations_file="${output_dir}/qwen_1.7B_ckpt_13032800_evaluations.jsonl"

mkdir -p $output_dir

# Evaluation
judge_host="jwb0129i"
judge_port=30000
judge_model_path="Qwen/Qwen3-30B-A3B-Instruct-2507"
judge_context_length=40960
judge_prompt_file="data/prompts/judge_prompt_v2.md"
wandb_name="qwen_1.7B_ckpt_13032800"
wandb_tags="lora_run qwen_1.7B"
wandb_eval_type="handcrafted"


python src/evaluation/sglang_generations.py \
  --test_cases_file $test_cases_file \
  --generations_file $generations_file \
  --model_path $generator_model_path \
  --context_length=$context_length \
  --server_host $generator_host \
  --server_port $generator_port \
  --temperature $temperature \
  --top_p $top_p \
  --top_k $top_k \
  --min_p $min_p \
  --num_samples $num_samples \
  --lora_paths $lora_path \
  --mem_fraction_static 0.80 \
  --tp_size 4 \



python src/evaluation/sglang_eval.py \
  --wandb_name $wandb_name \
  --wandb_tags $wandb_tags \
  --generations_file $generations_file \
  --evaluations_file $evaluations_file \
  --judge_model_path $judge_model_path \
  --server_host $judge_host \
  --server_port $judge_port \
  --context_length $judge_context_length \
  --judge_prompt_file $judge_prompt_file \
  --temperature $temperature \
  --wandb_eval_type $wandb_eval_type \
  --no_launch_server
[tab] slurm/dev/alfred/juwels/evals/evaluate/host_qwen_1.7B.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=2:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --partition=develbooster
#SBATCH --account=envcomp
#SBATCH --mem=200GB
#SBATCH --output=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --error=/p/home/jusers/nguyen31/juwels/envcomp/logs/%x_%j.log
#SBATCH --job-name=host_qwen3_1.7B

FLASHINFER_CACHE_DIR=$PROJECT_envcomp/$USER/.cache/flashinfer

source .venv/bin/activate


module load Stages/2026 CUDA/13

model_id="/p/home/jusers/nguyen31/juwels/envcomp/huggingface/hub/models--Qwen--Qwen3-1.7B/blobs/4783fe10ac3adce15ac8f358ef5462739852c569"

python3 -m sglang.launch_server \
    --model-path $model_id \
    --reasoning-parser deepseek-r1 \
    --host 0.0.0.0 \
    --tp-size 1

    # --reasoning-parser qwen3 \
[content] slurm/jobs/mihir/juelich/crowd-pilot/eval/test.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=04:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=300GB
#SBATCH --partition=booster
#SBATCH --account=envcomp
#SBATCH --output=/p/project1/envcomp/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/eval/eval_miles_run/%x_%j.log
#SBATCH --job-name=eval_miles_run
[tab] slurm/jobs/mihir/juelich/crowd-pilot/eval/launch_judge_sglang.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=04:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=300GB
#SBATCH --partition=booster
#SBATCH --account=envcomp
#SBATCH --output=/p/project1/envcomp/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/eval/eval-judge-qwen-coder/%x_%j.log
#SBATCH --job-name=run_judge

module load Stages/2026 CUDA/13

export WANDB_MODE=offline

FLASHINFER_CACHE_DIR=$PROJECT/.cache/flashinfer

source .venv/bin/activate

export HF_HUB_ENABLE_HF_TRANSFER=1 

model_id="/p/scratch/envcomp/mihir/huggingface/Qwen3-32B"

python3 -m sglang.launch_server --model-path $model_id --host 0.0.0.0
[content] slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run.sh (shellscript)
" ;;
    8B|8b)       BASE_MODEL="Qwen3-8B" ;;
    14B|14b)     BASE_MODEL="Qwen3-14B" ;;
    32B|32b)     BASE_MODEL="Qwen3-32B" ;;
    30B-A3B|30b-a3b)     BASE_MODEL="Qwen3-30B-A3B" ;;
    235B-A22B|235b-a22b) BASE_MODEL="Qwen3-235B-A22B" ;;
    *)
        echo "Error: Unknown model size '$model_size' in RUN_NAME '$RUN_NAME'"
        exit 1
        ;;
esac

echo "Extracted from RUN_NAME: RUN_ID=$RUN_ID, BASE_MODEL=$BASE_MODEL"
[content] /p/project1/envcomp/mahajan1/miles/tab-model-eval/src/evaluation/sglang_eval.py (python)
        # For offline mode: ensure we always write to the same run directory
        # by setting WANDB_DIR to a consistent location based on run ID
        wandb_dir = os.path.join(os.getcwd(), "wandb_runs", args.wandb_id)
        os.makedirs(wandb_dir, exist_ok=True)
        os.environ["WANDB_DIR"] = wandb_dir
        os.environ["WANDB_RESUME"] = "allow"
        
        wandb_init_kwargs.update(
            {
                "id": args.wandb_id,
                "resume": "allow",
                "dir": wandb_dir,
[tab] slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_anta_claus.sh (shellscript)
# LAUNCH ANTA CLAUS EVAL JOBS



# 4B
sbatch /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run.sh qwen4B_no_lora_bs_8_13032814
sbatch /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run.sh qwen4B_lora_bs_8_13032811
sbatch /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run.sh qwen4B_lora_bs_16_13032809
sbatch /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run.sh qwen4B_no_lora_bs_16_13032812
sbatch /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run.sh qwen4B_no_lora_bs_32_13032813
sbatch /p/home/jusers/mahajan1/juwels/projects/envcomp/miles/slurm/jobs/mihir/juelich/crowd-pilot/eval/eval_miles_run.sh qwen4B_lora_bs_32_13032810