p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/franz/berlin/coinrun/mila_submission/250M_dataset/coinrun_lam_base.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/coinrun/lam/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/coinrun/lam/%x_%j.log
#SBATCH --job-name=lam_coinrun_mila_submission
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi

# Log the sbatch script
cat $0

source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

tags="coinrun lam 250m_dataset mila_submission"

array_records_dir="/fast/project/HFMI_SynergyUnit/jafar_ws/data/coinrun/array_records_250M_npy_arr_rec/array_record/"
CHECKPOINT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/checkpoints/coinrun/lam/${job_name}_${slurm_job_id}"
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

current_branch=$(git rev-parse --abbrev-ref HEAD)
if [ "$current_branch" != "prepend-action-maskgit" ]; then
    echo "This script must be run from the prepend-action-maskgit branch. Current branch is $current_branch. Exiting."
    exit 1
fi

srun python jasmine/train_lam.py \
    --patch_size=16 \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --name="${job_name}_${slurm_job_id}" \
    --tags $tags \
    --entity instant-uv \
    --project jafar \
    --data_dir="${array_records_dir}/train" \
    --val_data_dir="${array_records_dir}/val" &

child_pid=$!

wait $child_pid
[tab] slurm/jobs/franz/berlin/coinrun/mila_submission/250M_dataset/coinrun_tokenizer_base.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --gres=gpu:1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=8
#SBATCH --output=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/coinrun/tokenizer/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/jafar_ws/logs/franz/coinrun/tokenizer/%x_%j.log
#SBATCH --job-name=tokenizer_coinrun_mila_submission
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi

# Log the sbatch script
cat $0

source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

tags="coinrun tokenizer 250m_dataset mila_submission"

array_records_dir="/fast/project/HFMI_SynergyUnit/jafar_ws/data/coinrun/array_records_250M_npy_arr_rec/array_record/"
CHECKPOINT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/checkpoints/coinrun/tokenizer/${job_name}_${slurm_job_id}"
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

current_branch=$(git rev-parse --abbrev-ref HEAD)
if [ "$current_branch" != "prepend-action-maskgit" ]; then
    echo "This script must be run from the prepend-action-maskgit branch. Current branch is $current_branch. Exiting."
    exit 1
fi

srun python jasmine/train_tokenizer.py \
    --patch_size=16 \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --name="${job_name}_${slurm_job_id}" \
    --tags $tags \
    --entity instant-uv \
    --project jafar \
    --data_dir="${array_records_dir}/train" \
    --val_data_dir="${array_records_dir}/val" &

child_pid=$!

wait $child_pid
[tab] data/test_vis/actions.txt (plaintext)
0	8
1	1
2	5
3	9
4	8
5	2
6	9
7	1
8	13
9	12
10	12
11	3
12	6
13	5
14	9
15	0
16	14
17	10
18	7
19	1
20	2
21	0
22	6
23	10
24	7
25	9
26	6
27	0
28	2
29	11
30	12
31	14
32	12
33	12
34	4
35	7
36	1
37	12
38	6
39	6
40	1
41	11
42	7
43	11
44	0
45	0
46	1
47	5
48	12
49	9
50	9
51	10
52	4
53	0
54	3
55	2
56	2
57	8
58	3
59	5
60	1
61	7
62	4
63	12
64	2
65	5
66	9
67	5
68	13
69	13
70	0
71	8
72	10
73	4
74	5
75	14
76	0
77	6
78	0
79	10
80	5
81	10
82	8
83	8
84	13
85	3
86	10
87	13
88	4
89	12
90	11
91	9
92	10
93	1
94	14
95	10
96	11
97	13
98	5
99	10
100	12
101	3
102	3
103	2
104	13
105	2
106	0
107	2
108	13
109	14
110	7
111	14
112	4
113	7
114	11
115	2
116	0
117	13
118	14
119	8
120	4
121	14
122	3
123	4
124	3
125	13
126	13
127	11
128	0
129	0
130	10
131	6
132	2
133	12
134	3
135	1
136	13
137	9
138	10
139	12
140	5
141	14
142	4
143	6
144	5
145	4
146	12
147	13
148	11
149	8
150	9
151	10
152	5
153	2
154	7
155	6
156	6
157	3
158	13
159	13