p-doom/crowd-code-dataset-0.2 huggingface dataset:
[terminal_command] TERMINAL ()
/bin/python /hkfs/home/project/hk-project-pai00039/tum_ind3695/.cursor-server/extensions/ms-python.python-2025.6.1-linux-x64/python_files/printEnvVariablesToFile.py /hkfs/home/project/hk-project-pai00039/tum_ind3695/.cursor-server/extensions/ms-python.python-2025.6.1-linux-x64/python_files/deactivate/bash/envVars.txt
[tab] slurm/dev/alfred/train_lam_dev/train_lam_single_batch.sh (shellscript)
module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID


CHECKPOINT_DIR=$ws_dir/checkpoints/lam/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR


# data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_1_shard_overfit'
data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards'
# data_dir = '/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_tfrecords_500_shards_overfit_1'

# srun python train_lam.py \
python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=48 \
    --log_checkpoint_interval=1000 \
    --log_image_interval=10 \
    --seed=0 \
    --min_lr=0.0000433 \
    --max_lr=0.0000433 \
    --log \
    --entity instant-uv \
    --data_dir $data_dir \
    --project jafar
[tab] slurm/dev/alfred/preprocess/preprocess_video_to_npy.sbatch (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=01:30:00
#SBATCH --partition=cpuonly
#SBATCH --account=hk-project-pai00039
#SBATCH --cpus-per-task=72
#SBATCH --output=logs/logs_preprocessing/%x_%j.log
#SBATCH --error=logs/logs_preprocessing/%x_%j.log
#SBATCH --mail-user=avocadoaling@gmail.com
#SBATCH --job-name=preprocess_video_to_npy
#SBATCH --mail-type=ALL

# Log the sbatch script
cat $0

source .venv/bin/activate

input_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms/"
output_path="/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_npy_tmp"

start_time=$(date +%s)  
python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
end_time=$(date +%s)
echo "Time taken: $((end_time - start_time)) seconds"




python utils/preprocess_video_to_npy.py --input_path $input_path --output_path $output_path
[tab] .gitignore (ignore)
*.pyc
*.npy
*.png
*.gif

wandb_key
checkpoints/
wandb/
__pycache__/

logs/
sandbox/
sbatch_scripts/
vs-code-recorder/
data/
data_tfrecords/
sh_scripts/
utils/clip_checker.py
utils/dataloader_seeding.py
utils/preprocess_video_splitter_tmp.py
requirements_franz.txt
sample_resolution_batches.py
train_dynamics_*
train_lam_*
train_tokenizer_*
notes.md
shell_scripts/
slurm/
[tab] slurm/dev/alfred/overfit_sample/train_tokenizer_overfit_sample_size_0_5.sh (shellscript)
# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv_jafar/bin/activate

ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'
tf_records_dir=$ws_dir/data/knoms_tfrecords_200_shards

slurm_job_id=$SLURM_JOB_ID

tags="overfit_sample tokenizer debug alfred"

CHECKPOINT_DIR=$ws_dir/checkpoints/$job_name_$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

# model_dim=32
# num_blocks=4
lr=1e-4

model_dim=256
num_blocks=8

job_name=cointok_mod_dim_${model_dim}_num_blocks_${num_blocks}_lr_${lr}

python train_tokenizer_single_sample.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=1 \
    --min_lr=$lr \
    --max_lr=$lr \
    --log_image_interval=500 \
    --log \
    --entity instant-uv \
    --project jafar \
    --name $job_name \
    --tags $tags \
    --model_dim $model_dim \
    --num_blocks $num_blocks \
    --data_dir $tf_records_dir
[tab] /home/hk-project-pai00039/tum_ind3695/.bashrc (shellscript)
# .bashrc

# Source global definitions
if [ -f /etc/bashrc ]; then
    . /etc/bashrc
fi

# User specific environment
if ! [[ "$PATH" =~ "$HOME/.local/bin:$HOME/bin:" ]]
then
    PATH="$HOME/.local/bin:$HOME/bin:$PATH"
fi
export PATH


alias queue='watch -n 1 squeue --me'
alias fqueue='squeue -o "%.10i %.16P %.60j %.8u %.8T %.10M %.9l %.6D %R"'
alias idle='sinfo_t_idle'
alias smi='watch -n 1 nvidia-smi'
alias fsacct_week='sacct --format="JobID%15,JobName%30,Partition%16,AllocCPUS%3,State%12,Elapsed%10,Timelimit%10" --starttime $(date -d "last week" +%Y-%m-%d) | grep -vE "*.batch|*.extern|*.inter|bash|python|CANCELLED|echo"'
alias fsacct_today='sacct --format="JobID%15,JobName%55,Partition%16,AllocCPUS%3,State%12,Elapsed%10,Timelimit%10" --starttime $(date +%Y-%m-%d) | grep -vE "*.batch|*.extern|*.inter|bash"'
alias execute_order_66_gpu_1='salloc --account=hk-project-p0023960 --time=01:00:00 --partition=accelerated --nodes=1 --gres=gpu:1  --cpus-per-task=8 --mem=450G --ntasks-per-node=1'
alias execute_order_66_gpu_4='salloc --account=hk-project-p0023960 --time=01:00:00 --partition=accelerated --nodes=1 --gres=gpu:4  --cpus-per-task=8 --ntasks-per-node=4'
alias execute_order_66_cpuonly='salloc --account=hk-project-p0023960 --time=1:00:00 --partition=dev_cpuonly --nodes=1 --cpus-per-task=128 --ntasks-per-node=1'

export ws_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/'
export HF_HOME="$ws_dir/huggingface/"

alias binger='cd $ws_dir/'
alias dinger='cd /home/hk-project-pai00039/tum_ind3695/projects/jafar'

alias gs='git status'


export PATH=/home/hk-project-pai00039/tum_ind3695/projects/ffmpeg-7.0.2-amd64-static:$PATH            

overlapper() {
    if [ -z "$1" ]; then
        echo "Usage: overlap <jobid>"
    else
        srun --jobid="$1" --pty /usr/bin/bash
    fi
}