[tab] /fast/project/HFMI_ProFOUND/mihir/logs/train/dyn/diffusion/train_dyn_dit_32931.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --mem=100GB
#SBATCH --output=/fast/project/HFMI_ProFOUND/mihir/logs/train/dyn/diffusion/%x_%j.log
#SBATCH --error=/fast/project/HFMI_ProFOUND/mihir/logs/train/dyn/diffusion/%x_%j.log
#SBATCH --job-name=train_dyn_dit

cat $0

source .venv/bin/activate

array_records_dir_train=/fast/project/HFMI_ProFOUND/mihir/data/data_doom/doom_episodes_50m_60x80/train
array_records_dir_val=/fast/project/HFMI_ProFOUND/mihir/data/data_doom/doom_episodes_50m_60x80/val

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/fast/project/HFMI_ProFOUND/mihir/checkpoints/dyn/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

tokenizer_checkpoint=/fast/project/HFMI_ProFOUND/mihir/checkpoints/tokenizer/mae/doom_train_tokenizer_mae_36/32750


srun python jasmine/baselines/diffusion/train_dynamics_diffusion.py \
    --save_ckpt \
    --image_height=60 \
    --image_width=80 \
    --ckpt_dir $CHECKPOINT_DIR \
    --diffusion_denoise_steps=128 \
    --name=train_dyn_dit_doom_$slurm_job_id \
    --tags dyn diffusion doom \
    --entity instant-uv \
    --project jafar \
    --data_dir $array_records_dir_train \
    --val_data_dir $array_records_dir_val \
    --tokenizer_checkpoint=$tokenizer_checkpoint \
/var/spool/slurm/job32931/slurm_script: line 15: .venv/bin/activate: No such file or directory
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.22.1
wandb: Run data is saved locally in /fast/home/mihir.mahajan/Projects/jasmine_runner_1/wandb/run-20251028_003030-122dbczi
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run train_dyn_dit_doom_32931
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/122dbczi
Running on 1 devices.
Counting all components: ['dynamics', 'lam', 'tokenizer']
Parameter counts:
{'dynamics': 40510560, 'lam': 17640416, 'tokenizer': 34457696, 'total': 92608672}
Traceback (most recent call last):
  File "/fast/home/mihir.mahajan/Projects/jasmine_runner_1/jasmine/baselines/diffusion/train_dynamics_diffusion.py", line 777, in <module>
    main(args)
  File "/fast/home/mihir.mahajan/Projects/jasmine_runner_1/jasmine/baselines/diffusion/train_dynamics_diffusion.py", line 434, in main
    train_iterator = build_dataloader(args, args.data_dir)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/jasmine_runner_1/jasmine/baselines/diffusion/train_dynamics_diffusion.py", line 212, in build_dataloader
    for x in os.listdir(data_dir)
             ^^^^^^^^^^^^^^^^^^^^
FileNotFoundError: [Errno 2] No such file or directory: '/fast/project/HFMI_ProFOUND/mihir/data/data_doom/doom_episodes_50m_60x80/train'
wandb: 
wandb: 🚀 View run train_dyn_dit_doom_32931 at: https://wandb.ai/instant-uv/jafar/runs/122dbczi
wandb: Find logs at: ../../../../fast/home/mihir.mahajan/Projects/jasmine_runner_1/wandb/run-20251028_003030-122dbczi/logs
W1028 00:30:34.661639 3616273 pjrt_client.cc:1469] WatchJobStateAsync failed for task goo.gle/debugproto  job_name: "jax_worker": UNAVAILABLE: failed to connect to all addresses; last error: UNKNOWN: ipv4:10.86.2.40:61603: Failed to connect to remote host: Connection refused
Additional GRPC error information from remote target coordination_service while calling /tensorflow.CoordinationService/WatchJobState:
:UNKNOWN:Error received from peer  {grpc_status:14, grpc_message:"failed to connect to all addresses; last error: UNKNOWN: ipv4:10.86.2.40:61603: Failed to connect to remote host: Connection refused"}
srun: error: hai004: task 0: Exited with exit code 1