[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/train_lam_coinrun_311M_3469569.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=2-00:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:1
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/lam/%x_%j.log
#SBATCH --job-name=train_lam_coinrun_311M
#SBATCH --requeue
#SBATCH --signal=b:usr1@300 # 5 min before timeout

# --- signal trap to requeue job before timeout ---
requeue_job() {
    echo "[$(date)] caught sigusr1 (timeout warning), requeueing slurm job $SLURM_JOB_ID..."
    # optional: trigger checkpoint saving here
    # e.g., touch $checkpoint_dir/requeue_trigger
    scontrol requeue $SLURM_JOB_ID
    exit 0
}

trap requeue_job sigusr1

# set checkpoint flag based on restart count
restart_count=$(scontrol show job $SLURM_JOB_ID | grep -o 'Restarts=[0-9]*' | cut -d'=' -f2)

if [ $restart_count -eq 0 ]; then
    restore_ckpt_flag="--no-restore-ckpt"
else
    restore_ckpt_flag="--restore-ckpt"
fi

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/mihir/coinrun/lam/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

array_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train
array_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/val

env | grep SLURM

srun python train_lam.py \
    --save_ckpt \
    $restore_ckpt_flag \
    --wandb_id $SLURM_JOB_ID \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=24 \
    --image_height=64 \
    --image_width=64 \
    --max_lr=3e-6 \
    --log_image_interval=1000 \
    --log_checkpoint_interval=1000 \
    --log \
    --name="${job_name}_${slurm_job_id}" \
    --tags "coinrun lam_ablation modelsize" \
    --entity instant-uv \
    --project jafar \
    --lr_schedule wsd \
    --model_dim=896 \
    --num_blocks=12 \
    --num_heads=14 \
    --latent_dim=48 \
    --ffn_dim=3584 \
    --val_data_dir $array_records_dir_val \
    --data_dir $array_records_dir &

child_pid=$!

wait $child_pid
/var/spool/slurmd/job3469569/slurm_script: line 40: .venv/bin/activate: No such file or directory
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=1
SLURM_JOB_UID=999226
SLURM_TASK_PID=1810805
SLURM_JOB_GPUS=2
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs
SLURMD_NODENAME=hkn0729
SLURM_JOB_START_TIME=1757104195
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1757276995
SLURM_CPUS_ON_NODE=6
SLURM_JOB_CPUS_PER_NODE=6
SLURM_GPUS_ON_NODE=1
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=1
SLURM_JOBID=3469569
SLURM_JOB_QOS=normal
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=1
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e9.hkn0729
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_NODELIST=hkn0729
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=1
SLURM_NNODES=1
SLURM_SUBMIT_HOST=hkn1991.localdomain
SLURM_JOB_ID=3469569
SLURM_NODEID=0
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_lam_coinrun_311M
SLURM_NTASKS_PER_NODE=1
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn0729
GpuFreq=control_disabled
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: creating run
wandb: Tracking run with wandb version 0.21.3
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/wandb/run-20250905_223410-3469569
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run train_lam_coinrun_311M_3469569
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/3469569
Running on 1 devices.
Counting all components: ['action_in', 'action_up', 'decoder', 'encoder', 'patch_up', 'vq']
Parameter counts:
{'action_in': 768, 'action_up': 43904, 'decoder': 155841152, 'encoder': 155080368, 'patch_up': 689024, 'vq': 288, 'total': 311655504}
Starting training from step 0...
2025-09-05 22:34:35.724233: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-09-05 22:34:35.725528: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
Step 0, loss: 0.32670068740844727
Step 1, loss: 0.3228756785392761
Step 2, loss: 0.30648404359817505
Step 3, loss: 0.31563735008239746
Step 4, loss: 0.31263256072998047
Step 5, loss: 0.3193112313747406
Step 6, loss: 0.3218037486076355
Step 7, loss: 0.31278547644615173
Step 8, loss: 0.3285486698150635
Step 9, loss: 0.3155764937400818
Step 10, loss: 0.3073958456516266
Step 11, loss: 0.31856849789619446
Step 12, loss: 0.32533809542655945
Step 13, loss: 0.3097471594810486
Step 14, loss: 0.3183121681213379
Step 15, loss: 0.3102788031101227
Step 16, loss: 0.32966354489326477
Step 17, loss: 0.31318211555480957
Step 18, loss: 0.32258638739585876
Step 19, loss: 0.3312549293041229
Step 20, loss: 0.3190544545650482
Step 21, loss: 0.31717970967292786
Step 22, loss: 0.31445828080177307
Step 23, loss: 0.30292263627052307
Step 24, loss: 0.3158635199069977
Step 25, loss: 0.30581068992614746
Step 26, loss: 0.31896838545799255
Step 27, loss: 0.3197079300880432
Step 28, loss: 0.3125552833080292
Step 29, loss: 0.31916916370391846
Step 30, loss: 0.30208319425582886
Step 31, loss: 0.32716408371925354
Step 32, loss: 0.3113912045955658
Step 33, loss: 0.31773051619529724
Step 34, loss: 0.31623485684394836
Step 35, loss: 0.3208811283111572
Step 36, loss: 0.30788886547088623
Step 37, loss: 0.31254085898399353
Step 38, loss: 0.3172925114631653
Step 39, loss: 0.32344064116477966
Step 40, loss: 0.319547176361084
Step 41, loss: 0.3224325180053711
Step 42, loss: 0.312593936920166
Step 43, loss: 0.3126896619796753
Step 44, loss: 0.3141651749610901
Step 45, loss: 0.3250216841697693
Step 46, loss: 0.31160393357276917
Step 47, loss: 0.313493013381958
Step 48, loss: 0.31896457076072693
Step 49, loss: 0.31110015511512756
Step 50, loss: 0.3126383423805237
Step 51, loss: 0.30285364389419556
Step 52, loss: 0.31733959913253784
Step 53, loss: 0.30982792377471924
Step 54, loss: 0.323069304227829
Step 55, loss: 0.31710734963417053
Step 56, loss: 0.3167848587036133
Step 57, loss: 0.3216778337955475
Step 58, loss: 0.31999433040618896
Step 59, loss: 0.30398330092430115
Step 60, loss: 0.32730430364608765
Step 61, loss: 0.3017076551914215
Step 62, loss: 0.29655691981315613
Step 63, loss: 0.3123442530632019
Step 64, loss: 0.3231739103794098
Step 65, loss: 0.317227303981781
Step 66, loss: 0.3153745234012604
Step 67, loss: 0.30899426341056824
Step 68, loss: 0.3081183135509491
Step 69, loss: 0.3052864670753479
Step 70, loss: 0.3287593126296997
Step 71, loss: 0.3133338987827301
Step 72, loss: 0.303838849067688
Step 73, loss: 0.30702194571495056
Step 74, loss: 0.31240546703338623
Step 75, loss: 0.3141060769557953
Step 76, loss: 0.31803300976753235
Step 77, loss: 0.3125794529914856
Step 78, loss: 0.3034350574016571
Step 79, loss: 0.3142947256565094
Step 80, loss: 0.31903520226478577
Step 81, loss: 0.2951797842979431
Step 82, loss: 0.30238768458366394
Step 83, loss: 0.3132435381412506
Step 84, loss: 0.301560640335083
Step 85, loss: 0.308278888463974
Step 86, loss: 0.3150296211242676
Step 87, loss: 0.3059684634208679
Step 88, loss: 0.3029228448867798
Step 89, loss: 0.3038482069969177
Step 90, loss: 0.3083123564720154
Step 91, loss: 0.3066179156303406
Step 92, loss: 0.31548452377319336
Step 93, loss: 0.3054041564464569
Step 94, loss: 0.3039185702800751
Step 95, loss: 0.30549612641334534
Step 96, loss: 0.30805495381355286
Step 97, loss: 0.30435970425605774
Step 98, loss: 0.3041527271270752
Step 99, loss: 0.3102133274078369
Step 100, loss: 0.30485549569129944
Step 101, loss: 0.31429123878479004
Step 102, loss: 0.30076900124549866
Step 103, loss: 0.30202779173851013
Step 104, loss: 0.2946684658527374
Step 105, loss: 0.30218788981437683
Step 106, loss: 0.30854514241218567
Step 107, loss: 0.30455154180526733
Step 108, loss: 0.30714699625968933
Step 109, loss: 0.2986069917678833
Step 110, loss: 0.3004385828971863
Step 111, loss: 0.29880836606025696
Step 112, loss: 0.3037300407886505
Step 113, loss: 0.3036937415599823
Step 114, loss: 0.3110331892967224
Step 115, loss: 0.2991844713687897
Step 116, loss: 0.2988225817680359
Step 117, loss: 0.3057701289653778
Step 118, loss: 0.28649982810020447
Step 119, loss: 0.30023378133773804
Step 120, loss: 0.29765522480010986
Step 121, loss: 0.2933956980705261
Step 122, loss: 0.29907849431037903
Step 123, loss: 0.2925976812839508
Step 124, loss: 0.30064457654953003
Step 125, loss: 0.2866140305995941
Step 126, loss: 0.2927999496459961
Step 127, loss: 0.3033636510372162
Step 128, loss: 0.2963865101337433
Step 129, loss: 0.2819899320602417
Step 130, loss: 0.2967420220375061
Step 131, loss: 0.291944295167923
Step 132, loss: 0.2916223406791687
Step 133, loss: 0.30107709765434265
Step 134, loss: 0.287319153547287
Step 135, loss: 0.29340046644210815
Step 136, loss: 0.30247968435287476
Step 137, loss: 0.28980252146720886
Step 138, loss: 0.2904524505138397
Step 139, loss: 0.27949678897857666
Step 140, loss: 0.2963244318962097
Step 141, loss: 0.2978780269622803
Step 142, loss: 0.28218576312065125
Step 143, loss: 0.2948848307132721
Step 144, loss: 0.2825896739959717
Step 145, loss: 0.2882777750492096
Step 146, loss: 0.29133933782577515
Step 147, loss: 0.2841861844062805
Step 148, loss: 0.2764188349246979
Step 149, loss: 0.2844504117965698
Step 150, loss: 0.3015214204788208
Step 151, loss: 0.29093649983406067
Step 152, loss: 0.2761401832103729
Step 153, loss: 0.30154940485954285
Step 154, loss: 0.3009883165359497
Step 155, loss: 0.2932864725589752
Step 156, loss: 0.2817378044128418
Step 157, loss: 0.2874589264392853
Step 158, loss: 0.2813238799571991
Step 159, loss: 0.287861168384552
Step 160, loss: 0.2766549289226532
Step 161, loss: 0.27095741033554077
Step 162, loss: 0.27655458450317383
Step 163, loss: 0.2799340486526489
Step 164, loss: 0.2690490186214447
Step 165, loss: 0.28489950299263
Step 166, loss: 0.27373117208480835
Step 167, loss: 0.2920382618904114
Step 168, loss: 0.28401583433151245
Step 169, loss: 0.2693176865577698
Step 170, loss: 0.2884095311164856
Step 171, loss: 0.272987425327301
Step 172, loss: 0.2679787576198578
Step 173, loss: 0.2734033465385437
Step 174, loss: 0.2726956903934479
Step 175, loss: 0.2687923312187195
Step 176, loss: 0.2865637540817261
Step 177, loss: 0.2780385911464691
Step 178, loss: 0.26178669929504395
Step 179, loss: 0.2797602415084839
Step 180, loss: 0.2675800323486328
Step 181, loss: 0.2812725305557251
Step 182, loss: 0.26849111914634705
Step 183, loss: 0.26423898339271545
Step 184, loss: 0.27565035223960876
Step 185, loss: 0.27826595306396484
Step 186, loss: 0.26448512077331543
Step 187, loss: 0.2822693884372711
Step 188, loss: 0.2683868110179901
Step 189, loss: 0.28559553623199463
Step 190, loss: 0.26472073793411255
Step 191, loss: 0.27913129329681396
Step 192, loss: 0.2664516866207123
Step 193, loss: 0.2643465995788574
Step 194, loss: 0.2573527991771698
Step 195, loss: 0.2718404233455658
Step 196, loss: 0.2705581784248352
Step 197, loss: 0.2716696262359619
Step 198, loss: 0.26432889699935913
Step 199, loss: 0.2671912908554077
Step 200, loss: 0.2546460032463074
Step 201, loss: 0.26619911193847656
Step 202, loss: 0.24569392204284668
Step 203, loss: 0.26645198464393616
Step 204, loss: 0.26042166352272034
Step 205, loss: 0.25757142901420593
Step 206, loss: 0.2628953754901886
Step 207, loss: 0.27754756808280945
Step 208, loss: 0.27348777651786804
Step 209, loss: 0.258472740650177
Step 210, loss: 0.2582595646381378
Step 211, loss: 0.2612609565258026
Step 212, loss: 0.253089040517807
Step 213, loss: 0.25047042965888977
Step 214, loss: 0.2630004286766052
Step 215, loss: 0.2651938796043396
Step 216, loss: 0.24314671754837036
Step 217, loss: 0.25238633155822754
Step 218, loss: 0.2568165063858032
Step 219, loss: 0.2562321424484253
Step 220, loss: 0.24474281072616577
Step 221, loss: 0.2735799252986908
Step 222, loss: 0.25888368487358093
Step 223, loss: 0.25260937213897705
Step 224, loss: 0.2620387077331543
Step 225, loss: 0.25532102584838867
Step 226, loss: 0.25093603134155273
Step 227, loss: 0.24302887916564941
Step 228, loss: 0.2526725232601166
Step 229, loss: 0.2605861723423004
Step 230, loss: 0.2454603761434555
Step 231, loss: 0.2491515725851059
Step 232, loss: 0.24311965703964233
Step 233, loss: 0.25126779079437256
Step 234, loss: 0.24701818823814392
Step 235, loss: 0.238130584359169
Step 236, loss: 0.237676203250885
Step 237, loss: 0.2440456748008728
Step 238, loss: 0.2432679384946823
Step 239, loss: 0.23657268285751343
Step 240, loss: 0.25140389800071716
Step 241, loss: 0.2698136270046234
Step 242, loss: 0.24991227686405182
Step 243, loss: 0.23837321996688843
Step 244, loss: 0.24084030091762543
Step 245, loss: 0.24020417034626007
Step 246, loss: 0.24412108957767487
Step 247, loss: 0.2478320449590683
Step 248, loss: 0.24218617379665375
Step 249, loss: 0.2365211397409439
Step 250, loss: 0.22476673126220703
Step 251, loss: 0.2402079850435257
Step 252, loss: 0.23486708104610443
Step 253, loss: 0.2356840819120407
Step 254, loss: 0.24330373108386993
Step 255, loss: 0.24493975937366486
Step 256, loss: 0.23736388981342316
Step 257, loss: 0.2284037172794342
Step 258, loss: 0.22898347675800323
Step 259, loss: 0.2290492206811905
Step 260, loss: 0.23123012483119965
Step 261, loss: 0.24172040820121765
Step 262, loss: 0.23622317612171173
Step 263, loss: 0.25332581996917725
Step 264, loss: 0.24660693109035492
Step 265, loss: 0.23799890279769897
Step 266, loss: 0.23667289316654205
Step 267, loss: 0.22990407049655914
Step 268, loss: 0.23478823900222778
Step 269, loss: 0.22096313536167145
Step 270, loss: 0.239072784781456
Step 271, loss: 0.2272433489561081
Step 272, loss: 0.22379888594150543
Step 273, loss: 0.2392652928829193
Step 274, loss: 0.22771723568439484
Step 275, loss: 0.23688723146915436
Step 276, loss: 0.23435154557228088
Step 277, loss: 0.22166632115840912
Step 278, loss: 0.22263823449611664
Step 279, loss: 0.2335406243801117
Step 280, loss: 0.2471243441104889
Step 281, loss: 0.23125340044498444
Step 282, loss: 0.22488494217395782
Step 283, loss: 0.23454345762729645
Step 284, loss: 0.2281060367822647
Step 285, loss: 0.23506440222263336
Step 286, loss: 0.2282322347164154
Step 287, loss: 0.21082821488380432
Step 288, loss: 0.21880458295345306
Step 289, loss: 0.22249245643615723
Step 290, loss: 0.21938663721084595
Step 291, loss: 0.2269812524318695
Step 292, loss: 0.22862949967384338
Step 293, loss: 0.21480806171894073
Step 294, loss: 0.21888825297355652
Step 295, loss: 0.23148277401924133
Step 296, loss: 0.21830207109451294
Step 297, loss: 0.22938120365142822
Step 298, loss: 0.21606995165348053
Step 299, loss: 0.2271086424589157
Step 300, loss: 0.2257154881954193
Step 301, loss: 0.21104273200035095
Step 302, loss: 0.22756081819534302
Step 303, loss: 0.21932125091552734
Step 304, loss: 0.21070994436740875
Step 305, loss: 0.2149253636598587
Step 306, loss: 0.22370828688144684
Step 307, loss: 0.21656553447246552
Step 308, loss: 0.20614753663539886
Step 309, loss: 0.21314524114131927
Step 310, loss: 0.2204509824514389
Step 311, loss: 0.2015097290277481
Step 312, loss: 0.23286345601081848
Step 313, loss: 0.2188306748867035
Step 314, loss: 0.2326751947402954
Step 315, loss: 0.2071838080883026
Step 316, loss: 0.23261015117168427
Step 317, loss: 0.22815413773059845
Step 318, loss: 0.20368419587612152
Step 319, loss: 0.20782515406608582
Step 320, loss: 0.2044333517551422
Step 321, loss: 0.20802314579486847
Step 322, loss: 0.20440468192100525
Step 323, loss: 0.2129337191581726
Step 324, loss: 0.2095731496810913
Step 325, loss: 0.228922501206398
Step 326, loss: 0.21261431276798248
Step 327, loss: 0.21287697553634644
Step 328, loss: 0.21538157761096954
Step 329, loss: 0.21878191828727722
Step 330, loss: 0.22037620842456818
Step 331, loss: 0.20773941278457642
Step 332, loss: 0.23064997792243958
Step 333, loss: 0.20720407366752625
Step 334, loss: 0.20441503822803497
Step 335, loss: 0.2060972899198532
Step 336, loss: 0.2080194056034088
Step 337, loss: 0.2105853259563446
Step 338, loss: 0.21978075802326202
Step 339, loss: 0.2006307691335678
Step 340, loss: 0.21199773252010345
Step 341, loss: 0.21112029254436493
Step 342, loss: 0.2135261446237564
Step 343, loss: 0.21250925958156586
Step 344, loss: 0.20047903060913086
Step 345, loss: 0.20581355690956116
Step 346, loss: 0.19387589395046234
Step 347, loss: 0.21998614072799683
Step 348, loss: 0.2048615962266922
Step 349, loss: 0.2202419936656952
Step 350, loss: 0.21770228445529938
Step 351, loss: 0.21040920913219452
Step 352, loss: 0.19186832010746002
Step 353, loss: 0.2037704735994339
Step 354, loss: 0.21062155067920685
Step 355, loss: 0.21234062314033508
Step 356, loss: 0.20150254666805267
Step 357, loss: 0.19836518168449402
Step 358, loss: 0.20317357778549194
Step 359, loss: 0.21396100521087646
Step 360, loss: 0.1969567984342575
Step 361, loss: 0.19251909852027893
Step 362, loss: 0.19323450326919556
Step 363, loss: 0.20286889374256134
Step 364, loss: 0.2111835479736328
Step 365, loss: 0.20162421464920044
Step 366, loss: 0.20648787915706635
Step 367, loss: 0.20664988458156586
Step 368, loss: 0.19660057127475739
Step 369, loss: 0.19763965904712677
Step 370, loss: 0.20880377292633057
Step 371, loss: 0.2000325322151184
Step 372, loss: 0.20846863090991974
Step 373, loss: 0.19173415005207062
Step 374, loss: 0.20416457951068878
Step 375, loss: 0.1992243528366089
Step 376, loss: 0.1912972629070282
Step 377, loss: 0.18899005651474
Step 378, loss: 0.1899210810661316
Step 379, loss: 0.18376410007476807
Step 380, loss: 0.19180577993392944
Step 381, loss: 0.20443682372570038
Step 382, loss: 0.19527588784694672
Step 383, loss: 0.1958610713481903
Step 384, loss: 0.20258666574954987
Step 385, loss: 0.21475450694561005
Step 386, loss: 0.1943417638540268
Step 387, loss: 0.18732081353664398
Step 388, loss: 0.2046518325805664
Step 389, loss: 0.18458230793476105
Step 390, loss: 0.20499855279922485
Step 391, loss: 0.18604519963264465
Step 392, loss: 0.18526548147201538
Step 393, loss: 0.1953166127204895
Step 394, loss: 0.18036390841007233
Step 395, loss: 0.20992445945739746
Step 396, loss: 0.1954040825366974
Step 397, loss: 0.1919981986284256
Step 398, loss: 0.18635490536689758
Step 399, loss: 0.18054939806461334
Step 400, loss: 0.2149868607521057
Step 401, loss: 0.18286970257759094
Step 402, loss: 0.20246313512325287
Step 403, loss: 0.1921776682138443
Step 404, loss: 0.19730931520462036
Step 405, loss: 0.18826575577259064
Step 406, loss: 0.2063121795654297
Step 407, loss: 0.19595804810523987
Step 408, loss: 0.18491500616073608
Step 409, loss: 0.20304246246814728
Step 410, loss: 0.18718010187149048
Step 411, loss: 0.20099672675132751
Step 412, loss: 0.1810406595468521
Step 413, loss: 0.18184152245521545
Step 414, loss: 0.18948443233966827
Step 415, loss: 0.19590061902999878
Step 416, loss: 0.1864279955625534
Step 417, loss: 0.19372746348381042
Step 418, loss: 0.19026023149490356
Step 419, loss: 0.20213495194911957
Step 420, loss: 0.18851925432682037
Step 421, loss: 0.19247500598430634
Step 422, loss: 0.17617785930633545
Step 423, loss: 0.18701443076133728
Step 424, loss: 0.19657638669013977
Step 425, loss: 0.19105301797389984
Step 426, loss: 0.1875758320093155
Step 427, loss: 0.17497578263282776
Step 428, loss: 0.17246121168136597
Step 429, loss: 0.18725895881652832
Step 430, loss: 0.19588768482208252
Step 431, loss: 0.17520764470100403
Step 432, loss: 0.17805221676826477
Step 433, loss: 0.1806318759918213
Step 434, loss: 0.19859139621257782
Step 435, loss: 0.17744287848472595
Step 436, loss: 0.17955563962459564
Step 437, loss: 0.18990911543369293
Step 438, loss: 0.17663979530334473
Step 439, loss: 0.17919078469276428
Step 440, loss: 0.18656392395496368
Step 441, loss: 0.17515042424201965
Step 442, loss: 0.17883522808551788
Step 443, loss: 0.1795118749141693
Step 444, loss: 0.1858094483613968
Step 445, loss: 0.185130313038826
Step 446, loss: 0.1834694892168045
Step 447, loss: 0.18378876149654388
Step 448, loss: 0.19133582711219788
Step 449, loss: 0.18580874800682068
Step 450, loss: 0.18512387573719025
Step 451, loss: 0.17702800035476685
Step 452, loss: 0.18347252905368805
Step 453, loss: 0.18237482011318207
Step 454, loss: 0.2025652974843979
Step 455, loss: 0.1869395524263382
Step 456, loss: 0.18317709863185883
Step 457, loss: 0.16225655376911163
Step 458, loss: 0.18912789225578308
Step 459, loss: 0.16795752942562103
Step 460, loss: 0.17059314250946045
Step 461, loss: 0.18770727515220642
Step 462, loss: 0.1796184629201889
Step 463, loss: 0.16848596930503845
Step 464, loss: 0.1785201132297516
Step 465, loss: 0.18194423615932465
Step 466, loss: 0.17280732095241547
Step 467, loss: 0.17404136061668396
Step 468, loss: 0.16946744918823242
Step 469, loss: 0.17461803555488586
Step 470, loss: 0.1683589369058609
Step 471, loss: 0.17864489555358887
Step 472, loss: 0.16948556900024414
Step 473, loss: 0.17252053320407867
Step 474, loss: 0.1604686826467514
Step 475, loss: 0.1795974224805832
Step 476, loss: 0.1750379353761673
Step 477, loss: 0.18138891458511353
Step 478, loss: 0.17055998742580414
Step 479, loss: 0.1828087866306305
Step 480, loss: 0.16300563514232635
Step 481, loss: 0.1701754629611969
Step 482, loss: 0.18532662093639374
Step 483, loss: 0.17192967236042023
Step 484, loss: 0.17124198377132416
Step 485, loss: 0.17479649186134338
Step 486, loss: 0.170439213514328
Step 487, loss: 0.17095831036567688
Step 488, loss: 0.16867448389530182
Step 489, loss: 0.15807010233402252
Step 490, loss: 0.17137901484966278
Step 491, loss: 0.17856192588806152
Step 492, loss: 0.18580619990825653
Step 493, loss: 0.1733253300189972
Step 494, loss: 0.154290109872818
Step 495, loss: 0.19775621592998505
Step 496, loss: 0.16369877755641937
Step 497, loss: 0.1778690069913864
Step 498, loss: 0.17637021839618683
Step 499, loss: 0.18393085896968842
Step 500, loss: 0.18458597362041473
Step 501, loss: 0.17756971716880798
Step 502, loss: 0.1706598401069641
Step 503, loss: 0.16400325298309326
Step 504, loss: 0.1672447919845581
Step 505, loss: 0.16240426898002625
Step 506, loss: 0.16706815361976624
Step 507, loss: 0.1590700000524521
Step 508, loss: 0.15771426260471344
Step 509, loss: 0.16245245933532715
Step 510, loss: 0.16479183733463287
Step 511, loss: 0.16443252563476562
Step 512, loss: 0.1684126853942871
Step 513, loss: 0.1589934527873993
Step 514, loss: 0.1659197062253952
Step 515, loss: 0.17597174644470215
Step 516, loss: 0.1641179919242859
Step 517, loss: 0.16144892573356628
Step 518, loss: 0.15814563632011414
Step 519, loss: 0.1676492542028427
Step 520, loss: 0.17011266946792603
Step 521, loss: 0.1588173657655716
Step 522, loss: 0.16074715554714203
Step 523, loss: 0.15585818886756897
Step 524, loss: 0.16754458844661713
Step 525, loss: 0.16731105744838715
Step 526, loss: 0.14795486629009247
Step 527, loss: 0.17333023250102997
Step 528, loss: 0.16228698194026947
Step 529, loss: 0.16809794306755066
Step 530, loss: 0.16663968563079834
Step 531, loss: 0.166520357131958
Step 532, loss: 0.15445561707019806
Step 533, loss: 0.16671240329742432
Step 534, loss: 0.163197860121727
Step 535, loss: 0.16401763260364532
Step 536, loss: 0.15650203824043274
Step 537, loss: 0.15332654118537903
Step 538, loss: 0.15485499799251556
Step 539, loss: 0.1647006869316101
Step 540, loss: 0.1615280658006668
Step 541, loss: 0.1539774239063263
Step 542, loss: 0.1469665914773941
Step 543, loss: 0.16052976250648499
Step 544, loss: 0.1469196081161499
Step 545, loss: 0.13795733451843262
Step 546, loss: 0.16244752705097198
Step 547, loss: 0.16175831854343414
Step 548, loss: 0.15162412822246552
Step 549, loss: 0.1604224145412445
Step 550, loss: 0.15166215598583221
Step 551, loss: 0.1679271161556244
Step 552, loss: 0.15525811910629272
Step 553, loss: 0.1449344903230667
Step 554, loss: 0.1623028665781021
Step 555, loss: 0.14766810834407806
Step 556, loss: 0.15308725833892822
Step 557, loss: 0.16635051369667053
Step 558, loss: 0.1460389643907547
Step 559, loss: 0.14736074209213257
Step 560, loss: 0.17026668787002563
Step 561, loss: 0.15264534950256348
Step 562, loss: 0.15372909605503082
Step 563, loss: 0.15454962849617004
Step 564, loss: 0.16662302613258362
Step 565, loss: 0.16087806224822998
Step 566, loss: 0.1647089570760727
Step 567, loss: 0.14671236276626587
Step 568, loss: 0.15271322429180145
Step 569, loss: 0.15063823759555817
Step 570, loss: 0.1450859159231186
Step 571, loss: 0.14377903938293457
Step 572, loss: 0.1442921757698059
Step 573, loss: 0.14989376068115234
Step 574, loss: 0.14476725459098816
Step 575, loss: 0.1496201902627945
Step 576, loss: 0.15061773359775543
Step 577, loss: 0.14703772962093353
Step 578, loss: 0.15735933184623718
Step 579, loss: 0.1399516612291336
Step 580, loss: 0.15842531621456146
Step 581, loss: 0.15423400700092316
Step 582, loss: 0.16235484182834625
Step 583, loss: 0.1622205227613449
Step 584, loss: 0.1542559117078781
Step 585, loss: 0.14118140935897827
Step 586, loss: 0.13962240517139435
Step 587, loss: 0.14828960597515106
Step 588, loss: 0.15096931159496307
Step 589, loss: 0.15816150605678558
Step 590, loss: 0.1465573012828827
Step 591, loss: 0.15597453713417053
Step 592, loss: 0.14010082185268402
Step 593, loss: 0.14112845063209534
Step 594, loss: 0.13825038075447083
Step 595, loss: 0.14013102650642395
Step 596, loss: 0.14642807841300964
Step 597, loss: 0.14606882631778717
Step 598, loss: 0.14983971416950226
Step 599, loss: 0.14821529388427734
Step 600, loss: 0.13567803800106049
Step 601, loss: 0.1461533010005951
Step 602, loss: 0.14493730664253235
Step 603, loss: 0.13547742366790771
Step 604, loss: 0.14097489416599274
Step 605, loss: 0.12082186341285706
Step 606, loss: 0.12985292077064514
Step 607, loss: 0.1411941796541214
Step 608, loss: 0.13241830468177795
Step 609, loss: 0.1457589864730835
Step 610, loss: 0.14153125882148743
Step 611, loss: 0.13635534048080444
Step 612, loss: 0.14519572257995605
Step 613, loss: 0.13944000005722046
Step 614, loss: 0.1268577128648758
Step 615, loss: 0.14653265476226807
Step 616, loss: 0.13592612743377686
Step 617, loss: 0.15010732412338257
Step 618, loss: 0.13507229089736938
Step 619, loss: 0.16583743691444397
Step 620, loss: 0.13568015396595
Step 621, loss: 0.14552512764930725
Step 622, loss: 0.1326853334903717
Step 623, loss: 0.13952861726284027
Step 624, loss: 0.14392122626304626
Step 625, loss: 0.14059510827064514
Step 626, loss: 0.12331005185842514
Step 627, loss: 0.1378980576992035
Step 628, loss: 0.14141395688056946
Step 629, loss: 0.13303832709789276
Step 630, loss: 0.1312895268201828
Step 631, loss: 0.13734431564807892
Step 632, loss: 0.12853211164474487
Step 633, loss: 0.13736139237880707
Step 634, loss: 0.1414404809474945
Step 635, loss: 0.13009203970432281
Step 636, loss: 0.1283910572528839
Step 637, loss: 0.12979847192764282
Step 638, loss: 0.13683289289474487
Step 639, loss: 0.13759808242321014
Step 640, loss: 0.1240217387676239
Step 641, loss: 0.12856023013591766
Step 642, loss: 0.13105811178684235
Step 643, loss: 0.1408415585756302
Step 644, loss: 0.12534308433532715
Step 645, loss: 0.1308641880750656
Step 646, loss: 0.1438308209180832
Step 647, loss: 0.12023308873176575
Step 648, loss: 0.13485988974571228
Step 649, loss: 0.12024299055337906
Step 650, loss: 0.13430900871753693
Step 651, loss: 0.1325358748435974
Step 652, loss: 0.1280888170003891
Step 653, loss: 0.1394578218460083
Step 654, loss: 0.13560831546783447
Step 655, loss: 0.1331646591424942
Step 656, loss: 0.12449931353330612
Step 657, loss: 0.1180102750658989
Step 658, loss: 0.12185180932283401
Step 659, loss: 0.132879376411438
Step 660, loss: 0.11253056675195694
Step 661, loss: 0.1321847140789032
Step 662, loss: 0.13573090732097626
Step 663, loss: 0.1190265640616417
Step 664, loss: 0.1377076357603073
Step 665, loss: 0.14531099796295166
Step 666, loss: 0.12540745735168457
Step 667, loss: 0.11568422615528107
Step 668, loss: 0.13449546694755554
Step 669, loss: 0.11809086054563522
Step 670, loss: 0.13356786966323853
Step 671, loss: 0.12594865262508392
Step 672, loss: 0.12419725954532623
Step 673, loss: 0.13017435371875763
Step 674, loss: 0.1164168193936348
Step 675, loss: 0.12107794731855392
Step 676, loss: 0.12250156700611115
Step 677, loss: 0.1288304626941681
Step 678, loss: 0.11841060221195221
Step 679, loss: 0.12229615449905396
Step 680, loss: 0.12391740083694458
Step 681, loss: 0.1287333369255066
Step 682, loss: 0.1331411749124527
Step 683, loss: 0.1291358768939972
Step 684, loss: 0.12653714418411255
Step 685, loss: 0.1274345964193344
Step 686, loss: 0.12998853623867035
Step 687, loss: 0.1375662237405777
Step 688, loss: 0.12892384827136993
Step 689, loss: 0.12216907739639282
Step 690, loss: 0.129177063703537
Step 691, loss: 0.11670426279306412
Step 692, loss: 0.12254981696605682
Step 693, loss: 0.1401817798614502
Step 694, loss: 0.11784442514181137
Step 695, loss: 0.12857796251773834
Step 696, loss: 0.12656690180301666
Step 697, loss: 0.12039724737405777
Step 698, loss: 0.11538124829530716
Step 699, loss: 0.12679149210453033
Step 700, loss: 0.11436452716588974
Step 701, loss: 0.1152573749423027
Step 702, loss: 0.12185928225517273
Step 703, loss: 0.11087769269943237
Step 704, loss: 0.11919064819812775
Step 705, loss: 0.11994696408510208
Step 706, loss: 0.11460404843091965
Step 707, loss: 0.1217706948518753
Step 708, loss: 0.1230340525507927
Step 709, loss: 0.11855746805667877
Step 710, loss: 0.11308516561985016
Step 711, loss: 0.1160062700510025
Step 712, loss: 0.11633268743753433
Step 713, loss: 0.12277749180793762
Step 714, loss: 0.11866027116775513
Step 715, loss: 0.10843092948198318
Step 716, loss: 0.12014388293027878
Step 717, loss: 0.11597073078155518
Step 718, loss: 0.11100265383720398
Step 719, loss: 0.114290252327919
Step 720, loss: 0.11378930509090424
Step 721, loss: 0.1184566468000412
Step 722, loss: 0.11124507337808609
Step 723, loss: 0.11602503806352615
Step 724, loss: 0.10488992929458618
Step 725, loss: 0.11717075854539871
Step 726, loss: 0.10992296040058136
Step 727, loss: 0.11158798635005951
Step 728, loss: 0.12337098270654678
Step 729, loss: 0.11695913970470428
Step 730, loss: 0.11190351843833923
Step 731, loss: 0.10596224665641785
Step 732, loss: 0.10450296849012375
Step 733, loss: 0.11720841377973557
Step 734, loss: 0.11383353918790817
Step 735, loss: 0.10908037424087524
Step 736, loss: 0.10804563760757446
Step 737, loss: 0.11010054498910904
Step 738, loss: 0.11928626149892807
Step 739, loss: 0.12162226438522339
Step 740, loss: 0.10604893416166306
Step 741, loss: 0.11713293939828873
Step 742, loss: 0.10664701461791992
Step 743, loss: 0.11478489637374878
Step 744, loss: 0.10862031579017639
Step 745, loss: 0.11689151078462601
Step 746, loss: 0.11187475174665451
Step 747, loss: 0.11440619081258774
Step 748, loss: 0.1185237243771553
Step 749, loss: 0.09798216074705124
Step 750, loss: 0.09777869284152985
Step 751, loss: 0.11431670933961868
Step 752, loss: 0.108920156955719
Step 753, loss: 0.10862236469984055
Step 754, loss: 0.10661527514457703
Step 755, loss: 0.08923164755105972
Step 756, loss: 0.10476091504096985
Step 757, loss: 0.11053522676229477
Step 758, loss: 0.10478796064853668
Step 759, loss: 0.09750369936227798
Step 760, loss: 0.11086364835500717
Step 761, loss: 0.10581715404987335
Step 762, loss: 0.10510163009166718
Step 763, loss: 0.12012892961502075
Step 764, loss: 0.10762418806552887
Step 765, loss: 0.10797873884439468
Step 766, loss: 0.10080129653215408
Step 767, loss: 0.10539045929908752
Step 768, loss: 0.10508052259683609
Step 769, loss: 0.11244411766529083
Step 770, loss: 0.10409258306026459
Step 771, loss: 0.09774546325206757
Step 772, loss: 0.09939653426408768
Step 773, loss: 0.10476257652044296
Step 774, loss: 0.11139380931854248
Step 775, loss: 0.10150767117738724
Step 776, loss: 0.10181727260351181
Step 777, loss: 0.09250614792108536
Step 778, loss: 0.10011837631464005
Step 779, loss: 0.09270156919956207
Step 780, loss: 0.09608977288007736
Step 781, loss: 0.10491417348384857
Step 782, loss: 0.10268176347017288
Step 783, loss: 0.09409435093402863
Step 784, loss: 0.09706410765647888
Step 785, loss: 0.10806555300951004
Step 786, loss: 0.10648339986801147
Step 787, loss: 0.10174886882305145
Step 788, loss: 0.10245326906442642
Step 789, loss: 0.11041299998760223
Step 790, loss: 0.09822877496480942
Step 791, loss: 0.10524068772792816
Step 792, loss: 0.10658266395330429
Step 793, loss: 0.10114723443984985
Step 794, loss: 0.08827095478773117
Step 795, loss: 0.10126037895679474
Step 796, loss: 0.09913986176252365
Step 797, loss: 0.10174839198589325
Step 798, loss: 0.09956628829240799
Step 799, loss: 0.10379683971405029
Step 800, loss: 0.1008194237947464
Step 801, loss: 0.09997855126857758
Step 802, loss: 0.1053793728351593
Step 803, loss: 0.11114832758903503
Step 804, loss: 0.09779958426952362
Step 805, loss: 0.09838789701461792
Step 806, loss: 0.09474051743745804
Step 807, loss: 0.10701855272054672
Step 808, loss: 0.09214746206998825
Step 809, loss: 0.10147971659898758
Step 810, loss: 0.09995065629482269
Step 811, loss: 0.09729427099227905
Step 812, loss: 0.10052237659692764
Step 813, loss: 0.09948166459798813
Step 814, loss: 0.0977642610669136
Step 815, loss: 0.09753444790840149
Step 816, loss: 0.09836119413375854
Step 817, loss: 0.09560737013816833
Step 818, loss: 0.10264724493026733
Step 819, loss: 0.09799481928348541
Step 820, loss: 0.1086382195353508
Step 821, loss: 0.10146200656890869
Step 822, loss: 0.0941559225320816
Step 823, loss: 0.10024306923151016
Step 824, loss: 0.10120076686143875
Step 825, loss: 0.10122958570718765
Step 826, loss: 0.09710301458835602
Step 827, loss: 0.09718598425388336
Step 828, loss: 0.09759515523910522
Step 829, loss: 0.08956622332334518
Step 830, loss: 0.09351219981908798
Step 831, loss: 0.08972302824258804
Step 832, loss: 0.10099417716264725
Step 833, loss: 0.09009118378162384
Step 834, loss: 0.09879595041275024
Step 835, loss: 0.08425207436084747
Step 836, loss: 0.09416583925485611
Step 837, loss: 0.09833770245313644
Step 838, loss: 0.10166560113430023
Step 839, loss: 0.09757618606090546
Step 840, loss: 0.09132245182991028
Step 841, loss: 0.08452916145324707
Step 842, loss: 0.09208028763532639
Step 843, loss: 0.08603155612945557
Step 844, loss: 0.09362247586250305
Step 845, loss: 0.09389328211545944
Step 846, loss: 0.09670235216617584
Step 847, loss: 0.08223637193441391
Step 848, loss: 0.09724485874176025
Step 849, loss: 0.09867923706769943
Step 850, loss: 0.09648924320936203
Step 851, loss: 0.09133214503526688
Step 852, loss: 0.0919305756688118
Step 853, loss: 0.09196174889802933
Step 854, loss: 0.09482873231172562
Step 855, loss: 0.08248186111450195
Step 856, loss: 0.08580174297094345
Step 857, loss: 0.08512967079877853
Step 858, loss: 0.09314458817243576
Step 859, loss: 0.09920888394117355
Step 860, loss: 0.09327910840511322
Step 861, loss: 0.09319480508565903
Step 862, loss: 0.09891754388809204
Step 863, loss: 0.09009914845228195
Step 864, loss: 0.09909714013338089
Step 865, loss: 0.08753421902656555
Step 866, loss: 0.08976718038320541
Step 867, loss: 0.09085462987422943
Step 868, loss: 0.08940781652927399
Step 869, loss: 0.09631145745515823
Step 870, loss: 0.08161468803882599
Step 871, loss: 0.08463296294212341
Step 872, loss: 0.07781345397233963
Step 873, loss: 0.0905420109629631
Step 874, loss: 0.09272245317697525
Step 875, loss: 0.09359772503376007
Step 876, loss: 0.09707276523113251
Step 877, loss: 0.09201294928789139
Step 878, loss: 0.08891638368368149
Step 879, loss: 0.0860452950000763
Step 880, loss: 0.08942006528377533
Step 881, loss: 0.08601448684930801
Step 882, loss: 0.08423952013254166
Step 883, loss: 0.09707909822463989
Step 884, loss: 0.08760077506303787
Step 885, loss: 0.09261993318796158
Step 886, loss: 0.08634145557880402
Step 887, loss: 0.09433414041996002
Step 888, loss: 0.08238493651151657
Step 889, loss: 0.09113699197769165
Step 890, loss: 0.08266004920005798
Step 891, loss: 0.08712980896234512
Step 892, loss: 0.08928230404853821
Step 893, loss: 0.08224313706159592
Step 894, loss: 0.0874875858426094
Step 895, loss: 0.09045640379190445
Step 896, loss: 0.08254431933164597
Step 897, loss: 0.0905306488275528
Step 898, loss: 0.08707711845636368
Step 899, loss: 0.08874648809432983
Step 900, loss: 0.09037651121616364
Step 901, loss: 0.09148481488227844
Step 902, loss: 0.08462053537368774
Step 903, loss: 0.09254521131515503
Step 904, loss: 0.08742266148328781
Step 905, loss: 0.08461026847362518
Step 906, loss: 0.0818031057715416
Step 907, loss: 0.08320505917072296
Step 908, loss: 0.07993368804454803
Step 909, loss: 0.08304351568222046
Step 910, loss: 0.08912152051925659
Step 911, loss: 0.08884666860103607
Step 912, loss: 0.08878818154335022
Step 913, loss: 0.0775521844625473
Step 914, loss: 0.09132090955972672
Step 915, loss: 0.09034376591444016
Step 916, loss: 0.08351545035839081
Step 917, loss: 0.08032996952533722
Step 918, loss: 0.08241572231054306
Step 919, loss: 0.0864022895693779
Step 920, loss: 0.07888314127922058
Step 921, loss: 0.08483795821666718
Step 922, loss: 0.07833589613437653
Step 923, loss: 0.08603530377149582
Step 924, loss: 0.09251593053340912
Step 925, loss: 0.08615638315677643
Step 926, loss: 0.08134191483259201
Step 927, loss: 0.08138392120599747
Step 928, loss: 0.08150901645421982
Step 929, loss: 0.08756126463413239
Step 930, loss: 0.08216021209955215
Step 931, loss: 0.09471791237592697
Step 932, loss: 0.07998700439929962
Step 933, loss: 0.08480123430490494
Step 934, loss: 0.07715199142694473
Step 935, loss: 0.08756698668003082
Step 936, loss: 0.07835286855697632
Step 937, loss: 0.08176742494106293
Step 938, loss: 0.07896874845027924
Step 939, loss: 0.09003804624080658
Step 940, loss: 0.07780475914478302
Step 941, loss: 0.08770690858364105
Step 942, loss: 0.08561493456363678
Step 943, loss: 0.075486920773983
Step 944, loss: 0.07506661117076874
Step 945, loss: 0.0814749002456665
Step 946, loss: 0.08538120239973068
Step 947, loss: 0.07776882499456406
Step 948, loss: 0.07888418436050415
Step 949, loss: 0.07454055547714233
Step 950, loss: 0.08397838473320007
Step 951, loss: 0.07802443206310272
Step 952, loss: 0.08031848073005676
Step 953, loss: 0.08511370420455933
Step 954, loss: 0.0867534875869751
Step 955, loss: 0.08922683447599411
Step 956, loss: 0.0775240957736969
Step 957, loss: 0.08229722082614899
Step 958, loss: 0.07464175671339035
Step 959, loss: 0.07697487622499466
Step 960, loss: 0.09046315401792526
Step 961, loss: 0.08281946182250977
Step 962, loss: 0.07445630431175232
Step 963, loss: 0.07952401041984558
Step 964, loss: 0.07730196416378021
Step 965, loss: 0.07921254634857178
Step 966, loss: 0.07939676940441132
Step 967, loss: 0.0866658017039299
Step 968, loss: 0.07874897867441177
Step 969, loss: 0.0822242796421051
Step 970, loss: 0.07978344708681107
Step 971, loss: 0.08035261929035187
Step 972, loss: 0.08111467957496643
Step 973, loss: 0.07890137284994125
Step 974, loss: 0.0807160809636116
Step 975, loss: 0.0786462128162384
Step 976, loss: 0.08104701340198517
Step 977, loss: 0.06922311335802078
Step 978, loss: 0.07651462405920029
Step 979, loss: 0.0779964029788971
Step 980, loss: 0.08512242883443832
Step 981, loss: 0.07634199410676956
Step 982, loss: 0.07581279426813126
Step 983, loss: 0.07041185349225998
Step 984, loss: 0.0808395966887474
Step 985, loss: 0.08379776775836945
Step 986, loss: 0.08144515752792358
Step 987, loss: 0.06697431206703186
Step 988, loss: 0.07608408480882645
Step 989, loss: 0.08168733865022659
Step 990, loss: 0.08434491604566574
Step 991, loss: 0.07341980934143066
Step 992, loss: 0.085722915828228
Step 993, loss: 0.07922551035881042
Step 994, loss: 0.07952472567558289
Step 995, loss: 0.07539788633584976
Step 996, loss: 0.07679484784603119
Step 997, loss: 0.083320252597332
Step 998, loss: 0.07130467891693115
Step 999, loss: 0.08044004440307617
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/train_lam.py", line 464, in <module>
    checkpoint_manager.save(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/checkpoint_manager.py", line 1467, in save
    self._checkpointer.save(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/async_checkpointer.py", line 543, in save
    commit_ops = asyncio_utils.run_sync(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 36, in run_sync
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/async_checkpointer.py", line 488, in _save
    await self._handler.async_save(tmpdir.get(), args=ckpt_args) or []
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/composite_checkpoint_handler.py", line 706, in async_save
    ).result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 429, in result
    return self._f.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 378, in result
    return self._t.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 327, in result
    self.join(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 324, in join
    raise self._exception
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 310, in run
    self._wait_for_signals()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 285, in _wait_for_signals
    client.blocking_key_value_get(barrier_key, self._timeout_secs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/signaling_client.py", line 137, in blocking_key_value_get
    return str(self._client.blocking_key_value_get(key, timeout_secs * 1000))
jaxlib._jax.XlaRuntimeError: DEADLINE_EXCEEDED: GetKeyValue() timed out with key: awaitable_signals_contract_1/item_directory_creation and duration: 5m
wandb: 
wandb: 🚀 View run train_lam_coinrun_311M_3469569 at: https://wandb.ai/instant-uv/jafar/runs/3469569
wandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine_jobs/wandb/run-20250905_223410-3469569/logs
/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 10 leaked shared_memory objects to clean up at shutdown
  warnings.warn('resource_tracker: There appear to be %d '
srun: error: hkn0729: task 0: Exited with exit code 1

============================= JOB FEEDBACK =============================

Job ID: 3469569
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 1)
Partition: accelerated
Nodes: 1
Cores per node: 6
Nodelist: hkn0729
CPU Utilized: 00:16:45
CPU Efficiency: 8.90% of 03:08:06 core-walltime
Job Wall-clock time: 00:31:21
Starttime: Fri Sep  5 22:29:55 2025
Endtime: Fri Sep  5 23:01:16 2025
Memory Utilized: 5.92 GB
Memory Efficiency: 0.00% of 0.00 MB
Energy Consumed: 1974844 Joule / 548.567777777778 Watthours
Average node power draw: 1049.89048378522 Watt