[tab] /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/train_lam_model_size_scaling_38M_3317231.log (log)
#!/usr/bin/env bash

#SBATCH --nodes=2
#SBATCH --ntasks-per-node=4
#SBATCH --time=36:00:00
#SBATCH --partition=accelerated
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/%x_%j.log
#SBATCH --job-name=train_lam_model_size_scaling_38M

# Log the sbatch script
cat $0

module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate

tf_records_dir=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/open_ai_minecraft_tfrecord

job_name=$SLURM_JOB_NAME
slurm_job_id=$SLURM_JOB_ID

CHECKPOINT_DIR=/home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/$job_name/$slurm_job_id
mkdir -p $CHECKPOINT_DIR

env | grep SLURM

srun python train_lam.py \
    --ckpt_dir $CHECKPOINT_DIR \
    --batch_size=96 \
    --min_lr=1e-4 \
    --max_lr=1e-4 \
    --log_image_interval=500 \
    --log_checkpoint_interval=500 \
    --log \
    --name=lam-model-size-scaling-38M-$slurm_job_id \
    --tags lam model-size-scaling 38M \
    --entity instant-uv \
    --project jafar \
    --data_dir $tf_records_dir
/var/spool/slurmd/job3317231/slurm_script: line 18: .venv/bin/activate: No such file or directory
SLURM_STEP_NUM_TASKS=1
SLURM_JOB_USER=tum_cte0515
SLURM_TASKS_PER_NODE=4(x2)
SLURM_JOB_UID=999226
SLURM_TASK_PID=3431510
SLURM_JOB_GPUS=0,1,2,3
SLURM_LOCALID=0
SLURM_SUBMIT_DIR=/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs
SLURMD_NODENAME=hkn0407
SLURM_JOB_START_TIME=1751625228
SLURM_STEP_NODELIST=hkn0625
SLURM_CLUSTER_NAME=hk
SLURM_JOB_END_TIME=1751754828
SLURM_PMI2_SRUN_PORT=38731
SLURM_CPUS_ON_NODE=24
SLURM_JOB_CPUS_PER_NODE=24(x2)
SLURM_GPUS_ON_NODE=4
SLURM_GTIDS=0
SLURM_JOB_PARTITION=accelerated
SLURM_TRES_PER_TASK=cpu=5
SLURM_OOM_KILL_STEP=0
SLURM_JOB_NUM_NODES=2
SLURM_STEPID=4294967290
SLURM_JOBID=3317231
SLURM_PTY_PORT=39987
SLURM_JOB_QOS=normal
SLURM_LAUNCH_NODE_IPADDR=10.0.7.201
SLURM_PTY_WIN_ROW=50
SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))
SLURMD_DEBUG=2
SLURM_PROCID=0
SLURM_CPUS_PER_TASK=5
SLURM_NTASKS=8
SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e12.hkn0407
SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.node
SLURM_SRUN_COMM_HOST=10.0.7.201
SLURM_SCRIPT_CONTEXT=prolog_task
SLURM_PTY_WIN_COL=108
SLURM_NODELIST=hkn[0407,0413]
SLURM_SRUN_COMM_PORT=39403
SLURM_STEP_ID=4294967290
SLURM_JOB_ACCOUNT=hk-project-p0023960
SLURM_PRIO_PROCESS=0
SLURM_NPROCS=8
SLURM_NNODES=2
SLURM_SUBMIT_HOST=hkn0625.localdomain
SLURM_JOB_ID=3317231
SLURM_NODEID=0
SLURM_STEP_NUM_NODES=1
SLURM_STEP_TASKS_PER_NODE=1
SLURM_MPI_TYPE=pmi2
SLURM_PMI2_STEP_NODES=hkn0625
SLURM_CONF=/etc/slurm/slurm.conf
SLURM_JOB_NAME=train_lam_model_size_scaling_38M
SLURM_NTASKS_PER_NODE=4
SLURM_STEP_LAUNCHER_PORT=39403
SLURM_JOB_GID=502226
SLURM_JOB_NODELIST=hkn[0407,0413]
GpuFreq=control_disabled
GpuFreq=control_disabled
2025-07-04 12:34:18.563664: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-04 12:34:18.564350: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-04 12:34:18.564265: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-04 12:34:18.564400: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625258.576912 3431574 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625258.577189 3431572 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625258.577723 3431573 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625258.577655 3431575 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1751625258.581523 3431572 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751625258.581512 3431574 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751625258.582008 3431573 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751625258.582078 3431575 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1751625258.594522 3431572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594538 3431572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594540 3431572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594541 3431572 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594759 3431575 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594775 3431575 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594777 3431575 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594779 3431575 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594840 3431573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594854 3431573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594856 3431573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594858 3431573 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594975 3431574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594991 3431574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594993 3431574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625258.594994 3431574 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625265.166956 3431573 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751625265.167093 3431575 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751625265.167514 3431572 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751625265.167486 3431574 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2025-07-04 12:34:27.131689: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-04 12:34:27.132513: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-04 12:34:27.132616: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
2025-07-04 12:34:27.133008: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:467] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625267.144733 3970046 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625267.145700 3970044 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625267.145852 3970045 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
E0000 00:00:1751625267.146051 3970047 cuda_dnn.cc:8579] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered
E0000 00:00:1751625267.149196 3970046 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751625267.149933 3970044 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751625267.150222 3970047 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
E0000 00:00:1751625267.150272 3970045 cuda_blas.cc:1407] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered
W0000 00:00:1751625267.163086 3970044 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163102 3970044 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163103 3970044 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163105 3970044 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163049 3970046 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163067 3970046 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163068 3970046 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163070 3970046 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163415 3970045 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163432 3970045 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163435 3970045 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163437 3970045 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163413 3970047 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163432 3970047 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163434 3970047 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625267.163436 3970047 computation_placer.cc:177] computation placer already registered. Please check linkage and avoid linking the same target more than once.
W0000 00:00:1751625282.181882 3970045 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751625282.201164 3970044 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751625282.201799 3970047 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
W0000 00:00:1751625282.201899 3970046 gpu_device.cc:2341] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.
Skipping registering GPU devices...
2025-07-04 12:34:57.857567: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:34:58.282523: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:34:58.317372: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:34:58.378935: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:34:59.269840: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:34:59.820007: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:34:59.937557: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:35:00.465693: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
wandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force relogin
wandb: Tracking run with wandb version 0.19.11
wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/wandb/run-20250704_123513-tqnpuqtl
wandb: Run `wandb offline` to turn off syncing.
wandb: Syncing run lam-model-size-scaling-38M-3317231
wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafar
wandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/tqnpuqtl
2025-07-04 12:35:39.601678: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:35:39.679408: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:35:41.441449: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
2025-07-04 12:35:41.441493: W external/xla/xla/service/gpu/autotuning/dot_search_space.cc:200] All configs were filtered out because none of them sufficiently match the hints. Maybe the hints set does not contain a good representative set of valid configs?Working around this by using the full hints set instead.
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 106, loss: 0.021131999790668488
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 106, loss: 0.021131999790668488
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 218, loss: 0.01277262531220913
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 218, loss: 0.01277262531220913
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 106, loss: 0.021131999790668488
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 106, loss: 0.021131999790668488
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 106, loss: 0.021131999790668488
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 106, loss: 0.021131999790668488
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 218, loss: 0.01277262531220913
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Step 218, loss: 0.01277262531220913
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 218, loss: 0.01277262531220913
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 218, loss: 0.01277262531220913
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Running on 8 devices.
Counting all components: ['action_in', 'encoder', 'vq', 'action_up', 'patch_up', 'decoder']
Parameter counts:
{'action_in': 768, 'encoder': 19348512, 'vq': 192, 'action_up': 16896, 'patch_up': 393728, 'decoder': 19594496, 'total': 39354592}
Starting training from step 0...
Step 0, loss: 0.3458659052848816
Step 1, loss: 0.27324220538139343
Step 2, loss: 0.2355320006608963
Step 3, loss: 0.2047692835330963
Step 4, loss: 0.1823614239692688
Step 5, loss: 0.16627132892608643
Step 6, loss: 0.15348856151103973
Step 7, loss: 0.14179682731628418
Step 8, loss: 0.12955041229724884
Step 9, loss: 0.11650186032056808
Step 10, loss: 0.11342614144086838
Step 11, loss: 0.10472942143678665
Step 12, loss: 0.10533195734024048
Step 13, loss: 0.09179207682609558
Step 14, loss: 0.09982488304376602
Step 15, loss: 0.093877412378788
Step 16, loss: 0.09457507729530334
Step 17, loss: 0.08418048173189163
Step 18, loss: 0.08506400883197784
Step 19, loss: 0.08447198569774628
Step 20, loss: 0.08065280318260193
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 21, loss: 0.07959970086812973
Step 22, loss: 0.0701492577791214
Step 23, loss: 0.06879846006631851
Step 24, loss: 0.07646529376506805
Step 25, loss: 0.0749850645661354
Step 26, loss: 0.07867615669965744
Step 27, loss: 0.06321500241756439
Step 28, loss: 0.06965070962905884
Step 29, loss: 0.06833157688379288
Step 30, loss: 0.06117798388004303
Step 31, loss: 0.05848240107297897
Step 32, loss: 0.06443576514720917
Step 33, loss: 0.057378269731998444
Step 34, loss: 0.059131670743227005
Step 35, loss: 0.05590574070811272
Step 36, loss: 0.05625388026237488
Step 37, loss: 0.0576728992164135
Step 38, loss: 0.05682222172617912
Step 39, loss: 0.05153699591755867
Step 40, loss: 0.052843961864709854
Step 41, loss: 0.057516202330589294
Step 42, loss: 0.056366223841905594
Step 43, loss: 0.05159895122051239
Step 44, loss: 0.04783410578966141
Step 45, loss: 0.048604611307382584
Step 46, loss: 0.04555995389819145
Step 47, loss: 0.0586760975420475
Step 48, loss: 0.04466046392917633
Step 49, loss: 0.044995322823524475
Step 106, loss: 0.021131999790668488
Step 50, loss: 0.04786723107099533
Step 51, loss: 0.04310321807861328
Step 52, loss: 0.042149629443883896
Step 53, loss: 0.041286516934633255
Step 54, loss: 0.040004074573516846
Step 55, loss: 0.03892122209072113
Step 56, loss: 0.040705833584070206
Step 57, loss: 0.03866808861494064
Step 58, loss: 0.038875285536050797
Step 59, loss: 0.0382038950920105
Step 60, loss: 0.03675457462668419
Step 61, loss: 0.036354538053274155
Step 62, loss: 0.034882936626672745
Step 63, loss: 0.03890756890177727
Step 64, loss: 0.035898201167583466
Step 65, loss: 0.036937594413757324
Step 66, loss: 0.034122467041015625
Step 67, loss: 0.03394459933042526
Step 68, loss: 0.03548620268702507
Step 69, loss: 0.0349210761487484
Step 70, loss: 0.031405285000801086
Step 71, loss: 0.03282681480050087
Step 72, loss: 0.0314599983394146
Step 73, loss: 0.033970631659030914
Step 74, loss: 0.03401556611061096
Step 75, loss: 0.0314575731754303
Step 76, loss: 0.03303542733192444
Step 77, loss: 0.031338371336460114
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 78, loss: 0.030676575377583504
Step 79, loss: 0.029457416385412216
Step 80, loss: 0.028541158884763718
Step 81, loss: 0.026114456355571747
Step 82, loss: 0.02477286010980606
Step 83, loss: 0.027484996244311333
Step 84, loss: 0.027294613420963287
Step 85, loss: 0.02647419273853302
Step 86, loss: 0.02623198740184307
Step 87, loss: 0.02527686208486557
Step 88, loss: 0.026870649307966232
Step 89, loss: 0.024022037163376808
Step 90, loss: 0.02493225783109665
Step 91, loss: 0.026188870891928673
Step 92, loss: 0.022658172994852066
Step 93, loss: 0.02400360256433487
Step 94, loss: 0.02161259762942791
Step 95, loss: 0.02284260280430317
Step 96, loss: 0.02147476002573967
Step 97, loss: 0.021621625870466232
Step 98, loss: 0.021774405613541603
Step 99, loss: 0.02198221907019615
Step 100, loss: 0.022473150864243507
Step 101, loss: 0.020701657980680466
Step 102, loss: 0.023333365097641945
Step 103, loss: 0.02230330929160118
Step 104, loss: 0.018153030425310135
Step 105, loss: 0.019788343459367752
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 106, loss: 0.021131999790668488
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 107, loss: 0.018866421654820442
Step 108, loss: 0.0192547757178545
Step 109, loss: 0.021436048671603203
Step 110, loss: 0.019234558567404747
Step 111, loss: 0.01894272491335869
Step 112, loss: 0.019345417618751526
Step 113, loss: 0.018582109361886978
Step 114, loss: 0.01975010521709919
Step 115, loss: 0.019443674013018608
Step 116, loss: 0.019243387505412102
Step 117, loss: 0.01915769651532173
Step 118, loss: 0.018687965348362923
Step 119, loss: 0.017585808411240578
Step 120, loss: 0.01869960129261017
Step 121, loss: 0.01882147043943405
Step 122, loss: 0.01733372174203396
Step 123, loss: 0.016829082742333412
Step 124, loss: 0.016461463645100594
Step 125, loss: 0.016653893515467644
Step 126, loss: 0.01756332814693451
Step 127, loss: 0.017925303429365158
Step 128, loss: 0.014997621066868305
Step 129, loss: 0.01633789762854576
Step 130, loss: 0.01663167215883732
Step 131, loss: 0.014768940396606922
Step 132, loss: 0.017631065100431442
Step 133, loss: 0.01582164317369461
Step 134, loss: 0.01458656881004572
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Step 135, loss: 0.015421072021126747
Step 136, loss: 0.01632264442741871
Step 137, loss: 0.01814889907836914
Step 138, loss: 0.01644924096763134
Step 139, loss: 0.018358947709202766
Step 140, loss: 0.015989867970347404
Step 141, loss: 0.016648894175887108
Step 142, loss: 0.014685309492051601
Step 143, loss: 0.014101077802479267
Step 144, loss: 0.016734518110752106
Step 145, loss: 0.01576446369290352
Step 146, loss: 0.015219144523143768
Step 147, loss: 0.015657775104045868
Step 148, loss: 0.01360900979489088
Step 149, loss: 0.015071406960487366
Step 150, loss: 0.014548876322805882
Step 151, loss: 0.01495143212378025
Step 152, loss: 0.014997485093772411
Step 153, loss: 0.015895482152700424
Step 154, loss: 0.016132650896906853
Step 155, loss: 0.01631644368171692
Step 156, loss: 0.015095235779881477
Step 157, loss: 0.01362325344234705
Step 158, loss: 0.01491172518581152
Step 159, loss: 0.01417522318661213
Step 160, loss: 0.015491900034248829
Step 161, loss: 0.015892736613750458
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Step 162, loss: 0.015378686599433422
Step 163, loss: 0.015704510733485222
Step 164, loss: 0.013949031010270119
Step 165, loss: 0.014782312326133251
Step 166, loss: 0.012765937484800816
Step 167, loss: 0.0158412903547287
Step 168, loss: 0.013001887127757072
Step 169, loss: 0.013110092841088772
Step 170, loss: 0.012727662920951843
Step 171, loss: 0.014002700336277485
Step 172, loss: 0.013821480795741081
Step 173, loss: 0.013868322595953941
Step 174, loss: 0.013336071744561195
Step 175, loss: 0.0126531682908535
Step 176, loss: 0.01245906949043274
Step 177, loss: 0.014464332722127438
Step 178, loss: 0.014265761710703373
Step 179, loss: 0.01245016697794199
Step 180, loss: 0.01349999476224184
Step 181, loss: 0.014596237801015377
Step 182, loss: 0.012346822768449783
Step 183, loss: 0.013393871486186981
Step 184, loss: 0.012630213052034378
Step 185, loss: 0.014780499041080475
Step 186, loss: 0.012312822043895721
Step 187, loss: 0.012942979112267494
Step 188, loss: 0.014443048276007175
Step 218, loss: 0.01277262531220913
Step 189, loss: 0.01292951125651598
Step 190, loss: 0.013043065555393696
Step 191, loss: 0.012180130928754807
Step 192, loss: 0.011598044075071812
Step 193, loss: 0.011594828218221664
Step 194, loss: 0.012007116340100765
Step 195, loss: 0.013300358317792416
Step 196, loss: 0.010977167636156082
Step 197, loss: 0.008753481321036816
Step 198, loss: 0.012383922934532166
Step 199, loss: 0.011551115661859512
Step 200, loss: 0.011783098801970482
Step 201, loss: 0.012840569950640202
Step 202, loss: 0.013121400028467178
Step 203, loss: 0.011892827227711678
Step 204, loss: 0.011296260170638561
Step 205, loss: 0.012015299871563911
Step 206, loss: 0.012021327391266823
Step 207, loss: 0.011571449227631092
Step 208, loss: 0.011294446885585785
Step 209, loss: 0.01234876736998558
Step 210, loss: 0.010591709986329079
Step 211, loss: 0.011829040944576263
Step 212, loss: 0.010938741266727448
Step 213, loss: 0.010416306555271149
Step 214, loss: 0.011827440932393074
Step 215, loss: 0.012671248987317085
Step 216, loss: 0.010480878874659538
Step 217, loss: 0.012251722626388073
Step 218, loss: 0.01277262531220913
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 219, loss: 0.012239567935466766
Step 220, loss: 0.012136262841522694
Step 221, loss: 0.011409077793359756
Step 222, loss: 0.010637045837938786
Step 223, loss: 0.011017616838216782
Step 224, loss: 0.010045788250863552
Step 225, loss: 0.010749712586402893
Step 226, loss: 0.01251320168375969
Step 227, loss: 0.01298257615417242
Step 228, loss: 0.010440370999276638
Step 229, loss: 0.010860598646104336
Step 230, loss: 0.010668632574379444
Step 231, loss: 0.010373189114034176
Step 232, loss: 0.011863458901643753
Step 233, loss: 0.009629851207137108
Step 234, loss: 0.01165230292826891
Step 235, loss: 0.009904655627906322
Step 236, loss: 0.0106987738981843
Step 237, loss: 0.010785465128719807
Step 238, loss: 0.011452878825366497
Step 239, loss: 0.010458522476255894
Step 240, loss: 0.011552414856851101
Step 241, loss: 0.012715709395706654
Step 242, loss: 0.012320378795266151
Step 243, loss: 0.012911004014313221
Step 244, loss: 0.011917850002646446
Step 245, loss: 0.01167557667940855
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 246, loss: 0.01232021301984787
Step 247, loss: 0.011485982686281204
Step 248, loss: 0.011529898270964622
Step 249, loss: 0.010411110706627369
Step 250, loss: 0.010955167934298515
Step 251, loss: 0.009564117528498173
Step 252, loss: 0.01063984539359808
Step 253, loss: 0.011305712163448334
Step 254, loss: 0.010590657591819763
Step 255, loss: 0.010182423517107964
Step 256, loss: 0.009896081872284412
Step 257, loss: 0.011948817409574986
Step 258, loss: 0.01115728635340929
Step 259, loss: 0.01027585007250309
Step 260, loss: 0.010528629645705223
Step 261, loss: 0.011556326411664486
Step 262, loss: 0.009801896288990974
Step 263, loss: 0.01022157073020935
Step 264, loss: 0.010655556805431843
Step 265, loss: 0.01084305252879858
Step 266, loss: 0.011191139928996563
Step 267, loss: 0.013045080006122589
Step 268, loss: 0.009735999628901482
Step 269, loss: 0.011502969078719616
Step 270, loss: 0.009793016128242016
Step 271, loss: 0.01102183386683464
Step 272, loss: 0.010651337914168835
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 273, loss: 0.009178319945931435
Step 274, loss: 0.009884879924356937
Step 275, loss: 0.011208660900592804
Step 276, loss: 0.010228198952972889
Step 277, loss: 0.010056668892502785
Step 278, loss: 0.011009006761014462
Step 279, loss: 0.010305356234312057
Step 280, loss: 0.011615189723670483
Step 281, loss: 0.010169465094804764
Step 282, loss: 0.009980586357414722
Step 283, loss: 0.01036019902676344
Step 284, loss: 0.009485349990427494
Step 285, loss: 0.009453698992729187
Step 286, loss: 0.010658933781087399
Step 287, loss: 0.009532594121992588
Step 288, loss: 0.011737916618585587
Step 289, loss: 0.01038997434079647
Step 290, loss: 0.010077100247144699
Step 291, loss: 0.011366307735443115
Step 292, loss: 0.009410296566784382
Step 293, loss: 0.012366150505840778
Step 294, loss: 0.010770988650619984
Step 295, loss: 0.00953879114240408
Step 296, loss: 0.009707973338663578
Step 297, loss: 0.009966018609702587
Step 298, loss: 0.008968329057097435
Step 299, loss: 0.009785414673388004
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 300, loss: 0.010559550486505032
Step 301, loss: 0.010274802334606647
Step 302, loss: 0.0096120685338974
Step 303, loss: 0.009493944235146046
Step 304, loss: 0.00902603380382061
Step 305, loss: 0.00974448025226593
Step 306, loss: 0.008781787008047104
Step 307, loss: 0.01023856457322836
Step 308, loss: 0.011048885062336922
Step 309, loss: 0.011098461225628853
Step 310, loss: 0.009219963103532791
Step 311, loss: 0.011409247294068336
Step 312, loss: 0.009977485984563828
Step 313, loss: 0.008882073685526848
Step 314, loss: 0.009449816308915615
Step 315, loss: 0.009574308060109615
Step 316, loss: 0.010238993912935257
Step 317, loss: 0.009443148970603943
Step 318, loss: 0.008387591689825058
Step 319, loss: 0.009112976491451263
Step 320, loss: 0.009829151444137096
Step 321, loss: 0.011092578992247581
Step 322, loss: 0.008623616769909859
Step 323, loss: 0.009793819859623909
Step 324, loss: 0.010933523066341877
Step 325, loss: 0.00999410916119814
Step 326, loss: 0.009676044806838036
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 327, loss: 0.009505968540906906
Step 328, loss: 0.010849532671272755
Step 329, loss: 0.009458868764340878
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 330, loss: 0.009576462209224701
Step 331, loss: 0.009329484775662422
Step 332, loss: 0.00852852500975132
Step 333, loss: 0.009237044490873814
Step 334, loss: 0.008740132674574852
Step 335, loss: 0.008466384373605251
Step 336, loss: 0.008425428532063961
Step 337, loss: 0.009161649271845818
Step 338, loss: 0.009374246001243591
Step 339, loss: 0.00892417412251234
Step 340, loss: 0.009873379953205585
Step 341, loss: 0.00905925314873457
Step 342, loss: 0.008672410622239113
Step 343, loss: 0.008448871783912182
Step 344, loss: 0.009507987648248672
Step 345, loss: 0.00877519603818655
Step 346, loss: 0.009930066764354706
Step 347, loss: 0.010428224690258503
Step 348, loss: 0.008662153966724873
Step 349, loss: 0.010449899360537529
Step 350, loss: 0.009567062370479107
Step 351, loss: 0.008004763163626194
Step 352, loss: 0.009722573682665825
Step 353, loss: 0.00915669184178114
Step 354, loss: 0.00920177809894085
Step 355, loss: 0.010156840085983276
Step 356, loss: 0.010528013110160828
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 357, loss: 0.00898850616067648
Step 358, loss: 0.008326129987835884
Step 359, loss: 0.009215407073497772
Step 360, loss: 0.009671839885413647
Step 361, loss: 0.009215022437274456
Step 362, loss: 0.008158013224601746
Step 363, loss: 0.010732646100223064
Step 364, loss: 0.01066561508923769
Step 365, loss: 0.010638725012540817
Step 366, loss: 0.011673283763229847
Step 367, loss: 0.00986537430435419
Step 368, loss: 0.010853351093828678
Step 369, loss: 0.010740447789430618
Step 370, loss: 0.009509650990366936
Step 371, loss: 0.010256513953208923
Step 372, loss: 0.008613554760813713
Step 373, loss: 0.01091273594647646
Step 374, loss: 0.011150242760777473
Step 375, loss: 0.008022014051675797
Step 376, loss: 0.010051189921796322
Step 377, loss: 0.008960792794823647
Step 378, loss: 0.00869739893823862
Step 379, loss: 0.00863656960427761
Step 380, loss: 0.008822939358651638
Step 381, loss: 0.01000864990055561
Step 382, loss: 0.00885055959224701
Step 383, loss: 0.009356740862131119
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 384, loss: 0.008903071284294128
Step 385, loss: 0.007909400388598442
Step 386, loss: 0.008600326254963875
Step 387, loss: 0.009601408615708351
Step 388, loss: 0.0083704749122262
Step 389, loss: 0.010405316948890686
Step 390, loss: 0.009074941277503967
Step 391, loss: 0.00924819614738226
Step 392, loss: 0.008861569687724113
Step 393, loss: 0.008937230333685875
Step 394, loss: 0.008747048676013947
Step 395, loss: 0.00938418135046959
Step 396, loss: 0.008845445699989796
Step 397, loss: 0.009564661420881748
Step 398, loss: 0.008053848519921303
Step 399, loss: 0.008553571999073029
Step 400, loss: 0.009211188182234764
Step 401, loss: 0.00885711144655943
Step 402, loss: 0.010055947117507458
Step 403, loss: 0.008107040077447891
Step 404, loss: 0.008024687878787518
Step 405, loss: 0.009033575654029846
Step 406, loss: 0.007763940375298262
Step 407, loss: 0.009587476961314678
Step 408, loss: 0.009001743979752064
Step 409, loss: 0.00820583663880825
Step 410, loss: 0.00929223746061325
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 411, loss: 0.008519451133906841
Step 412, loss: 0.009908848442137241
Step 413, loss: 0.009280987083911896
Step 414, loss: 0.008972147479653358
Step 415, loss: 0.008368583396077156
Step 416, loss: 0.008258205838501453
Step 417, loss: 0.008972324430942535
Step 418, loss: 0.009582041762769222
Step 419, loss: 0.008781615644693375
Step 420, loss: 0.009339810349047184
Step 421, loss: 0.010058864951133728
Step 422, loss: 0.010141564533114433
Step 423, loss: 0.008403551764786243
Step 424, loss: 0.008909630589187145
Step 425, loss: 0.008917036466300488
Step 426, loss: 0.008785653859376907
Step 427, loss: 0.009656546637415886
Step 428, loss: 0.008250284008681774
Step 429, loss: 0.008734671398997307
Step 430, loss: 0.009341408498585224
Step 431, loss: 0.010086811147630215
Step 432, loss: 0.009699289686977863
Step 433, loss: 0.008093277923762798
Step 434, loss: 0.00828953180462122
Step 435, loss: 0.009567582048475742
Step 436, loss: 0.008768579922616482
Step 437, loss: 0.007845825515687466
Step 438, loss: 0.009181627072393894
Step 439, loss: 0.008273782208561897
Step 440, loss: 0.009182600304484367
Step 441, loss: 0.009179329499602318
Step 442, loss: 0.008932436816394329
Step 443, loss: 0.010333123616874218
Step 444, loss: 0.01019835751503706
Step 445, loss: 0.009480638429522514
Step 446, loss: 0.009658812545239925
Step 447, loss: 0.008269150741398335
Step 448, loss: 0.011066804639995098
Step 449, loss: 0.009733232669532299
Step 450, loss: 0.008351570926606655
Step 451, loss: 0.008620851673185825
Step 452, loss: 0.008884967304766178
Step 453, loss: 0.00846097432076931
Step 454, loss: 0.009176055900752544
Step 455, loss: 0.009038573130965233
Step 456, loss: 0.008845883421599865
Step 457, loss: 0.008126926608383656
Step 458, loss: 0.009394383989274502
Step 459, loss: 0.008409657515585423
Step 460, loss: 0.009345205500721931
Step 461, loss: 0.008491947315633297
Step 462, loss: 0.0075165280140936375
Step 463, loss: 0.00986082199960947
Step 464, loss: 0.008375565521419048
Step 465, loss: 0.010128077119588852
Step 466, loss: 0.008682378567755222
Step 467, loss: 0.008810698054730892
Step 468, loss: 0.009016426280140877
Step 469, loss: 0.00868071150034666
Step 470, loss: 0.008902246132493019
Step 471, loss: 0.00858953595161438
Step 472, loss: 0.009066066704690456
Step 473, loss: 0.008939971216022968
Step 474, loss: 0.008642027154564857
Step 475, loss: 0.011356632225215435
Step 476, loss: 0.008728811517357826
Step 477, loss: 0.007195563986897469
Step 478, loss: 0.009304963052272797
Step 479, loss: 0.008956977166235447
Step 480, loss: 0.008378159254789352
Step 481, loss: 0.010088315233588219
Step 482, loss: 0.008494030684232712
Step 483, loss: 0.009125368669629097
Step 484, loss: 0.008969607762992382
Step 485, loss: 0.009077407419681549
Step 486, loss: 0.007636579684913158
Step 487, loss: 0.008236422203481197
Step 488, loss: 0.008791486732661724
Step 489, loss: 0.00945952720940113
Step 490, loss: 0.008376586250960827
Step 491, loss: 0.007897509261965752
Step 492, loss: 0.009305316023528576
Step 493, loss: 0.00865597277879715
Step 494, loss: 0.008607441559433937
Step 495, loss: 0.008625464513897896
Step 496, loss: 0.008877462707459927
Step 497, loss: 0.008101128041744232
Step 498, loss: 0.00966381374746561
Step 499, loss: 0.008945940062403679
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 159, in wait_for_base_dir_creation
    await asyncio.sleep(0.25)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 605, in sleep
    return await future
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 162, in _maybe_create_base_dir
Step 441, loss: 0.009179329499602318
Step 442, loss: 0.008932436816394329
Step 443, loss: 0.010333123616874218
Step 444, loss: 0.01019835751503706
Step 445, loss: 0.009480638429522514
Step 446, loss: 0.009658812545239925
Step 447, loss: 0.008269150741398335
Step 448, loss: 0.011066804639995098
Step 449, loss: 0.009733232669532299
Step 450, loss: 0.008351570926606655
Step 451, loss: 0.008620851673185825
Step 452, loss: 0.008884967304766178
Step 453, loss: 0.00846097432076931
Step 454, loss: 0.009176055900752544
Step 455, loss: 0.009038573130965233
Step 456, loss: 0.008845883421599865
Step 457, loss: 0.008126926608383656
Step 458, loss: 0.009394383989274502
Step 459, loss: 0.008409657515585423
Step 460, loss: 0.009345205500721931
Step 461, loss: 0.008491947315633297
Step 462, loss: 0.0075165280140936375
Step 463, loss: 0.00986082199960947
Step 464, loss: 0.008375565521419048
Step 465, loss: 0.010128077119588852
Step 466, loss: 0.008682378567755222
Step 467, loss: 0.008810698054730892
Step 468, loss: 0.009016426280140877
Step 469, loss: 0.00868071150034666
Step 470, loss: 0.008902246132493019
Step 471, loss: 0.00858953595161438
Step 472, loss: 0.009066066704690456
Step 473, loss: 0.008939971216022968
Step 474, loss: 0.008642027154564857
Step 475, loss: 0.011356632225215435
Step 476, loss: 0.008728811517357826
Step 477, loss: 0.007195563986897469
Step 478, loss: 0.009304963052272797
Step 479, loss: 0.008956977166235447
Step 480, loss: 0.008378159254789352
Step 481, loss: 0.010088315233588219
Step 482, loss: 0.008494030684232712
Step 483, loss: 0.009125368669629097
Step 484, loss: 0.008969607762992382
Step 485, loss: 0.009077407419681549
Step 486, loss: 0.007636579684913158
Step 487, loss: 0.008236422203481197
Step 488, loss: 0.008791486732661724
Step 489, loss: 0.00945952720940113
Step 490, loss: 0.008376586250960827
Step 491, loss: 0.007897509261965752
Step 492, loss: 0.009305316023528576
Step 493, loss: 0.00865597277879715
Step 494, loss: 0.008607441559433937
Step 495, loss: 0.008625464513897896
Step 496, loss: 0.008877462707459927
Step 497, loss: 0.008101128041744232
Step 498, loss: 0.00966381374746561
Step 499, loss: 0.008945940062403679
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 159, in wait_for_base_dir_creation
    await asyncio.wait_for(
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    await asyncio.sleep(0.25)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 605, in sleep
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 275, in <module>
    return await future
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    orbax_checkpointer.save(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/checkpointer.py", line 255, in save
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 162, in _maybe_create_base_dir
    self._handler.save(tmpdir.get(), args=ckpt_args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/pytree_checkpoint_handler.py", line 594, in save
    await asyncio.wait_for(
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 275, in <module>
    self._handler_impl.save(directory, args=args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 554, in save
    orbax_checkpointer.save(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/checkpointer.py", line 255, in save
    asyncio_utils.run_sync(async_save(directory, *args, **kwargs))
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    self._handler.save(tmpdir.get(), args=ckpt_args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/pytree_checkpoint_handler.py", line 594, in save
    self._handler_impl.save(directory, args=args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 554, in save
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
Step 441, loss: 0.009179329499602318
Step 442, loss: 0.008932436816394329
Step 443, loss: 0.010333123616874218
Step 444, loss: 0.01019835751503706
Step 445, loss: 0.009480638429522514
Step 446, loss: 0.009658812545239925
Step 447, loss: 0.008269150741398335
Step 448, loss: 0.011066804639995098
Step 449, loss: 0.009733232669532299
Step 450, loss: 0.008351570926606655
Step 451, loss: 0.008620851673185825
Step 452, loss: 0.008884967304766178
Step 453, loss: 0.00846097432076931
Step 454, loss: 0.009176055900752544
Step 455, loss: 0.009038573130965233
Step 456, loss: 0.008845883421599865
Step 457, loss: 0.008126926608383656
Step 458, loss: 0.009394383989274502
Step 459, loss: 0.008409657515585423
Step 460, loss: 0.009345205500721931
Step 461, loss: 0.008491947315633297
Step 462, loss: 0.0075165280140936375
Step 463, loss: 0.00986082199960947
Step 464, loss: 0.008375565521419048
Step 465, loss: 0.010128077119588852
Step 466, loss: 0.008682378567755222
Step 467, loss: 0.008810698054730892
Step 468, loss: 0.009016426280140877
Step 469, loss: 0.00868071150034666
Step 470, loss: 0.008902246132493019
Step 471, loss: 0.00858953595161438
Step 472, loss: 0.009066066704690456
Step 473, loss: 0.008939971216022968
Step 474, loss: 0.008642027154564857
Step 475, loss: 0.011356632225215435
Step 476, loss: 0.008728811517357826
Step 477, loss: 0.007195563986897469
Step 478, loss: 0.009304963052272797
Step 479, loss: 0.008956977166235447
Step 480, loss: 0.008378159254789352
Step 481, loss: 0.010088315233588219
Step 482, loss: 0.008494030684232712
Step 483, loss: 0.009125368669629097
Step 484, loss: 0.008969607762992382
Step 485, loss: 0.009077407419681549
Step 486, loss: 0.007636579684913158
Step 487, loss: 0.008236422203481197
Step 488, loss: 0.008791486732661724
Step 489, loss: 0.00945952720940113
Step 490, loss: 0.008376586250960827
Step 491, loss: 0.007897509261965752
Step 492, loss: 0.009305316023528576
Step 493, loss: 0.00865597277879715
Step 494, loss: 0.008607441559433937
Step 495, loss: 0.008625464513897896
Step 496, loss: 0.008877462707459927
Step 497, loss: 0.008101128041744232
Step 498, loss: 0.00966381374746561
Step 499, loss: 0.008945940062403679
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 159, in wait_for_base_dir_creation
    asyncio_utils.run_sync(async_save(directory, *args, **kwargs))
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    await asyncio.sleep(0.25)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 605, in sleep
    return await future
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 162, in _maybe_create_base_dir
    await asyncio.wait_for(
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 275, in <module>
    orbax_checkpointer.save(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/checkpointer.py", line 255, in save
    self._handler.save(tmpdir.get(), args=ckpt_args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/pytree_checkpoint_handler.py", line 594, in save
    self._handler_impl.save(directory, args=args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 554, in save
    asyncio_utils.run_sync(async_save(directory, *args, **kwargs))
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 552, in async_save
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 552, in async_save
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 552, in async_save
    f.result()  # Block on result.
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 174, in result
    f.result()  # Block on result.
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 174, in result
    f.result()  # Block on result.
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 174, in result
    f.result(timeout=time_remaining)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 397, in result
    f.result(timeout=time_remaining)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 397, in result
    f.result(timeout=time_remaining)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 397, in result
    return self._f.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 348, in result
    return self._f.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 348, in result
    return self._t.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 297, in result
    return self._t.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 297, in result
    return self._f.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 348, in result
    return self._t.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 297, in result
    self.join(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 294, in join
    self.join(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 294, in join
    self.join(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 294, in join
    raise self._exception
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 281, in run
    raise self._exception
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 281, in run
    raise self._exception
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 281, in run
    super().run()
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/threading.py", line 953, in run
    super().run()
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/threading.py", line 953, in run
    super().run()
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 230, in _target_setting_result
    self._result = target()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 341, in <lambda>
    target=lambda: asyncio_utils.run_sync(coro),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    self._target(*self._args, **self._kwargs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 230, in _target_setting_result
    self._result = target()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 341, in <lambda>
    self._target(*self._args, **self._kwargs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 230, in _target_setting_result
    self._result = target()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 341, in <lambda>
    target=lambda: asyncio_utils.run_sync(coro),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    target=lambda: asyncio_utils.run_sync(coro),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py", line 1105, in _background_serialize
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py", line 1105, in _background_serialize
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    await asyncio.gather(*write_coros)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 194, in write
    await self._maybe_create_base_dir(file_path.parent)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 169, in _maybe_create_base_dir
    await asyncio.gather(*write_coros)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 194, in write
    await self._maybe_create_base_dir(file_path.parent)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 169, in _maybe_create_base_dir
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py", line 1105, in _background_serialize
    raise ValueError(
ValueError: [process_index=5] Timed out waiting for array_metadatas base directory creation: /home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/train_lam_model_size_scaling_38M/3317231/lam_1751625282_500.orbax-checkpoint-tmp-0/array_metadatas. timeout=600 seconds. primary_process=0
    raise ValueError(
ValueError: [process_index=6] Timed out waiting for array_metadatas base directory creation: /home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/train_lam_model_size_scaling_38M/3317231/lam_1751625282_500.orbax-checkpoint-tmp-0/array_metadatas. timeout=600 seconds. primary_process=0
    await asyncio.gather(*write_coros)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 194, in write
    await self._maybe_create_base_dir(file_path.parent)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 169, in _maybe_create_base_dir
    raise ValueError(
ValueError: [process_index=4] Timed out waiting for array_metadatas base directory creation: /home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/train_lam_model_size_scaling_38M/3317231/lam_1751625282_500.orbax-checkpoint-tmp-0/array_metadatas. timeout=600 seconds. primary_process=0
Step 441, loss: 0.009179329499602318
Step 442, loss: 0.008932436816394329
Step 443, loss: 0.010333123616874218
Step 444, loss: 0.01019835751503706
Step 445, loss: 0.009480638429522514
Step 446, loss: 0.009658812545239925
Step 447, loss: 0.008269150741398335
Step 448, loss: 0.011066804639995098
Step 449, loss: 0.009733232669532299
Step 450, loss: 0.008351570926606655
Step 451, loss: 0.008620851673185825
Step 452, loss: 0.008884967304766178
Step 453, loss: 0.00846097432076931
Step 454, loss: 0.009176055900752544
Step 455, loss: 0.009038573130965233
Step 456, loss: 0.008845883421599865
Step 457, loss: 0.008126926608383656
Step 458, loss: 0.009394383989274502
Step 459, loss: 0.008409657515585423
Step 460, loss: 0.009345205500721931
Step 461, loss: 0.008491947315633297
Step 462, loss: 0.0075165280140936375
Step 463, loss: 0.00986082199960947
Step 464, loss: 0.008375565521419048
Step 465, loss: 0.010128077119588852
Step 466, loss: 0.008682378567755222
Step 467, loss: 0.008810698054730892
Step 468, loss: 0.009016426280140877
Step 469, loss: 0.00868071150034666
Step 470, loss: 0.008902246132493019
Step 471, loss: 0.00858953595161438
Step 472, loss: 0.009066066704690456
Step 473, loss: 0.008939971216022968
Step 474, loss: 0.008642027154564857
Step 475, loss: 0.011356632225215435
Step 476, loss: 0.008728811517357826
Step 477, loss: 0.007195563986897469
Step 478, loss: 0.009304963052272797
Step 479, loss: 0.008956977166235447
Step 480, loss: 0.008378159254789352
Step 481, loss: 0.010088315233588219
Step 482, loss: 0.008494030684232712
Step 483, loss: 0.009125368669629097
Step 484, loss: 0.008969607762992382
Step 485, loss: 0.009077407419681549
Step 486, loss: 0.007636579684913158
Step 487, loss: 0.008236422203481197
Step 488, loss: 0.008791486732661724
Step 489, loss: 0.00945952720940113
Step 490, loss: 0.008376586250960827
Step 491, loss: 0.007897509261965752
Step 492, loss: 0.009305316023528576
Step 493, loss: 0.00865597277879715
Step 494, loss: 0.008607441559433937
Step 495, loss: 0.008625464513897896
Step 496, loss: 0.008877462707459927
Step 497, loss: 0.008101128041744232
Step 498, loss: 0.00966381374746561
Step 499, loss: 0.008945940062403679
Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 159, in wait_for_base_dir_creation
    await asyncio.sleep(0.25)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 605, in sleep
    return await future
asyncio.exceptions.CancelledError

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 456, in wait_for
    return fut.result()
asyncio.exceptions.CancelledError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 162, in _maybe_create_base_dir
    await asyncio.wait_for(
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/tasks.py", line 458, in wait_for
    raise exceptions.TimeoutError() from exc
asyncio.exceptions.TimeoutError

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar_jobs/train_lam.py", line 275, in <module>
    orbax_checkpointer.save(
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/checkpointers/checkpointer.py", line 255, in save
    self._handler.save(tmpdir.get(), args=ckpt_args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/pytree_checkpoint_handler.py", line 594, in save
    self._handler_impl.save(directory, args=args)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 554, in save
    asyncio_utils.run_sync(async_save(directory, *args, **kwargs))
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/handlers/base_pytree_checkpoint_handler.py", line 552, in async_save
    f.result()  # Block on result.
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 174, in result
    f.result(timeout=time_remaining)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 397, in result
    return self._f.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 348, in result
    return self._t.result(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 297, in result
    self.join(timeout=timeout)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 294, in join
    raise self._exception
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 281, in run
    super().run()
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/threading.py", line 953, in run
    self._target(*self._args, **self._kwargs)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 230, in _target_setting_result
    self._result = target()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/futures/future.py", line 341, in <lambda>
    target=lambda: asyncio_utils.run_sync(coro),
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/asyncio_utils.py", line 50, in run_sync
    return asyncio.run(coro)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/runners.py", line 44, in run
    return loop.run_until_complete(main)
  File "/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/serialization/type_handlers.py", line 1105, in _background_serialize
    await asyncio.gather(*write_coros)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 194, in write
    await self._maybe_create_base_dir(file_path.parent)
  File "/hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jafar/.venv/lib/python3.10/site-packages/orbax/checkpoint/_src/metadata/array_metadata_store.py", line 169, in _maybe_create_base_dir
    raise ValueError(
ValueError: [process_index=7] Timed out waiting for array_metadatas base directory creation: /home/hk-project-p0023960/tum_cte0515/Projects/checkpoints/train_lam_model_size_scaling_38M/3317231/lam_1751625282_500.orbax-checkpoint-tmp-0/array_metadatas. timeout=600 seconds. primary_process=0
2025-07-04 12:55:25.377014: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1689] Shutdown barrier in coordination service has failed:
DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']
This suggests that the workers are out of sync. Either at least one worker (a) crashed early due to program error or scheduler events (e.g. preemption, eviction), (b) was too fast in its execution, or (c) too slow / hanging. Check the logs (both the program and scheduler events) for an earlier error to identify the root cause.
2025-07-04 12:55:25.377040: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service.cc:1730] Use error polling to propagate the following error to all tasks: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError=''] [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-07-04 12:55:25.377344: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
2025-07-04 12:55:25.377478: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
2025-07-04 12:55:25.377313: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
2025-07-04 12:55:25.377456: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-07-04 12:55:25.377520: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
2025-07-04 12:55:25.377310: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
2025-07-04 12:55:25.377523: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
2025-07-04 12:55:25.377539: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-07-04 12:55:25.377641: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

2025-07-04 12:55:25.377701: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-07-04 12:55:25.377505: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
2025-07-04 12:55:25.377929: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
2025-07-04 12:55:25.377671: E external/xla/xla/tsl/distributed_runtime/coordination/coordination_service_agent.cc:427] Polled an error from coordination service (this can be an error from this or another task).
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-07-04 12:55:25.377973: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

2025-07-04 12:55:25.378265: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
2025-07-04 12:55:25.378118: F external/xla/xla/pjrt/distributed/client.h:80] Terminating process because the JAX distributed service detected fatal errors. This most likely indicates that another task died; see the other task logs for more details. Disable Python buffering, i.e. `python -u`, to be sure to see all the previous output. absl::Status: INTERNAL: Shutdown barrier has failed.
Barrier result: 'DEADLINE_EXCEEDED: Barrier timed out. Id: Shutdown::14727415199572219422::0. This usually happens because a task triggered the barrier too early or too slowly. Please look at the task logs (both timed out and first task) to debug further.
# of tasks that reached the barrier: 4/8.
The first task at the barrier: /job:jax_worker/replica:0/task:6. Some timed out task names:
/job:jax_worker/replica:0/task:3
/job:jax_worker/replica:0/task:1
 [type.googleapis.com/tensorflow.BarrierError='
\x1eShutdown::14727415199572219422'] [type.googleapis.com/tensorflow.CoordinationServiceError='']

RPC: /tensorflow.CoordinationService/PollForError [type.googleapis.com/tensorflow.CoordinationServiceError='']
srun: error: hkn0407: tasks 0,2: Aborted (core dumped)
srun: error: hkn0413: tasks 4-5: Aborted (core dumped)
srun: error: hkn0407: task 3: Aborted (core dumped)
srun: error: hkn0413: task 7: Aborted (core dumped)
srun: error: hkn0413: task 6: Aborted (core dumped)
srun: error: hkn0407: task 1: Aborted (core dumped)

============================= JOB FEEDBACK =============================

Job ID: 3317231
Cluster: hk
User/Group: tum_cte0515/hk-project-p0023960
Account: hk-project-p0023960
State: FAILED (exit code 134)
Partition: accelerated
Nodes: 2
Cores per node: 24
Nodelist: hkn[0407,0413]
CPU Utilized: 02:21:28
CPU Efficiency: 13.61% of 17:19:12 core-walltime
Job Wall-clock time: 00:21:39
Starttime: Fri Jul  4 12:33:48 2025
Endtime: Fri Jul  4 12:55:27 2025
Memory Utilized: 91.02 GB (estimated maximum)
Memory Efficiency: 0.00% of 0.00 MB (0.00 MB/node)
Energy Consumed: 5092945 Joule / 1414.70694444444 Watthours
Average node power draw: 3920.66589684373 Watt