[terminal_command] TERMINAL ()
[tum_cte0515@hkn0403 jasmine]$ source .venv/bin/acceleratebash: .venv/bin/accelerate: No such file or directory[tum_cte0515@hkn0403 jasmine]$ source .venv/bin/activate(jasmine) [tum_cte0515@hkn0403 jasmine]$ sh slurm/jobs/mihir/horeka/coinrun/train_tokenizer_single_gpu.sh#!/usr/bin/env bash#SBATCH --nodes=1#SBATCH --ntasks-per-node=4#SBATCH --time=00:20:00#SBATCH --partition=dev_accelerated#SBATCH --cpus-per-task=5#SBATCH --gres=gpu:1#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizer/%x_%j.log#SBATCH --job-name=train_tokenizer_1e-4#SBATCH --requeue#SBATCH --signal=b:usr1@300 # 5 min before timeout# Log the sbatch scriptcat $0module unload mpi/openmpi/5.0module unload devel/cuda/12.4source .venv/bin/activatearray_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_test# array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/train# array_records_dir_train=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_chunkedarray_records_dir_val=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_10m_gt_actions_split/valjob_name=$SLURM_JOB_NAMEslurm_job_id=$SLURM_JOB_IDCHECKPOINT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/checkpoints/big-runs/tokenizer/$job_name/$slurm_job_idmkdir -p $CHECKPOINT_DIRenv | grep SLURMsrun python train_tokenizer.py \    --save_ckpt \    --image_height=64 \    --image_width=64 \    --ckpt_dir $CHECKPOINT_DIR \    --batch_size=120 \    --init_lr=0 \    --max_lr=1e-4 \    --log_image_interval=250 \    --log_checkpoint_interval=250 \    --log \    --name=coinrun-tokenizer-dataset-test-$slurm_job_id \    --tags tokenizer coinrun dev \    --entity instant-uv \    --project jafar \    --warmup_steps 0 \    --wsd_decay_steps 0 \    --num_steps 250 \    --data_dir $array_records_dir_trainSLURM_STEP_NUM_TASKS=1SLURM_JOB_USER=tum_cte0515SLURM_TASKS_PER_NODE=1SLURM_JOB_UID=999226SLURM_TASK_PID=67647SLURM_JOB_GPUS=0SLURM_LOCALID=0SLURM_SUBMIT_DIR=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/coinrun/tokenizerSLURMD_NODENAME=hkn0403SLURM_JOB_START_TIME=1757499919SLURM_STEP_NODELIST=hkn0403SLURM_CLUSTER_NAME=hkSLURM_JOB_END_TIME=1757503519SLURM_PMI2_SRUN_PORT=32775SLURM_CPUS_ON_NODE=8SLURM_JOB_CPUS_PER_NODE=8SLURM_GPUS_ON_NODE=1SLURM_GTIDS=0SLURM_JOB_PARTITION=dev_acceleratedSLURM_TRES_PER_TASK=cpu=8SLURM_OOM_KILL_STEP=0SLURM_JOB_NUM_NODES=1SLURM_STEPID=4294967290SLURM_JOBID=3482653SLURM_PTY_PORT=44507SLURM_JOB_QOS=normalSLURM_LAUNCH_NODE_IPADDR=10.0.7.201SLURM_PTY_WIN_ROW=32SLURM_PMI2_PROC_MAPPING=(vector,(0,1,1))SLURMD_DEBUG=2SLURM_PROCID=0SLURM_CPUS_PER_TASK=8SLURM_TOPOLOGY_ADDR=hkibb.hkibbi1.hkibbi1e11.hkn0403SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.switch.nodeSLURM_SRUN_COMM_HOST=10.0.7.201SLURM_SCRIPT_CONTEXT=prolog_taskSLURM_PTY_WIN_COL=129SLURM_NODELIST=hkn0403SLURM_SRUN_COMM_PORT=43229SLURM_STEP_ID=4294967290SLURM_JOB_ACCOUNT=hk-project-p0023960SLURM_PRIO_PROCESS=0SLURM_NNODES=1SLURM_SUBMIT_HOST=hkn1993.localdomainSLURM_JOB_ID=3482653SLURM_NODEID=0SLURM_STEP_NUM_NODES=1SLURM_STEP_TASKS_PER_NODE=1SLURM_MPI_TYPE=pmi2SLURM_PMI2_STEP_NODES=hkn0403SLURM_CONF=/etc/slurm/slurm.confSLURM_JOB_NAME=interactiveSLURM_STEP_LAUNCHER_PORT=43229SLURM_JOB_GID=502226SLURM_JOB_NODELIST=hkn0403GpuFreq=control_disabledwandb: Currently logged in as: mihir-mahajan2002 (instant-uv) to https://api.wandb.ai. Use `wandb login --relogin` to force reloginwandb: creating runwandb: Tracking run with wandb version 0.21.3wandb: Run data is saved locally in /hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/wandb/run-20250910_122628-qot85m4uwandb: Run `wandb offline` to turn off syncing.wandb: Syncing run coinrun-tokenizer-dataset-test-3482653wandb: ⭐️ View project at https://wandb.ai/instant-uv/jafarwandb: 🚀 View run at https://wandb.ai/instant-uv/jafar/runs/qot85m4uRunning on 1 devices.Counting all components: ['decoder', 'encoder', 'vq']Parameter counts:{'decoder': 16858736, 'encoder': 16858752, 'vq': 32768, 'total': 33750256}Starting training from step 0...Step 0, loss: 0.25492268800735474Step 1, loss: 0.23155003786087036Step 2, loss: 0.21997617185115814Step 3, loss: 0.19638122618198395Step 4, loss: 0.18517403304576874Step 5, loss: 0.1829017549753189Step 6, loss: 0.18066121637821198Step 7, loss: 0.16971272230148315Step 8, loss: 0.15260319411754608Step 9, loss: 0.151905819773674Step 10, loss: 0.15158188343048096Step 11, loss: 0.14474010467529297Step 12, loss: 0.13934583961963654Step 13, loss: 0.12601177394390106Step 14, loss: 0.12056466937065125Step 15, loss: 0.12011177092790604Step 16, loss: 0.11393854767084122Step 17, loss: 0.10769293457269669Step 18, loss: 0.11575568467378616Step 19, loss: 0.11627855896949768Step 20, loss: 0.11476963758468628Step 21, loss: 0.11665616184473038Step 22, loss: 0.10657517611980438Step 23, loss: 0.10253696143627167Step 24, loss: 0.09440644085407257Step 25, loss: 0.09521863609552383Step 26, loss: 0.0916718989610672Step 27, loss: 0.09089316427707672Step 28, loss: 0.09240813553333282Step 29, loss: 0.09195040911436081Step 30, loss: 0.0861515998840332Step 31, loss: 0.0765690803527832Step 32, loss: 0.08566474169492722Step 33, loss: 0.08110545575618744Step 34, loss: 0.08845805376768112Step 35, loss: 0.08059680461883545Step 36, loss: 0.08095452934503555Step 37, loss: 0.084492027759552Step 38, loss: 0.08701610565185547Step 39, loss: 0.07902377098798752Step 40, loss: 0.08127189427614212Step 41, loss: 0.08010697364807129Step 42, loss: 0.07532629370689392Step 43, loss: 0.07829190045595169Step 44, loss: 0.07659339904785156Step 45, loss: 0.07268880307674408Step 46, loss: 0.07575363665819168Step 47, loss: 0.076294906437397Step 48, loss: 0.07010173052549362Step 49, loss: 0.06656155735254288Step 50, loss: 0.07291295379400253Step 51, loss: 0.06674183905124664Step 52, loss: 0.07309184968471527Step 53, loss: 0.06621835380792618Step 54, loss: 0.06883115321397781Step 55, loss: 0.06711291521787643Step 56, loss: 0.06834907829761505Step 57, loss: 0.06370905786752701Step 58, loss: 0.06290154904127121Step 59, loss: 0.05990591645240784Step 60, loss: 0.060743048787117004Step 61, loss: 0.061962805688381195Step 62, loss: 0.05804363265633583Step 63, loss: 0.0585540235042572Step 64, loss: 0.0601714588701725Step 65, loss: 0.058881182223558426Step 66, loss: 0.06039590761065483Step 67, loss: 0.06332925707101822Step 68, loss: 0.05859855189919472Step 69, loss: 0.05876442790031433Step 70, loss: 0.05736853554844856Step 71, loss: 0.055592916905879974Step 72, loss: 0.05422021448612213Step 73, loss: 0.05880211666226387Step 74, loss: 0.05730080232024193Step 75, loss: 0.057205867022275925Step 76, loss: 0.057411033660173416Step 77, loss: 0.05906601995229721Step 78, loss: 0.05951167270541191Step 79, loss: 0.056288156658411026Step 80, loss: 0.05478360503911972Step 81, loss: 0.056506186723709106Step 82, loss: 0.05400453135371208Step 83, loss: 0.052197568118572235Step 84, loss: 0.053463149815797806Step 85, loss: 0.052063532173633575Step 86, loss: 0.05331426486372948Step 87, loss: 0.049770474433898926Step 88, loss: 0.04975126311182976Step 89, loss: 0.04965030401945114Step 90, loss: 0.049226582050323486Step 91, loss: 0.052000295370817184Step 92, loss: 0.04945254325866699Step 93, loss: 0.04968021810054779Step 94, loss: 0.04842541366815567Step 95, loss: 0.05083601176738739Step 96, loss: 0.04857897385954857Step 97, loss: 0.04600849002599716Step 98, loss: 0.04805907607078552Step 99, loss: 0.04669881612062454Step 100, loss: 0.04979506507515907Step 101, loss: 0.046441927552223206Step 102, loss: 0.04292256757616997Step 103, loss: 0.04762355610728264Step 104, loss: 0.041822608560323715Step 105, loss: 0.04520750045776367Step 106, loss: 0.04296313226222992Step 107, loss: 0.04390386492013931Step 108, loss: 0.041791364550590515Step 109, loss: 0.039953991770744324Step 110, loss: 0.04232088103890419Step 111, loss: 0.04248344898223877Step 112, loss: 0.04207051172852516Step 113, loss: 0.03938259184360504Step 114, loss: 0.039665739983320236Step 115, loss: 0.0393700897693634Step 116, loss: 0.04036534205079079Step 117, loss: 0.03720397502183914Step 118, loss: 0.03542076796293259Step 119, loss: 0.03688682243227959Step 120, loss: 0.03782087564468384Step 121, loss: 0.038084086030721664Step 122, loss: 0.03957195580005646Step 123, loss: 0.03905578702688217Step 124, loss: 0.0378570631146431Step 125, loss: 0.0370800644159317Step 126, loss: 0.03965211659669876Step 127, loss: 0.03981009125709534Step 128, loss: 0.04012076556682587Step 129, loss: 0.040115244686603546Step 130, loss: 0.0382639579474926Step 131, loss: 0.039118774235248566Step 132, loss: 0.039002273231744766Step 133, loss: 0.03909499570727348Step 134, loss: 0.03646230325102806Step 135, loss: 0.04100118577480316Step 136, loss: 0.03662954270839691Step 137, loss: 0.034403588622808456Step 138, loss: 0.036315180361270905Step 139, loss: 0.03457498177886009Step 140, loss: 0.03585319221019745Step 141, loss: 0.036204028874635696Step 142, loss: 0.036179814487695694Step 143, loss: 0.03851379081606865Step 144, loss: 0.036285944283008575Step 145, loss: 0.03762957453727722Step 146, loss: 0.03457581251859665Step 147, loss: 0.03525669872760773Step 148, loss: 0.035004399716854095Step 149, loss: 0.034588370472192764Step 150, loss: 0.03630198538303375Step 151, loss: 0.035807982087135315Step 152, loss: 0.03583860397338867Step 153, loss: 0.036339547485113144Step 154, loss: 0.03647119179368019Step 155, loss: 0.03615832328796387Step 156, loss: 0.03480179235339165Step 157, loss: 0.036498136818408966Step 158, loss: 0.03384288400411606Step 159, loss: 0.03347717598080635Step 160, loss: 0.03399645909667015Step 161, loss: 0.032946132123470306Step 162, loss: 0.035169415175914764Step 163, loss: 0.032750651240348816Step 164, loss: 0.032805103808641434Step 165, loss: 0.03374134749174118Step 166, loss: 0.03438092768192291Step 167, loss: 0.03546205163002014Step 168, loss: 0.033540062606334686Step 169, loss: 0.03291282057762146Step 170, loss: 0.03323889896273613Step 171, loss: 0.03241699934005737Step 172, loss: 0.030643437057733536Step 173, loss: 0.032026149332523346Step 174, loss: 0.03137095645070076Step 175, loss: 0.03174600005149841Step 176, loss: 0.03141679987311363Step 177, loss: 0.03203446418046951Step 178, loss: 0.03206599876284599Step 179, loss: 0.03259724751114845Step 180, loss: 0.03236614912748337Step 181, loss: 0.031277600675821304Step 182, loss: 0.032317377626895905Step 183, loss: 0.030612919479608536Step 184, loss: 0.032499559223651886Step 185, loss: 0.029568273574113846Step 186, loss: 0.03130722418427467Step 187, loss: 0.03257034718990326Step 188, loss: 0.03242553025484085Step 189, loss: 0.031610120087862015Step 190, loss: 0.031977057456970215Step 191, loss: 0.031235316768288612Step 192, loss: 0.030282776802778244Step 193, loss: 0.031246397644281387Step 194, loss: 0.030697792768478394Step 195, loss: 0.02954351156949997Step 196, loss: 0.02890462428331375Step 197, loss: 0.029092837125062943Step 198, loss: 0.028627369552850723Step 199, loss: 0.03028913587331772Step 200, loss: 0.029435381293296814Step 201, loss: 0.02838410995900631Step 202, loss: 0.02816847525537014Step 203, loss: 0.028849242255091667Step 204, loss: 0.03060498647391796Step 205, loss: 0.02901904098689556Step 206, loss: 0.028475714847445488Step 207, loss: 0.027972152456641197Step 208, loss: 0.028789587318897247Step 209, loss: 0.02874019928276539Step 210, loss: 0.027473898604512215Step 211, loss: 0.026478031650185585Step 212, loss: 0.027614528313279152Step 213, loss: 0.027129311114549637Step 214, loss: 0.02819584123790264Step 215, loss: 0.027149518951773643Step 216, loss: 0.02642691321671009Step 217, loss: 0.027675848454236984Step 218, loss: 0.02863454446196556Step 219, loss: 0.027130628004670143Step 220, loss: 0.02811211720108986Step 221, loss: 0.027117958292365074Step 222, loss: 0.02845187671482563Step 223, loss: 0.02720900997519493Step 224, loss: 0.026743559166789055Step 225, loss: 0.0274973101913929Step 226, loss: 0.028546782210469246Step 227, loss: 0.026835495606064796Step 228, loss: 0.028459055349230766Step 229, loss: 0.02936214953660965Step 230, loss: 0.028209706768393517Step 231, loss: 0.02661646343767643Step 232, loss: 0.026257595047354698Step 233, loss: 0.026068981736898422Step 234, loss: 0.026419732719659805Step 235, loss: 0.02645086869597435Step 236, loss: 0.027382217347621918Step 237, loss: 0.025820616632699966Step 238, loss: 0.02517794258892536Step 239, loss: 0.02623983472585678Step 240, loss: 0.02619650959968567Step 241, loss: 0.027855759486556053Step 242, loss: 0.027149591594934464Step 243, loss: 0.027810823172330856Step 244, loss: 0.027006570249795914Step 245, loss: 0.027746018022298813Step 246, loss: 0.029927274212241173Step 247, loss: 0.029070468619465828Step 248, loss: 0.027839085087180138Step 249, loss: 0.02754037082195282Saved checkpoint at step 250wandb: wandb: 🚀 View run coinrun-tokenizer-dataset-test-3482653 at: https://wandb.ai/instant-uv/jafar/runs/qot85m4uwandb: Find logs at: ../../../../../hkfs/home/project/hk-project-p0023960/tum_cte0515/Projects/jasmine/wandb/run-20250910_122628-qot85m4u/logs/home/hk-project-p0023960/tum_cte0515/.local/share/uv/python/cpython-3.10.18-linux-x86_64-gnu/lib/python3.10/multiprocessing/resource_tracker.py:224: UserWarning: resource_tracker: There appear to be 10 leaked shared_memory objects to clean up at shutdown  warnings.warn('resource_tracker: There appear to be %d '(jasmine) [tum_cte0515@hkn0403 jasmine]$ git branch  action-mapper  add-wandb-name-and-tags  before-nnx  causal-mem-reduce  causal-spatiotemporal-kv-cache  causal-st-transformer  causal-transformer-dynamics-model  causal-transformer-nnx-no-kv-cache  coinrun-gt-actions  convert-to-jax-array-in-iter  correct-batched-sampling  dev  dont-let-tf-see-gpu  feat/darkness-filter  feat/explicit-image-dims  fix-action-padding-lam-future-information-access  fix-sampling  fix-transformer-forwardpass  fix/spatiotemporal-pe-once-in-STTransformer  grad-norm-log-and-clip  grain-dataloader* input_pipeline/add-npy2array_record  logging-variants  lr-schedules  main  maskgit-different-maskprob-per-sample  maskgit-sampling-iterative-unmasking-fix  metrics-logging-for-dynamics-model  monkey-patch  new-arch-sampling  preprocess_video(jasmine) [tum_cte0515@hkn0403 jasmine]$ git diff(jasmine) [tum_cte0515@hkn0403 jasmine]$ git checkout mainSwitched to branch 'main'Your branch is behind 'origin/main' by 3 commits, and can be fast-forwarded.  (use "git pull" to update your local branch)(jasmine) [tum_cte0515@hkn0403 jasmine]$ git pullremote: Enumerating objects: 3, done.remote: Counting objects: 100% (3/3), done.remote: Compressing objects: 100% (3/3), done.remote: Total 3 (delta 0), reused 0 (delta 0), pack-reused 0 (from 0)Unpacking objects: 100% (3/3), 2.81 KiB | 29.00 KiB/s, done.From github.com:p-doom/jasmine   deca2a6..69f1f75  input_pipeline/add-npy2array_record -> origin/input_pipeline/add-npy2array_recordUpdating bbef694..19c4b89Fast-forward README.md     |  1 + models/lam.py |  4 ++-- utils/nn.py   | 11 +++++------ 3 files changed, 8 insertions(+), 8 deletions(-)(jasmine) [tum_cte0515@hkn0403 jasmine]$ git branch -d tmperror: the branch 'tmp' is not fully merged.If you are sure you want to delete it, run 'git branch -D tmp'(jasmine) [tum_cte0515@hkn0403 jasmine]$ git branch -D tmpDeleted branch tmp (was 38e487c).(jasmine) [tum_cte0515@hkn0403 jasmine]$ git checkout input_pipeline/add-npy2array_recordSwitched to branch 'input_pipeline/add-npy2array_record'Your branch is behind 'origin/input_pipeline/add-npy2array_record' by 1 commit, and can be fast-forwarded.  (use "git pull" to update your local branch)(jasmine) [tum_cte0515@hkn0403 jasmine]$ git pullUpdating deca2a6..69f1f75Fast-forward generate_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)(jasmine) [tum_cte0515@hkn0403 jasmine]$ git branch  action-mapper  add-wandb-name-and-tags  before-nnx  causal-mem-reduce  causal-spatiotemporal-kv-cache  causal-st-transformer  causal-transformer-dynamics-model  causal-transformer-nnx-no-kv-cache  coinrun-gt-actions  convert-to-jax-array-in-iter  correct-batched-sampling  dev  dont-let-tf-see-gpu  feat/darkness-filter  feat/explicit-image-dims  fix-action-padding-lam-future-information-access  fix-sampling  fix-transformer-forwardpass  fix/spatiotemporal-pe-once-in-STTransformer  grad-norm-log-and-clip  grain-dataloader* input_pipeline/add-npy2array_record  logging-variants  lr-schedules  main  maskgit-different-maskprob-per-sample  maskgit-sampling-iterative-unmasking-fix  metrics-logging-for-dynamics-model  monkey-patch  new-arch-sampling  preprocess_video  refactor-tmp  revised-dataloader  runner  runner-grain(jasmine) [tum_cte0515@hkn0403 jasmine]$ git checkout validation-lossSwitched to branch 'validation-loss'Your branch is up to date with 'origin/validation-loss'.(jasmine) [tum_cte0515@hkn0403 jasmine]$ git pullAlready up to date.(jasmine) [tum_cte0515@hkn0403 jasmine]$ git merge input_pipeline/add-npy2array_recordAuto-merging generate_dataset.pyCONFLICT (content): Merge conflict in generate_dataset.pyAutomatic merge failed; fix conflicts and then commit the result.(jasmine) [tum_cte0515@hkn0403 jasmine]$ salloc: Job 3482653 has exceeded its time limit and its allocation has been revoked.                                     srun: Job step aborted: Waiting up to 32 seconds for job step to finish.slurmstepd: error: *** STEP 3482653.interactive ON hkn0403 CANCELLED AT 2025-09-10T13:25:49 DUE TO TIME LIMIT ***srun: error: hkn0403: task 0: Killed