[tab] notes.md (markdown)
hk-project-p0023960



how do I move a lot of files from one directory to another?

src: /hkfs/work/workspace/scratch/tum_ind3695-jafar_ws
dst: /hkfs/work/workspace/scratch/tum_ind3695-jafar_data



DIR=$(ws_find jafar_data)
setfacl -Rm g:hk-project-p0023960:rX,d:g:hk-project-p0023960:rX $DIR

DIR=$(ws_find jafar_workspace)


DIR=$(ws_find jafar_workspace)
getfacl $DIR

getfacl $(ws_find jafar_workspace)


setfacl -Rm g:hk-project-p0023960:rX,d:g:hk-project-p0023960:rX $DIR

ws_access_ls='getfacl $(ws_find jafar_workspace)'
ws_access_granter='setfacl -Rm g:hk-project-p0023960:rX,d:g:hk-project-p0023960:rX $DIR'

hk-project-p0023960


cp 

cp data_gen_gym_coinrun.sbatch data_gen_gym_bigfish.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_bossfight.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_caveflyer.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_chaser.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_climber.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_dodgeball.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_fruitbot.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_heist.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_jumper.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_leaper.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_maze.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_miner.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_ninja.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_plunder.sbatch
cp data_gen_gym_coinrun.sbatch data_gen_gym_starpilot.sbatch




curl -O https://omnomnom.vision.rwth-aachen.de/data/atari_v1_release/spaceinvaders.tar.gz
curl -O https://omnomnom.vision.rwth-aachen.de/data/atari_v1_release/qbert.tar.gz
curl -O https://omnomnom.vision.rwth-aachen.de/data/atari_v1_release/mspacman.tar.gz
curl -O https://omnomnom.vision.rwth-aachen.de/data/atari_v1_release/pinball.tar.gz
curl -O https://omnomnom.vision.rwth-aachen.de/data/atari_v1_release/revenge.tar.gz




## mp4 to jafar format

- make it k fps
- crop it/downsample it
- chunk it into n frames
- convert to npy
- save it to data/atari_v1_release/spaceinvaders/spaceinvaders.npy

```bash
sbatch --time=04:00:00 --partition=cpuonly --account=hk-project-pai00039 --cpus-per-task=64 --ntasks=1 --job-name=aws_download --wrap="aws s3 cp s3://minescrape-bucket/videos/knoms/ . --recursive"

python utils/preprocess_dataset.py --source_data_dir /hkfs/work/workspace/scratch/tum_ind3695-jafar_ws/minecraft_videos/preprocessed/10fps_160x90/ --output_tfrecords_dir /hkfs/work/workspace/scratch/tum_ind3695-jafar_ws/minecraft_videos/preprocessed/10fps_160x90/tfrecords/ 

```
1,6b: 80 nodes


```
salloc --time=01:00:00 --partition=accelerated-h100 --nodes=1 --gres=gpu:4  --cpus-per-task=8 --ntasks-per-node=4
accelerated-h100

srun python train_tokenizer.py --data_dir /hkfs/work/workspace/scratch/tum_ind3695-jafar_ws/data/tf_records --image_height 160 --image_width 90

/hkfs/work/workspace/scratch/tum_ind3695-jafar_ws/data/knoms






```

{'streams': [{'index': 0, 'codec_name': 'vp9', 'codec_long_name': 'Google VP9', 'profile': 'Profile 0', 'codec_type': 'video', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'width': 640, 'height': 360, 'coded_width': 640, 'coded_height': 360, 'closed_captions': 0, 'film_grain': 0, 'has_b_frames': 0, 'sample_aspect_ratio': '1:1', 'display_aspect_ratio': '16:9', 'pix_fmt': 'yuv420p', 'level': -99, 'color_range': 'tv', 'color_space': 'bt709', 'color_transfer': 'bt709', 'color_primaries': 'bt709', 'refs': 1, 'r_frame_rate': '30/1', 'avg_frame_rate': '30/1', 'time_base': '1/1000', 'start_pts': 0, 'start_time': '0.000000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng'}}], 'format': {'filename': '/hkfs/work/workspace/scratch/tum_ind3695-jafar_ws/data/knoms/293_KxCeK_wk-KA.webm', 'nb_streams': 1, 'nb_programs': 0, 'nb_stream_groups': 0, 'format_name': 'matroska,webm', 'format_long_name': 'Matroska / WebM', 'start_time': '0.000000', 'duration': '1155.767000', 'size': '50776376', 'bit_rate': '351464', 'probe_score': 100, 'tags': {'encoder': 'google/video-file'}}}

(Pdb) 

{'index': 0, 'codec_name': 'vp9', 'codec_long_name': 'Google VP9', 'profile': 'Profile 0', 'codec_type': 'video', 'codec_tag_string': '[0][0][0][0]', 'codec_tag': '0x0000', 'width': 640, 'height': 360, 'coded_width': 640, 'coded_height': 360, 'closed_captions': 0, 'film_grain': 0, 'has_b_frames': 0, 'sample_aspect_ratio': '1:1', 'display_aspect_ratio': '16:9', 'pix_fmt': 'yuv420p', 'level': -99, 'color_range': 'tv', 'color_space': 'bt709', 'color_transfer': 'bt709', 'color_primaries': 'bt709', 'refs': 1, 'r_frame_rate': '30/1', 'avg_frame_rate': '30/1', 'time_base': '1/1000', 'start_pts': 0, 'start_time': '0.000000', 'disposition': {'default': 1, 'dub': 0, 'original': 0, 'comment': 0, 'lyrics': 0, 'karaoke': 0, 'forced': 0, 'hearing_impaired': 0, 'visual_impaired': 0, 'clean_effects': 0, 'attached_pic': 0, 'timed_thumbnails': 0, 'non_diegetic': 0, 'captions': 0, 'descriptions': 0, 'metadata': 0, 'dependent': 0, 'still_image': 0}, 'tags': {'language': 'eng'}}





uid=996262(tum_dbd0378) gid=502226(hk-project-p0023960) groups=502226(hk-project-p0023960),501163(hk-access-ext),501959(tum_tu


12min 39s
12min 54s
delta: 15s

12*60* 10 = 7200
39*10 = 390
sum: 7590

```
        3237740      preprocess_video_splitter          cpuonly 152    COMPLETED   00:47:47   01:30:00 
        3238237      preprocess_video_splitter          cpuonly 152      TIMEOUT   01:30:11   01:30:00 
        3238924 preprocess_video_splitter_cop+          cpuonly 152    COMPLETED   00:47:40   01:30:00 
        3239678        preprocess_video_to_npy          cpuonly 152      TIMEOUT   01:30:25   01:30:00 
        3251122        preprocess_video_to_npy          cpuonly 152    COMPLETED   00:00:28   01:30:00 
        3251189      rsync_knoms_npy_to_shared          cpuonly 152    COMPLETED   00:16:50   01:30:00 
        3251202        preprocess_video_to_npy          cpuonly 152    COMPLETED   00:04:39   01:30:00 
        3251494        preprocess_video_to_npy          cpuonly 152    COMPLETED   00:19:45   01:30:00 
        3255047          train_tokenizer_knoms      accelerated   0       FAILED   00:00:00   00:30:00 
        3255048          train_tokenizer_knoms      accelerated   0       FAILED   00:00:00   00:30:00 
        3255049          train_tokenizer_knoms      accelerated   0       FAILED   00:00:00   00:30:00 
        3255050          train_tokenizer_knoms      accelerated  32       FAILED   00:05:08   00:30:00 
        3255482          train_tokenizer_knoms accelerated-h100  32       FAILED   00:00:44   00:15:00 
        3255483          train_tokenizer_knoms accelerated-h100  32       FAILED   00:03:53   00:15:00 
        3255493          data_gen_coinrun-test          cpuonly 152      TIMEOUT   00:30:17   00:30:00 
        3255506          data_gen_coinrun-test          cpuonly 152      TIMEOUT   01:30:12   01:30:00 
        3256929        train_tokenizer_coinrun      accelerated  32       FAILED   00:01:46   01:00:00 
        3257751        train_tokenizer_coinrun      accelerated   0 CANCELLED b+   00:00:00   06:00:00 
        3257924  train_tokenizer_knoms_overfit      accelerated   0 CANCELLED b+   00:00:00   04:00:00 
        3258011  train_tokenizer_knoms_overfit      accelerated   0 CANCELLED b+   00:00:00   06:00:00 
        3258018        train_tokenizer_coinrun      accelerated   0 CANCELLED b+   00:00:00   06:00:00 
        3258057  train_tokenizer_knoms_overfit      accelerated   0 CANCELLED b+   00:00:00   06:00:00 
        3258278  train_tokenizer_knoms_overfit      accelerated  32 CANCELLED b+   01:54:35   06:00:00 
        3258283        train_tokenizer_coinrun      accelerated  32      TIMEOUT   06:00:19   06:00:00 
        3259422  train_tokenizer_knoms_overfit accelerated-h100  32      TIMEOUT   06:00:12   06:00:00 
```



```
00:47:47
01:30:11
00:47:40
01:30:25
00:00:28
00:16:50
00:04:39
00:19:45
00:00:00
00:00:00
00:00:00
00:05:08
00:00:44
00:03:53
00:30:17
01:30:12
00:01:46
00:00:00
00:00:00
00:00:00
00:00:00
00:00:00
01:54:35
06:00:19
06:00:12

```

{'encoder': 18.978.432, 'vq': 32.768, 'decoder': 18.978.416, 'total': 37.989.616}
{'encoder': 4.770.688, 'vq': 32.768, 'decoder': 4.770.672, 'total': 9.574.128}
{'encoder': 10.694.912, 'vq': 32.768, 'decoder': 10.694.896, 'total': 21.422.576}
{'encoder': 8029184, 'vq': 32768, 'decoder': 8029168, 'total': 16091120}


**BASELINE 37.9mio**
model_dim = 512
latent_dim = 32
num_latents = 1024
patch_size = 4
num_blocks = 8
num_heads = 8
{'encoder': 18.978.432, 'vq': 32.768, 'decoder': 18.978.416, 'total': 37.989.616}

**BASELINE 9.6mio**
model_dim = 256
latent_dim = 32
num_latents = 1024
patch_size = 4
num_blocks = 8
num_heads = 8
{'encoder': 18.978.432, 'vq': 32.768, 'decoder': 18.978.416, 'total': 37.989.616}



**2.4 mio**
model_dim = 128
latent_dim = 32
num_latents = 1024
patch_size = 4
num_blocks = 8
num_heads = 8

{'encoder': 1205760, 'vq': 32768, 'decoder': 1205744, 'total': 2.444.272}

**648k**
model_dim = 64
latent_dim = 32
num_latents = 1024
patch_size = 4
num_blocks = 8
num_heads = 8

{'encoder': 308032, 'vq': 32768, 'decoder': 308016, 'total': 648.816}



**179k**
model_dim = 64           # drastically smaller transformer hidden size
latent_dim = 8           # low latent space
num_latents = 256        # smaller codebook
patch_size = 8           # shorter sequences
num_blocks = 2           # shallow encoder and decoder
{'encoder': 89032, 'vq': 2048, 'decoder': 88848, 'total': 179.928}

--model_dim 384 \
--num_blocks 6 \
{'encoder': 8029184, 'vq': 32768, 'decoder': 8029168, 'total': 16091120}


--model_dim 384 \
{'encoder': 10.694.912, 'vq': 32.768, 'decoder': 10.694.896, 'total': 21.422.576}

--model_dim 256\
{'encoder': 4770688, 'vq': 32768, 'decoder': 4770672, 'total': 9574128}


**MINIMAL**
model_dim = 128
num_blocks = 2
latent_dim = 8
num_latents = 256
patch_size = 16

scancel 3297722 3297723 3297726