p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] example_usage.py (python)
#!/usr/bin/env python3
"""
Example usage of ArrayRecord dataset with train_tokenizer.py

This script demonstrates how to:
1. Generate ArrayRecord dataset
2. Use it with the training script
"""

import os
import subprocess
import sys


def main():
    print("ArrayRecord Dataset Generation and Usage Example")
    print("=" * 50)
    
    # Step 1: Generate the ArrayRecord dataset
    print("
1. Generating ArrayRecord dataset...")
    
    # Create a small dataset for testing
    cmd = [
        sys.executable, "generate_arrayrecord_dataset.py",
        "--output_dir", "data_arrayrecord/coinrun",
        "--num_videos", "100",  # Small dataset for testing
        "--seq_len", "16",
        "--height", "90", 
        "--width", "160",
        "--channels", "3",
        "--records_per_file", "20",
        "--verify"
    ]
    
    try:
        result = subprocess.run(cmd, check=True, capture_output=True, text=True)
        print("Dataset generation successful!")
        print(result.stdout)
    except subprocess.CalledProcessError as e:
        print(f"Error generating dataset: {e}")
        print(f"Error output: {e.stderr}")
        return
    
    # Step 2: Show how to use with training script
    print("
2. Example training command with ArrayRecord dataset:")
    print("=" * 60)
    
    training_cmd = [
        sys.executable, "train_tokenizer.py",
        "--data_dir", "data_arrayrecord/coinrun",
        "--num_steps", "1000",  # Small number for testing
        "--batch_size", "8",    # Smaller batch size for testing
        "--log", "false",       # Disable wandb logging for testing
        "--ckpt_dir", "checkpoints"
    ]
    
    print("Command:")
    print(" ".join(training_cmd))
    
    print("
3. Key points about the ArrayRecord format:")
    print("- Each record contains a 'videos' key with numpy array")
    print("- Video shape: (seq_len, height, width, channels)")
    print("- Data type: float32, values in range [0, 1]")
    print("- Files are automatically sharded for efficient loading")
    
    print("
4. Dataset structure:")
    data_dir = "data_arrayrecord/coinrun"
    if os.path.exists(data_dir):
        files = [f for f in os.listdir(data_dir) if f.endswith('.array_record')]
        print(f"Generated {len(files)} ArrayRecord files:")
        for f in sorted(files):
            file_path = os.path.join(data_dir, f)
            size_mb = os.path.getsize(file_path) / (1024 * 1024)
            print(f"  - {f} ({size_mb:.1f} MB)")
    
    print("
5. To run training with this dataset:")
    print("   python train_tokenizer.py --data_dir data_arrayrecord/coinrun")
    print("
   Note: You may need to modify the dataloader in train_tokenizer.py")
    print("   to handle ArrayRecord format instead of TFRecord format.")


if __name__ == "__main__":
    main()
[content] experiments/tokenizer_dummy_time_measurement_grain.sh (shellscript)
#!/usr/bin/env bash
module unload mpi/openmpi/5.0
module unload devel/cuda/12.4
source .venv/bin/activate


data_dir='/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data/knoms_arrayrecords_500_shards_group_size_1'

# FIXME: the `time_measurement_interval` line is only for debugging and should be removed
srun python train_tokenizer.py \
    --time_measurement_interval 10 \
    --batch_size 48 \
    --num_steps 300000 \
    --warmup_steps 10000 \
    --seed 0 \
    --min_lr=0.0000866 \
    --max_lr=0.0000866 \
    --data_dir $data_dir