p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/jobs/mihir/horeka/preprocessing/coinrun_chunked.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=10:00:00
#SBATCH --partition=large
#SBATCH --cpus-per-task=16
#SBATCH --output=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --error=/hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/logs/logs_mihir/preprocess/coinrun/%x_%j.log
#SBATCH --job-name=preprocess_coinrun_chunked

source .venv/bin/activate

python input_pipeline/generate_coinrun_dataset.py \
    --min_episode_length 1000 \
    --output_dir /hkfs/work/workspace/scratch/tum_ind3695-jafa_ws_shared/data_coinrun/coinrun_episodes_test \
    --num_episodes 35
[content] input_pipeline/generate_coinrun_dataset.py (python)
        print(f"Warning: {len(chunks)} chunks are being dropped to maintain a consistent number of chunks per file. This may result in data loss and could affect the training performance. Consider adjusting the chunk size or the number of episodes to minimize data loss.")
[content] input_pipeline/pngs_to_array_records.py (python)

    if len(chunks) > 0:
        print(f"Warning: Dropping {len(chunks)} chunks for consistent number of chunks per file. ",
        "Consider changing the chunk_size and chunks_per_file parameters to prevent data-loss.")