p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] upload_dataset.sh (shellscript)
#!/bin/bash
#SBATCH --job-name=s3_to_hf_transfer
#SBATCH --output=logs/s3_to_hf_transfer_%j.out
#SBATCH --error=logs/s3_to_hf_transfer_%j.err
#SBATCH --time=24:00:00
#SBATCH --gres=gpu:0
#SBATCH --cpus-per-task=8
#SBATCH --mem=100G

# Activate your environment if needed
source .venv/bin/activate

echo "Starting s3_to_hf_transfer.py at $(date)"
python s3_to_hf_transfer.py
echo "Done at $(date)"