Spaces:
Running
on
L4
Running
on
L4
File size: 2,526 Bytes
a344f64 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
#!/bin/bash
#SBATCH --comment clap
#SBATCH --partition=g40423
#SBATCH --job-name=mclap
#SBATCH --nodes 3
#SBATCH --ntasks-per-node 8
#SBATCH --cpus-per-gpu=6
#SBATCH --exclusive
#SBATCH --output=%x_%j.out
module load openmpi
module load cuda/11.7
export NCCL_PROTO=simple
export FI_EFA_FORK_SAFE=1
export FI_LOG_LEVEL=1
export FI_EFA_USE_DEVICE_RDMA=1 # use for p4dn
export NCCL_DEBUG=info
export OMPI_MCA_mtl_base_verbose=1
export FI_EFA_ENABLE_SHM_TRANSFER=0
export FI_PROVIDER=efa
export FI_EFA_TX_MIN_CREDITS=64
export NCCL_TREE_THRESHOLD=0
# sent to sub script
export HOSTNAMES=`scontrol show hostnames "$SLURM_JOB_NODELIST"`
export MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" | head -n 1)
export MASTER_PORT=12802
export COUNT_NODE=`scontrol show hostnames "$SLURM_JOB_NODELIST" | wc -l`
echo go $COUNT_NODE
echo $HOSTNAMES
source /fsx/yusong/clap/bin/activate
cd /fsx/yusong/CLAP/src
export TRANSFORMERS_CACHE=/fsx/yusong/transformers_cache
srun --comment clap --cpu_bind=v --accel-bind=gn python -m training.main \
--save-frequency 5 \
--save-top-performance 3 \
--save-most-recent \
--dataset-type="webdataset" \
--precision="fp32" \
--batch-size=96 \
--lr=1e-4 \
--wd=0.0 \
--epochs=45 \
--workers=6 \
--use-bn-sync \
--amodel HTSAT-tiny \
--tmodel roberta \
--warmup 3200 \
--report-to "wandb" \
--wandb-notes "10.16-clap-dataset-2#-htsat-roberta-fusion" \
--datasetnames "Clotho" "audiocaps" "BBCSoundEffects" "free_to_use_sounds" "paramount_motion" "sonniss_game_effects" "wesoundeffects" "freesound_no_overlap_noesc50" "audiostock" "epidemic_sound_effects" "fsd50k_class_label" "MACS" "WavText5K" \
--full-train-dataset "BBCSoundEffects" "free_to_use_sounds" "paramount_motion" "sonniss_game_effects" "wesoundeffects" "audiostock" "epidemic_sound_effects" "fsd50k_class_label" \
--exclude-eval-dataset "freesound_no_overlap_noesc50" "MACS" "WavText5K" "fsd50k_class_label" \
--datasetinfos "train" "unbalanced_train" \
--top-k-checkpoint-select-dataset="Clotho-test" \
--top-k-checkpoint-select-metric="mAP@10" \
--openai-model-cache-dir /fsx/yusong/transformers_cache \
--logs /fsx/clap_logs \
--seed 3407 \
--remotedata \
--gather-with-grad \
--optimizer "adam" \
--data-filling "repeatpad" \
--data-truncating "fusion" \
--enable-fusion \
--fusion-type "aff_2d" \
--pretrained-audio /fsx/yusong/audio_pretrained_model/HTSAT-fullset-imagenet-map=0.467.ckpt
|