Spaces:
Runtime error
Runtime error
File size: 404 Bytes
83d8d3c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 |
ip_list=("ip1" "ip2" "ip3" "ip4")
config=wf42m_pfc03_32gpu_r100
for((node_rank=0;node_rank<${#ip_list[*]};node_rank++));
do
ssh ubuntu@${ip_list[node_rank]} "cd `pwd`;PATH=$PATH \
CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
torchrun \
--nproc_per_node=8 \
--nnodes=${#ip_list[*]} \
--node_rank=$node_rank \
--master_addr=${ip_list[0]} \
--master_port=22345 train.py configs/$config" &
done
|