File size: 1,280 Bytes
483de47 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
#!/usr/bin/env bash
# ------------------------------------------------------------------------
# Copyright (c) 2022 megvii-research. All Rights Reserved.
# ------------------------------------------------------------------------
set -x
PY_ARGS=${@:2}
set -o pipefail
OUTPUT_BASE=$(echo $1 | sed -e "s/configs/exps/g" | sed -e "s/.args$//g")
mkdir -p $OUTPUT_BASE
for RUN in $(seq 100); do
ls $OUTPUT_BASE | grep run$RUN && continue
OUTPUT_DIR=$OUTPUT_BASE/run$RUN
mkdir $OUTPUT_DIR && break
done
# clean up *.pyc files
rmpyc() {
rm -rf $(find -name __pycache__)
rm -rf $(find -name "*.pyc")
}
# run backup
echo "Backing up to log dir: $OUTPUT_DIR"
rmpyc && cp -r models datasets util main.py engine.py submit_dance.py $1 $OUTPUT_DIR
echo " ...Done"
# tar src to avoid future editing
cleanup() {
echo "Packing source code"
rmpyc
# tar -zcf models datasets util main.py engine.py eval.py submit.py --remove-files
echo " ...Done"
}
args=$(cat $1)
pushd $OUTPUT_DIR
trap cleanup EXIT
# log git status
echo "Logging git status"
git status > git_status
git rev-parse HEAD > git_tag
git diff > git_diff
echo $PY_ARGS > desc
echo " ...Done"
python -m torch.distributed.launch --nproc_per_node=8 --use_env main.py ${args} --output_dir . |& tee -a output.log
|