Spaces:
Runtime error
Runtime error
File size: 1,389 Bytes
217780a |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
#!/bin/bash
#SBATCH --job-name=tr_test-s3-cleanup-checkpoints
#SBATCH --ntasks=1
#SBATCH --nodes=1
#SBATCH --time=3:00:00
#SBATCH --partition=production-cluster
#SBATCH --output=/fsx/m4/experiments/local_experiment_dir/s3_async_temporary_checkpoint_folder/logs/%x-%j.out
set -e
# ----------------- Auto-Workdir -----------------
if [ -n $SLURM_JOB_ID ]; then
# check the original location through scontrol and $SLURM_JOB_ID
SCRIPT_PATH=$(scontrol show job $SLURM_JOB_ID | awk -F= '/Command=/{print $2}')
else
# otherwise: started with bash. Get the real location.
SCRIPT_PATH=$(realpath $0)
fi
SCRIPT_DIR=$(dirname ${SCRIPT_PATH})
M4_REPO_PATH=$(builtin cd $SCRIPT_DIR/../../; pwd)
# --------------------------------------------------
### EDIT ME START ###
CONDA_ENV_NAME=shared-m4
EXPERIMENT_NAME=tr_194_laion_cm4_mix
opt_step_num_list=(
"1000"
"2000"
)
### EDIT ME END ###
echo "START TIME: $(date)"
source /fsx/m4/start-m4-user
conda activate base
conda activate $CONDA_ENV_NAME
pushd $M4_REPO_PATH
export PYTHONPATH=$WORKING_DIR:$PYTHONPATH
for opt_step_num in ${opt_step_num_list[@]}
do
OPT_STEP_DIR="/fsx/m4/experiments/local_experiment_dir/s3_async_temporary_checkpoint_folder/${EXPERIMENT_NAME}/opt_step-${opt_step_num}"
rm -r $OPT_STEP_DIR
echo "Deleted $OPT_STEP_DIR of experiment: $EXPERIMENT_NAME"
done
echo "END TIME: $(date)"
|