#!/bin/bash
cd /ocean/projects/cis210027p/ganesank/karthik_new/espnet/egs2/sinhala/asr1
. ./path.sh
( echo '#' Running on `hostname`
  echo '#' Started at `date`
  set | grep SLURM | while read line; do echo "# $line"; done
  echo -n '# '; cat <<EOF
python3 -m espnet2.bin.asr_train --collect_stats true --use_preprocessor true --bpemodel none --token_type word --token_list data/en_token_list/word/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --train_data_path_and_name_and_type dump/raw/train/wav.scp,speech,sound --train_data_path_and_name_and_type dump/raw/train/text,text,text --valid_data_path_and_name_and_type dump/raw/valid/wav.scp,speech,sound --valid_data_path_and_name_and_type dump/raw/valid/text,text,text --train_shape_file exp/asr_stats_raw_en_word/logdir/train.${SLURM_ARRAY_TASK_ID}.scp --valid_shape_file exp/asr_stats_raw_en_word/logdir/valid.${SLURM_ARRAY_TASK_ID}.scp --output_dir exp/asr_stats_raw_en_word/logdir/stats.${SLURM_ARRAY_TASK_ID} --config conf/train_asr.yaml --frontend_conf fs=16k 
EOF
) >exp/asr_stats_raw_en_word/logdir/stats.$SLURM_ARRAY_TASK_ID.log
if [ "$CUDA_VISIBLE_DEVICES" == "NoDevFiles" ]; then
  ( echo CUDA_VISIBLE_DEVICES set to NoDevFiles, unsetting it... 
  )>>exp/asr_stats_raw_en_word/logdir/stats.$SLURM_ARRAY_TASK_ID.log
  unset CUDA_VISIBLE_DEVICES
fi
time1=`date +"%s"`
 ( python3 -m espnet2.bin.asr_train --collect_stats true --use_preprocessor true --bpemodel none --token_type word --token_list data/en_token_list/word/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --train_data_path_and_name_and_type dump/raw/train/wav.scp,speech,sound --train_data_path_and_name_and_type dump/raw/train/text,text,text --valid_data_path_and_name_and_type dump/raw/valid/wav.scp,speech,sound --valid_data_path_and_name_and_type dump/raw/valid/text,text,text --train_shape_file exp/asr_stats_raw_en_word/logdir/train.${SLURM_ARRAY_TASK_ID}.scp --valid_shape_file exp/asr_stats_raw_en_word/logdir/valid.${SLURM_ARRAY_TASK_ID}.scp --output_dir exp/asr_stats_raw_en_word/logdir/stats.${SLURM_ARRAY_TASK_ID} --config conf/train_asr.yaml --frontend_conf fs=16k  ) &>>exp/asr_stats_raw_en_word/logdir/stats.$SLURM_ARRAY_TASK_ID.log
ret=$?
sync || true
time2=`date +"%s"`
echo '#' Accounting: begin_time=$time1 >>exp/asr_stats_raw_en_word/logdir/stats.$SLURM_ARRAY_TASK_ID.log
echo '#' Accounting: end_time=$time2 >>exp/asr_stats_raw_en_word/logdir/stats.$SLURM_ARRAY_TASK_ID.log
echo '#' Accounting: time=$(($time2-$time1)) threads=1 >>exp/asr_stats_raw_en_word/logdir/stats.$SLURM_ARRAY_TASK_ID.log
echo '#' Finished at `date` with status $ret >>exp/asr_stats_raw_en_word/logdir/stats.$SLURM_ARRAY_TASK_ID.log
[ $ret -eq 137 ] && exit 100;
touch exp/asr_stats_raw_en_word/logdir/q/done.1797611.$SLURM_ARRAY_TASK_ID
exit $[$ret ? 1 : 0]
## submitted with:
# sbatch --export=PATH  --time 10:00:00 -p RM-shared  --open-mode=append -e exp/asr_stats_raw_en_word/logdir/q/stats.log -o exp/asr_stats_raw_en_word/logdir/q/stats.log --array 1-32 /ocean/projects/cis210027p/ganesank/karthik_new/espnet/egs2/sinhala/asr1/exp/asr_stats_raw_en_word/logdir/q/stats.sh >>exp/asr_stats_raw_en_word/logdir/q/stats.log 2>&1