diff --git "a/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.10.log" "b/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.10.log" new file mode 100644--- /dev/null +++ "b/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.10.log" @@ -0,0 +1,4548 @@ +# Running on gpua006.delta.ncsa.illinois.edu +# Started at Thu Feb 8 14:22:50 CST 2024 +# SLURMD_NODENAME=gpua006 +# SLURM_CLUSTER_NAME=delta +# SLURM_CONF=/var/spool/slurmd/conf-cache/slurm.conf +# SLURM_CPUS_ON_NODE=64 +# SLURM_CPUS_PER_TASK=64 +# SLURM_EXPORT_ENV=PATH +# SLURM_GET_USER_ENV=1 +# SLURM_GPUS_ON_NODE=4 +# SLURM_GTIDS=0 +# SLURM_JOBID=2958902 +# SLURM_JOB_ACCOUNT=bbjs-delta-gpu +# SLURM_JOB_CPUS_PER_NODE='64(x16)' +# SLURM_JOB_END_TIME=1707596551 +# SLURM_JOB_GID=202 +# SLURM_JOB_GPUS=0,1,2,3 +# SLURM_JOB_ID=2958902 +# SLURM_JOB_NAME=exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.log +# SLURM_JOB_NODELIST='gpua[006,012,016,033,038-040,042,049,054-055,057,079-080,085,089]' +# SLURM_JOB_NUM_NODES=16 +# SLURM_JOB_PARTITION=gpuA100x4 +# SLURM_JOB_QOS=bbjs-delta-gpu +# SLURM_JOB_RESERVATION=bbjs +# SLURM_JOB_START_TIME=1707423751 +# SLURM_JOB_UID=68077 +# SLURM_JOB_USER=peng6 +# SLURM_LOCALID=0 +# SLURM_MEM_PER_NODE=240000 +# SLURM_MPI_TYPE=pmi2 +# SLURM_NNODES=16 +# SLURM_NODEID=0 +# SLURM_NODELIST='gpua[006,012,016,033,038-040,042,049,054-055,057,079-080,085,089]' +# SLURM_NODE_ALIASES='(null)' +# SLURM_OPEN_MODE=a +# SLURM_PRIO_PROCESS=0 +# SLURM_PROCID=0 +# SLURM_SUBMIT_DIR=/scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1 +# SLURM_SUBMIT_HOST=dt-login03.delta.ncsa.illinois.edu +# SLURM_TASKS_PER_NODE='1(x16)' +# SLURM_TASK_PID=525439 +# SLURM_TOPOLOGY_ADDR=ss00.ss05.gpua006 +# SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.node +# SLURM_WORKING_CLUSTER=delta:dt-sched:6817:9984:109 +# srun --export=ALL python3 -m espnet2.bin.s2t_train --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_192e59e9-b15c-4b09-be74-79f4fc1abee3 +[gpua006:0/64] 2024-02-08 14:27:27,208 (distributed_c10d:319) INFO: Added key: store_based_barrier_key:1 to store for rank: 0 +[gpua006:0/64] 2024-02-08 14:27:28,841 (distributed_c10d:353) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 64 nodes. +[gpua006:0/64] 2024-02-08 14:27:28,871 (s2t:420) INFO: Vocabulary size: 50002 +[gpua006:0/64] 2024-02-08 14:27:42,689 (abs_task:1270) INFO: pytorch.version=1.13.1, cuda.available=True, cudnn.version=8500, cudnn.benchmark=False, cudnn.deterministic=True +[gpua006:0/64] 2024-02-08 14:27:42,700 (abs_task:1271) INFO: Model structure: +ESPnetS2TCTCModel( + (frontend): DefaultFrontend( + (stft): Stft(n_fft=512, win_length=400, hop_length=160, center=True, normalized=False, onesided=True) + (frontend): Frontend() + (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False) + ) + (specaug): SpecAug( + (freq_mask): MaskAlongAxis(mask_width_range=[0, 27], num_mask=2, axis=freq) + (time_mask): MaskAlongAxisVariableMaxWidth(mask_width_ratio_range=[0.0, 0.05], num_mask=10, axis=time) + ) + (normalize): GlobalMVN(stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz, norm_means=True, norm_vars=True) + (encoder): EBranchformerCTCEncoder( + (embed): Conv2dSubsampling8( + (conv): Sequential( + (0): Conv2d(1, 1024, kernel_size=(3, 3), stride=(2, 2)) + (1): ReLU() + (2): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2)) + (3): ReLU() + (4): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2)) + (5): ReLU() + ) + (out): Linear(in_features=9216, out_features=1024, bias=True) + (pos_enc): PositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (encoders): MultiSequential( + (0): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (1): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (2): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (3): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (4): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (5): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (6): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (7): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (8): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (9): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (10): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (11): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (12): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (13): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (14): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (15): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (16): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (17): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (18): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (19): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (20): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (21): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (22): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (23): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (24): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (25): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (26): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + ) + (after_norm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (conditioning_layer): Linear(in_features=50002, out_features=1024, bias=True) + ) + (prompt_encoder): TransformerEncoder( + (encoders): MultiSequential( + (0): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + ) + (embed): Embedding(50002, 512) + (pos_enc): PositionalEncoding( + (dropout): Dropout(p=0.0, inplace=False) + ) + (embed_proj): Linear(in_features=512, out_features=1024, bias=True) + (prompt_proj): Linear(in_features=512, out_features=1024, bias=True) + (ctc): CTC( + (ctc_lo): Linear(in_features=1024, out_features=50002, bias=True) + (ctc_loss): CTCLoss() + ) +) + +Model summary: + Class Name: ESPnetS2TCTCModel + Total Number of model parameters: 1.01 B + Number of trainable parameters: 1.01 B (100.0%) + Size: 4.02 GB + Type: torch.float32 +[gpua006:0/64] 2024-02-08 14:27:42,700 (abs_task:1274) INFO: Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: [0.9, 0.98] + capturable: False + eps: 1e-06 + foreach: None + initial_lr: 0.0002 + lr: 1.6666666666666667e-09 + maximize: False + weight_decay: 0.0 +) +[gpua006:0/64] 2024-02-08 14:27:42,701 (abs_task:1275) INFO: Scheduler: PiecewiseLinearWarmupLR(warmup_steps_list=[0, 30000, 60000], warmup_lr_list=[0.0, 5e-05, 0.0002]) +[gpua006:0/64] 2024-02-08 14:27:42,710 (abs_task:1284) INFO: Saving the configuration in exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/config.yaml +[gpua006:0/64] 2024-02-08 14:27:48,168 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 14:27:49,086 (abs_task:1660) INFO: [valid] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/dev_v3/wav.scp", "type": "kaldi_ark"} + text_prev: {"path": "dump/raw/dev_v3/text.prev", "type": "text"} + text_ctc: {"path": "dump/raw/dev_v3/text.ctc", "type": "text"} + text: {"path": "dump/raw/dev_v3/text", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 14:27:49,086 (abs_task:1661) INFO: [valid] Batch sampler: UnsortedBatchSampler(N-batch=4671, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/valid/speech_shape, +[gpua006:0/64] 2024-02-08 14:27:49,087 (abs_task:1662) INFO: [valid] mini-batch sizes summary: N-batch=4671, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 14:28:18,185 (trainer:167) INFO: The training was resumed using exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/checkpoint.pth +gpua006:525536:525536 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:525536:525536 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:525536:525536 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:525536:525536 [0] NCCL INFO cudaDriverVersion 12020 +NCCL version 2.14.3+cuda11.7 +[gpua006:0/64] 2024-02-08 14:28:24,647 (trainer:301) INFO: 22/45epoch started +[gpua006:0/64] 2024-02-08 14:28:24,691 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-08 14:28:42,841 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 14:28:46,231 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 14:28:46,232 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-08 14:28:46,235 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +gpua016:431493:431493 [1] NCCL INFO cudaDriverVersion 12020 +gpua016:431493:431493 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:431493:431493 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:431493:431493 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:431493:431566 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:431493:431566 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:431493:431566 [1] NCCL INFO Using network AWS Libfabric +gpua016:431493:431566 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua016:431493:431566 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:431493:431566 [1] NCCL INFO Trees [0] 10/4/-1->9->8 [1] 10/-1/-1->9->8 +gpua016:431493:431566 [1] NCCL INFO Channel 00/0 : 9[46000] -> 10[85000] via P2P/IPC/read +gpua016:431493:431566 [1] NCCL INFO Channel 01/0 : 9[46000] -> 10[85000] via P2P/IPC/read +gpua016:431493:431566 [1] NCCL INFO Connected all rings +gpua016:431493:431566 [1] NCCL INFO Channel 00/0 : 4[7000] -> 9[46000] [receive] via NET/AWS Libfabric/1 +gpua016:431493:431566 [1] NCCL INFO Channel 00/0 : 9[46000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua016:431493:431566 [1] NCCL INFO Channel 00/0 : 9[46000] -> 8[7000] via P2P/IPC/read +gpua016:431493:431566 [1] NCCL INFO Channel 01/0 : 9[46000] -> 8[7000] via P2P/IPC/read +gpua016:431493:431566 [1] NCCL INFO Connected all trees +gpua016:431493:431566 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua016:431493:431566 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:431493:431566 [1] NCCL INFO comm 0x56355cc509b0 rank 9 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua016:431494:431494 [2] NCCL INFO cudaDriverVersion 12020 +gpua016:431494:431494 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:431494:431494 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:431494:431494 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:431494:431568 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:431494:431568 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:431494:431568 [2] NCCL INFO Using network AWS Libfabric +gpua016:431494:431568 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua016:431494:431568 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:431494:431568 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 +gpua016:431494:431568 [2] NCCL INFO Channel 00/0 : 10[85000] -> 11[c7000] via P2P/IPC/read +gpua016:431494:431568 [2] NCCL INFO Channel 01/0 : 10[85000] -> 11[c7000] via P2P/IPC/read +gpua016:431494:431568 [2] NCCL INFO Connected all rings +gpua089:699759:699759 [1] NCCL INFO cudaDriverVersion 12020 +gpua089:699759:699759 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:699759:699759 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:699759:699759 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:699759:699827 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:699759:699827 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua089:699759:699827 [1] NCCL INFO Using network AWS Libfabric +gpua089:699759:699827 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua089:699759:699827 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:699759:699827 [1] NCCL INFO Trees [0] 62/-1/-1->61->60 [1] 62/-1/-1->61->60 +gpua089:699759:699827 [1] NCCL INFO Channel 00/0 : 61[46000] -> 62[85000] via P2P/IPC/read +gpua089:699759:699827 [1] NCCL INFO Channel 01/0 : 61[46000] -> 62[85000] via P2P/IPC/read +gpua089:699759:699827 [1] NCCL INFO Connected all rings +gpua016:431494:431568 [2] NCCL INFO Channel 00/0 : 10[85000] -> 9[46000] via P2P/IPC/read +gpua016:431494:431568 [2] NCCL INFO Channel 01/0 : 10[85000] -> 9[46000] via P2P/IPC/read +gpua016:431494:431568 [2] NCCL INFO Connected all trees +gpua016:431494:431568 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua016:431494:431568 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:431494:431568 [2] NCCL INFO comm 0x55822f999ef0 rank 10 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua089:699759:699827 [1] NCCL INFO Channel 00/0 : 61[46000] -> 60[7000] via P2P/IPC/read +gpua089:699759:699827 [1] NCCL INFO Channel 01/0 : 61[46000] -> 60[7000] via P2P/IPC/read +gpua089:699759:699827 [1] NCCL INFO Connected all trees +gpua089:699759:699827 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:699759:699827 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua089:699759:699827 [1] NCCL INFO comm 0x55efe92f1170 rank 61 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua016:431492:431492 [0] NCCL INFO cudaDriverVersion 12020 +gpua016:431492:431492 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:431492:431492 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:431492:431492 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:431492:431565 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:431492:431565 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:431492:431565 [0] NCCL INFO Using network AWS Libfabric +gpua016:431492:431565 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua016:431492:431565 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:431492:431565 [0] NCCL INFO Trees [0] 9/12/-1->8->17 [1] 9/-1/-1->8->5 +gpua016:431492:431565 [0] NCCL INFO Channel 00/0 : 7[c7000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Channel 01/0 : 7[c7000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua089:699758:699758 [0] NCCL INFO cudaDriverVersion 12020 +gpua089:699758:699758 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:699758:699758 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:699758:699758 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:699758:699825 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:699758:699825 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua089:699758:699825 [0] NCCL INFO Using network AWS Libfabric +gpua089:699758:699825 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua089:699758:699825 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:699758:699825 [0] NCCL INFO Trees [0] 61/-1/-1->60->56 [1] 61/28/-1->60->-1 +gpua089:699758:699825 [0] NCCL INFO Channel 00/0 : 59[c7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua089:699758:699825 [0] NCCL INFO Channel 01/0 : 59[c7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Channel 00/0 : 8[7000] -> 9[46000] via P2P/IPC/read +gpua016:431492:431565 [0] NCCL INFO Channel 01/0 : 8[7000] -> 9[46000] via P2P/IPC/read +gpua016:431492:431565 [0] NCCL INFO Connected all rings +gpua016:431492:431565 [0] NCCL INFO Channel 01/0 : 5[46000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Channel 00/0 : 8[7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Channel 00/0 : 8[7000] -> 17[46000] [send] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Channel 00/0 : 17[46000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Channel 00/0 : 12[7000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Channel 01/0 : 8[7000] -> 5[46000] [send] via NET/AWS Libfabric/1 +gpua016:431492:431565 [0] NCCL INFO Connected all trees +gpua016:431492:431565 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:699758:699825 [0] NCCL INFO Channel 00/0 : 60[7000] -> 61[46000] via P2P/IPC/read +gpua089:699758:699825 [0] NCCL INFO Channel 01/0 : 60[7000] -> 61[46000] via P2P/IPC/read +gpua089:699758:699825 [0] NCCL INFO Connected all rings +gpua089:699758:699825 [0] NCCL INFO Channel 00/0 : 56[7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua089:699758:699825 [0] NCCL INFO Channel 01/0 : 28[7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua089:699758:699825 [0] NCCL INFO Channel 01/0 : 60[7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua089:699758:699825 [0] NCCL INFO Channel 00/0 : 60[7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua089:699758:699825 [0] NCCL INFO Connected all trees +gpua089:699758:699825 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:699758:699825 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua089:699758:699825 [0] NCCL INFO comm 0x558a9f658710 rank 60 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua016:431492:431565 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:431492:431565 [0] NCCL INFO comm 0x55cb38fe7640 rank 8 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua089:699760:699760 [2] NCCL INFO cudaDriverVersion 12020 +gpua089:699760:699760 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:699760:699760 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:699760:699760 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:699760:699826 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:699760:699826 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua089:699760:699826 [2] NCCL INFO Using network AWS Libfabric +gpua089:699760:699826 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua089:699760:699826 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:699760:699826 [2] NCCL INFO Trees [0] 63/-1/-1->62->61 [1] 63/-1/-1->62->61 +gpua089:699760:699826 [2] NCCL INFO Channel 00/0 : 62[85000] -> 63[c7000] via P2P/IPC/read +gpua089:699760:699826 [2] NCCL INFO Channel 01/0 : 62[85000] -> 63[c7000] via P2P/IPC/read +gpua089:699760:699826 [2] NCCL INFO Connected all rings +gpua016:431495:431495 [3] NCCL INFO cudaDriverVersion 12020 +gpua016:431495:431495 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:431495:431495 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:431495:431495 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:431495:431567 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:431495:431567 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:431495:431567 [3] NCCL INFO Using network AWS Libfabric +gpua016:431495:431567 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua016:431495:431567 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:431495:431567 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] -1/-1/-1->11->10 +gpua016:431495:431567 [3] NCCL INFO Channel 00/0 : 11[c7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua016:431495:431567 [3] NCCL INFO Channel 01/0 : 11[c7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua089:699760:699826 [2] NCCL INFO Channel 00/0 : 62[85000] -> 61[46000] via P2P/IPC/read +gpua089:699760:699826 [2] NCCL INFO Channel 01/0 : 62[85000] -> 61[46000] via P2P/IPC/read +gpua089:699760:699826 [2] NCCL INFO Connected all trees +gpua089:699760:699826 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:699760:699826 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua089:699760:699826 [2] NCCL INFO comm 0x55619a731060 rank 62 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua016:431495:431567 [3] NCCL INFO Connected all rings +gpua016:431495:431567 [3] NCCL INFO Channel 00/0 : 11[c7000] -> 10[85000] via P2P/IPC/read +gpua016:431495:431567 [3] NCCL INFO Channel 01/0 : 11[c7000] -> 10[85000] via P2P/IPC/read +gpua016:431495:431567 [3] NCCL INFO Connected all trees +gpua016:431495:431567 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua016:431495:431567 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:431495:431567 [3] NCCL INFO comm 0x55e34eb45ca0 rank 11 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua089:699761:699761 [3] NCCL INFO cudaDriverVersion 12020 +gpua089:699761:699761 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:699761:699761 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:699761:699761 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:699761:699828 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:699761:699828 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua089:699761:699828 [3] NCCL INFO Using network AWS Libfabric +gpua089:699761:699828 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua089:699761:699828 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:699761:699828 [3] NCCL INFO Trees [0] -1/-1/-1->63->62 [1] -1/-1/-1->63->62 +gpua089:699761:699828 [3] NCCL INFO Channel 00/0 : 63[c7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua089:699761:699828 [3] NCCL INFO Channel 01/0 : 63[c7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua089:699761:699828 [3] NCCL INFO Connected all rings +gpua089:699761:699828 [3] NCCL INFO Channel 00/0 : 63[c7000] -> 62[85000] via P2P/IPC/read +gpua089:699761:699828 [3] NCCL INFO Channel 01/0 : 63[c7000] -> 62[85000] via P2P/IPC/read +gpua089:699761:699828 [3] NCCL INFO Connected all trees +gpua089:699761:699828 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:699761:699828 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua089:699761:699828 [3] NCCL INFO comm 0x55e35c653460 rank 63 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua042:92810:92810 [3] NCCL INFO cudaDriverVersion 12020 +gpua042:92810:92810 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.42<0> +gpua042:92810:92810 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua042:92810:92810 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua042:92810:92872 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua042:92810:92872 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua042:92810:92872 [3] NCCL INFO Using network AWS Libfabric +gpua042:92810:92872 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua042:92810:92872 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua042:92810:92872 [3] NCCL INFO Trees [0] -1/-1/-1->31->30 [1] -1/-1/-1->31->30 +gpua042:92810:92872 [3] NCCL INFO Channel 00/0 : 31[c7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua042:92810:92872 [3] NCCL INFO Channel 01/0 : 31[c7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua042:92810:92872 [3] NCCL INFO Connected all rings +gpua042:92810:92872 [3] NCCL INFO Channel 00/0 : 31[c7000] -> 30[85000] via P2P/IPC/read +gpua042:92810:92872 [3] NCCL INFO Channel 01/0 : 31[c7000] -> 30[85000] via P2P/IPC/read +gpua042:92810:92872 [3] NCCL INFO Connected all trees +gpua042:92810:92872 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua042:92810:92872 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua042:92810:92872 [3] NCCL INFO comm 0x55c3c7150100 rank 31 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua079:3539996:3539996 [1] NCCL INFO cudaDriverVersion 12020 +gpua079:3539996:3539996 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3539996:3539996 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3539996:3539996 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3539996:3540054 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3539996:3540054 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3539996:3540054 [1] NCCL INFO Using network AWS Libfabric +gpua079:3539996:3540054 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua079:3539996:3540054 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3539996:3540054 [1] NCCL INFO Trees [0] 50/40/-1->49->48 [1] 50/-1/-1->49->48 +gpua079:3539996:3540054 [1] NCCL INFO Channel 00/0 : 49[46000] -> 50[85000] via P2P/IPC/read +gpua079:3539996:3540054 [1] NCCL INFO Channel 01/0 : 49[46000] -> 50[85000] via P2P/IPC/read +gpua042:92807:92807 [0] NCCL INFO cudaDriverVersion 12020 +gpua042:92807:92807 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.42<0> +gpua042:92807:92807 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua042:92807:92807 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua042:92807:92871 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua042:92807:92871 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua042:92807:92871 [0] NCCL INFO Using network AWS Libfabric +gpua042:92807:92871 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua042:92807:92871 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua042:92807:92871 [0] NCCL INFO Trees [0] 29/-1/-1->28->24 [1] 29/12/-1->28->60 +gpua042:92807:92871 [0] NCCL INFO Channel 00/0 : 27[c7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Channel 01/0 : 27[c7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3539996:3540054 [1] NCCL INFO Connected all rings +gpua079:3539996:3540054 [1] NCCL INFO Channel 00/0 : 40[7000] -> 49[46000] [receive] via NET/AWS Libfabric/1 +gpua079:3539996:3540054 [1] NCCL INFO Channel 00/0 : 49[46000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua079:3539996:3540054 [1] NCCL INFO Channel 00/0 : 49[46000] -> 48[7000] via P2P/IPC/read +gpua079:3539996:3540054 [1] NCCL INFO Channel 01/0 : 49[46000] -> 48[7000] via P2P/IPC/read +gpua079:3539996:3540054 [1] NCCL INFO Connected all trees +gpua079:3539996:3540054 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3539996:3540054 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3539996:3540054 [1] NCCL INFO comm 0x55f7c0f50a60 rank 49 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua033:1490880:1490880 [3] NCCL INFO cudaDriverVersion 12020 +gpua033:1490880:1490880 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua033:1490880:1490880 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1490880:1490880 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1490880:1490945 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1490880:1490945 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1490880:1490945 [3] NCCL INFO Using network AWS Libfabric +gpua033:1490880:1490945 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua033:1490880:1490945 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1490880:1490945 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] -1/-1/-1->15->14 +gpua033:1490880:1490945 [3] NCCL INFO Channel 00/0 : 15[c7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua033:1490880:1490945 [3] NCCL INFO Channel 01/0 : 15[c7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Channel 00/0 : 28[7000] -> 29[46000] via P2P/IPC/read +gpua042:92807:92871 [0] NCCL INFO Channel 01/0 : 28[7000] -> 29[46000] via P2P/IPC/read +gpua042:92807:92871 [0] NCCL INFO Connected all rings +gpua042:92807:92871 [0] NCCL INFO Channel 00/0 : 24[7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Channel 01/0 : 12[7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Channel 01/0 : 60[7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Channel 01/0 : 28[7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Channel 01/0 : 28[7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Channel 00/0 : 28[7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua042:92807:92871 [0] NCCL INFO Connected all trees +gpua042:92807:92871 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:266921:266921 [3] NCCL INFO cudaDriverVersion 12020 +gpua054:266921:266921 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:266921:266921 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua054:266921:266921 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:266921:266992 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:266921:266992 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:266921:266992 [3] NCCL INFO Using network AWS Libfabric +gpua054:266921:266992 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua054:266921:266992 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:266921:266992 [3] NCCL INFO Trees [0] -1/-1/-1->39->38 [1] -1/-1/-1->39->38 +gpua054:266921:266992 [3] NCCL INFO Channel 00/0 : 39[c7000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua054:266921:266992 [3] NCCL INFO Channel 01/0 : 39[c7000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua049:3914992:3914992 [1] NCCL INFO cudaDriverVersion 12020 +gpua049:3914992:3914992 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua049:3914992:3914992 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:3914992:3914992 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:3914992:3915058 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:3914992:3915058 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:3914992:3915058 [1] NCCL INFO Using network AWS Libfabric +gpua049:3914992:3915058 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua049:3914992:3915058 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:3914992:3915058 [1] NCCL INFO Trees [0] 34/16/-1->33->32 [1] 34/-1/-1->33->32 +gpua049:3914992:3915058 [1] NCCL INFO Channel 00/0 : 33[46000] -> 34[85000] via P2P/IPC/read +gpua049:3914992:3915058 [1] NCCL INFO Channel 01/0 : 33[46000] -> 34[85000] via P2P/IPC/read +gpua057:3734968:3734968 [0] NCCL INFO cudaDriverVersion 12020 +gpua057:3734968:3734968 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3734968:3734968 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3734968:3734968 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3734968:3735041 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3734968:3735041 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3734968:3735041 [0] NCCL INFO Using network AWS Libfabric +gpua057:3734968:3735041 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua057:3734968:3735041 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3734968:3735041 [0] NCCL INFO Trees [0] 45/-1/-1->44->40 [1] 45/36/-1->44->29 +gpua057:3734968:3735041 [0] NCCL INFO Channel 00/0 : 43[c7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1490880:1490945 [3] NCCL INFO Connected all rings +gpua033:1490880:1490945 [3] NCCL INFO Channel 00/0 : 15[c7000] -> 14[85000] via P2P/IPC/read +gpua033:1490880:1490945 [3] NCCL INFO Channel 01/0 : 15[c7000] -> 14[85000] via P2P/IPC/read +gpua033:1490880:1490945 [3] NCCL INFO Connected all trees +gpua033:1490880:1490945 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1490880:1490945 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1490880:1490945 [3] NCCL INFO comm 0x55bdbdc4c160 rank 15 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua042:92807:92871 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua042:92807:92871 [0] NCCL INFO comm 0x55b0c250b070 rank 28 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua040:3706127:3706127 [2] NCCL INFO cudaDriverVersion 12020 +gpua040:3706127:3706127 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3706127:3706127 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3706127:3706127 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3706127:3706195 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua040:3706127:3706195 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3706127:3706195 [2] NCCL INFO Using network AWS Libfabric +gpua040:3706127:3706195 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua040:3706127:3706195 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3706127:3706195 [2] NCCL INFO Trees [0] 27/-1/-1->26->25 [1] 27/-1/-1->26->25 +gpua040:3706127:3706195 [2] NCCL INFO Channel 00/0 : 26[85000] -> 27[c7000] via P2P/IPC/read +gpua040:3706127:3706195 [2] NCCL INFO Channel 01/0 : 26[85000] -> 27[c7000] via P2P/IPC/read +gpua054:266921:266992 [3] NCCL INFO Connected all rings +gpua054:266921:266992 [3] NCCL INFO Channel 00/0 : 39[c7000] -> 38[85000] via P2P/IPC/read +gpua054:266921:266992 [3] NCCL INFO Channel 01/0 : 39[c7000] -> 38[85000] via P2P/IPC/read +gpua054:266921:266992 [3] NCCL INFO Connected all trees +gpua054:266921:266992 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:266921:266992 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:266921:266992 [3] NCCL INFO comm 0x55560dc61fc0 rank 39 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua079:3539998:3539998 [3] NCCL INFO cudaDriverVersion 12020 +gpua079:3539998:3539998 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3539998:3539998 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3539998:3539998 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3539998:3540052 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3539998:3540052 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3539998:3540052 [3] NCCL INFO Using network AWS Libfabric +gpua079:3539998:3540052 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua079:3539998:3540052 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3539998:3540052 [3] NCCL INFO Trees [0] -1/-1/-1->51->50 [1] -1/-1/-1->51->50 +gpua079:3539998:3540052 [3] NCCL INFO Channel 00/0 : 51[c7000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua079:3539998:3540052 [3] NCCL INFO Channel 01/0 : 51[c7000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua049:3914992:3915058 [1] NCCL INFO Connected all rings +gpua049:3914992:3915058 [1] NCCL INFO Channel 00/0 : 16[7000] -> 33[46000] [receive] via NET/AWS Libfabric/1 +gpua049:3914992:3915058 [1] NCCL INFO Channel 00/0 : 33[46000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua049:3914992:3915058 [1] NCCL INFO Channel 00/0 : 33[46000] -> 32[7000] via P2P/IPC/read +gpua049:3914992:3915058 [1] NCCL INFO Channel 01/0 : 33[46000] -> 32[7000] via P2P/IPC/read +gpua049:3914992:3915058 [1] NCCL INFO Connected all trees +gpua049:3914992:3915058 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:3914992:3915058 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:3914992:3915058 [1] NCCL INFO comm 0x5623e37cfde0 rank 33 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua057:3734968:3735041 [0] NCCL INFO Channel 01/0 : 43[c7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3734968:3735041 [0] NCCL INFO Channel 00/0 : 44[7000] -> 45[46000] via P2P/IPC/read +gpua057:3734968:3735041 [0] NCCL INFO Channel 01/0 : 44[7000] -> 45[46000] via P2P/IPC/read +gpua057:3734968:3735041 [0] NCCL INFO Connected all rings +gpua057:3734968:3735041 [0] NCCL INFO Channel 00/0 : 40[7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3734968:3735041 [0] NCCL INFO Channel 01/0 : 36[7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3734968:3735041 [0] NCCL INFO Channel 01/0 : 29[46000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3734968:3735041 [0] NCCL INFO Channel 01/0 : 44[7000] -> 29[46000] [send] via NET/AWS Libfabric/1 +gpua057:3734968:3735041 [0] NCCL INFO Channel 01/0 : 44[7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua057:3734968:3735041 [0] NCCL INFO Channel 00/0 : 44[7000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua033:1490877:1490877 [0] NCCL INFO cudaDriverVersion 12020 +gpua033:1490877:1490877 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua033:1490877:1490877 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1490877:1490877 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1490877:1490946 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1490877:1490946 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1490877:1490946 [0] NCCL INFO Using network AWS Libfabric +gpua033:1490877:1490946 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua033:1490877:1490946 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1490877:1490946 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] 13/4/-1->12->28 +gpua033:1490877:1490946 [0] NCCL INFO Channel 00/0 : 11[c7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua038:25307:25307 [1] NCCL INFO cudaDriverVersion 12020 +gpua038:25307:25307 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:25307:25307 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:25307:25307 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:25307:25373 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:25307:25373 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:25307:25373 [1] NCCL INFO Using network AWS Libfabric +gpua038:25307:25373 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua038:25307:25373 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:25307:25373 [1] NCCL INFO Trees [0] 18/8/-1->17->16 [1] 18/-1/-1->17->16 +gpua038:25307:25373 [1] NCCL INFO Channel 00/0 : 17[46000] -> 18[85000] via P2P/IPC/read +gpua038:25307:25373 [1] NCCL INFO Channel 01/0 : 17[46000] -> 18[85000] via P2P/IPC/read +gpua038:25307:25373 [1] NCCL INFO Connected all rings +gpua040:3706127:3706195 [2] NCCL INFO Connected all rings +gpua040:3706127:3706195 [2] NCCL INFO Channel 00/0 : 26[85000] -> 25[46000] via P2P/IPC/read +gpua040:3706127:3706195 [2] NCCL INFO Channel 01/0 : 26[85000] -> 25[46000] via P2P/IPC/read +gpua040:3706127:3706195 [2] NCCL INFO Connected all trees +gpua040:3706127:3706195 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3706127:3706195 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3706127:3706195 [2] NCCL INFO comm 0x558aa34cd650 rank 26 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua040:3706128:3706128 [3] NCCL INFO cudaDriverVersion 12020 +gpua040:3706128:3706128 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3706128:3706128 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3706128:3706128 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3706128:3706194 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3539998:3540052 [3] NCCL INFO Connected all rings +gpua079:3539998:3540052 [3] NCCL INFO Channel 00/0 : 51[c7000] -> 50[85000] via P2P/IPC/read +gpua079:3539998:3540052 [3] NCCL INFO Channel 01/0 : 51[c7000] -> 50[85000] via P2P/IPC/read +gpua079:3539998:3540052 [3] NCCL INFO Connected all trees +gpua079:3539998:3540052 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3539998:3540052 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3539998:3540052 [3] NCCL INFO comm 0x56521d240f20 rank 51 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua057:3734968:3735041 [0] NCCL INFO Connected all trees +gpua057:3734968:3735041 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3734968:3735041 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3734968:3735041 [0] NCCL INFO comm 0x55d713d4f620 rank 44 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua033:1490877:1490946 [0] NCCL INFO Channel 01/0 : 11[c7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1490877:1490946 [0] NCCL INFO Channel 00/0 : 12[7000] -> 13[46000] via P2P/IPC/read +gpua033:1490877:1490946 [0] NCCL INFO Channel 01/0 : 12[7000] -> 13[46000] via P2P/IPC/read +gpua033:1490877:1490946 [0] NCCL INFO Connected all rings +gpua033:1490877:1490946 [0] NCCL INFO Channel 00/0 : 8[7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1490877:1490946 [0] NCCL INFO Channel 01/0 : 4[7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1490877:1490946 [0] NCCL INFO Channel 01/0 : 12[7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua033:1490877:1490946 [0] NCCL INFO Channel 01/0 : 28[7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1490877:1490946 [0] NCCL INFO Channel 01/0 : 12[7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua033:1490877:1490946 [0] NCCL INFO Channel 00/0 : 12[7000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua038:25307:25373 [1] NCCL INFO Channel 00/0 : 8[7000] -> 17[46000] [receive] via NET/AWS Libfabric/1 +gpua038:25307:25373 [1] NCCL INFO Channel 00/0 : 17[46000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua038:25307:25373 [1] NCCL INFO Channel 00/0 : 17[46000] -> 16[7000] via P2P/IPC/read +gpua038:25307:25373 [1] NCCL INFO Channel 01/0 : 17[46000] -> 16[7000] via P2P/IPC/read +gpua038:25307:25373 [1] NCCL INFO Connected all trees +gpua038:25307:25373 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua038:25307:25373 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:25307:25373 [1] NCCL INFO comm 0x55f7ace57320 rank 17 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua040:3706128:3706194 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3706128:3706194 [3] NCCL INFO Using network AWS Libfabric +gpua040:3706128:3706194 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua040:3706128:3706194 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3706128:3706194 [3] NCCL INFO Trees [0] -1/-1/-1->27->26 [1] -1/-1/-1->27->26 +gpua040:3706128:3706194 [3] NCCL INFO Channel 00/0 : 27[c7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua040:3706128:3706194 [3] NCCL INFO Channel 01/0 : 27[c7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua040:3706128:3706194 [3] NCCL INFO Connected all rings +gpua040:3706128:3706194 [3] NCCL INFO Channel 00/0 : 27[c7000] -> 26[85000] via P2P/IPC/read +gpua040:3706128:3706194 [3] NCCL INFO Channel 01/0 : 27[c7000] -> 26[85000] via P2P/IPC/read +gpua040:3706128:3706194 [3] NCCL INFO Connected all trees +gpua040:3706128:3706194 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3734969:3734969 [1] NCCL INFO cudaDriverVersion 12020 +gpua057:3734969:3734969 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3734969:3734969 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3734969:3734969 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3734969:3735042 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3734969:3735042 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3734969:3735042 [1] NCCL INFO Using network AWS Libfabric +gpua057:3734969:3735042 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua057:3734969:3735042 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3734969:3735042 [1] NCCL INFO Trees [0] 46/-1/-1->45->44 [1] 46/52/-1->45->44 +gpua057:3734969:3735042 [1] NCCL INFO Channel 00/0 : 45[46000] -> 46[85000] via P2P/IPC/read +gpua057:3734969:3735042 [1] NCCL INFO Channel 01/0 : 45[46000] -> 46[85000] via P2P/IPC/read +gpua055:4159894:4159894 [3] NCCL INFO cudaDriverVersion 12020 +gpua055:4159894:4159894 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:4159894:4159894 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:4159894:4159894 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:4159894:4159950 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua055:4159894:4159950 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:4159894:4159950 [3] NCCL INFO Using network AWS Libfabric +gpua055:4159894:4159950 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua055:4159894:4159950 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:4159894:4159950 [3] NCCL INFO Trees [0] -1/-1/-1->43->42 [1] -1/-1/-1->43->42 +gpua055:4159894:4159950 [3] NCCL INFO Channel 00/0 : 43[c7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua055:4159894:4159950 [3] NCCL INFO Channel 01/0 : 43[c7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua033:1490877:1490946 [0] NCCL INFO Connected all trees +gpua033:1490877:1490946 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1490877:1490946 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1490877:1490946 [0] NCCL INFO comm 0x563350e34fd0 rank 12 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua006:525538:525538 [2] NCCL INFO cudaDriverVersion 12020 +gpua006:525538:525538 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:525538:525538 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:525538:525538 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:525538:525600 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:525538:525600 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:525538:525600 [2] NCCL INFO Using network AWS Libfabric +gpua006:525538:525600 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua006:525538:525600 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:525538:525600 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 +gpua006:525538:525600 [2] NCCL INFO Channel 00/0 : 2[85000] -> 3[c7000] via P2P/IPC/read +gpua006:525538:525600 [2] NCCL INFO Channel 01/0 : 2[85000] -> 3[c7000] via P2P/IPC/read +gpua006:525538:525600 [2] NCCL INFO Connected all rings +gpua040:3706128:3706194 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3706128:3706194 [3] NCCL INFO comm 0x55c618538260 rank 27 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua054:266919:266919 [1] NCCL INFO cudaDriverVersion 12020 +gpua054:266919:266919 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:266919:266919 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua054:266919:266919 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:266919:266990 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:266919:266990 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:266919:266990 [1] NCCL INFO Using network AWS Libfabric +gpua054:266919:266990 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua054:266919:266990 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:266919:266990 [1] NCCL INFO Trees [0] 38/-1/-1->37->36 [1] 38/40/-1->37->36 +gpua054:266919:266990 [1] NCCL INFO Channel 00/0 : 37[46000] -> 38[85000] via P2P/IPC/read +gpua054:266919:266990 [1] NCCL INFO Channel 01/0 : 37[46000] -> 38[85000] via P2P/IPC/read +gpua054:266919:266990 [1] NCCL INFO Connected all rings +gpua057:3734969:3735042 [1] NCCL INFO Connected all rings +gpua057:3734969:3735042 [1] NCCL INFO Channel 01/0 : 45[46000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua057:3734969:3735042 [1] NCCL INFO Channel 01/0 : 52[7000] -> 45[46000] [receive] via NET/AWS Libfabric/1 +gpua057:3734969:3735042 [1] NCCL INFO Channel 00/0 : 45[46000] -> 44[7000] via P2P/IPC/read +gpua057:3734969:3735042 [1] NCCL INFO Channel 01/0 : 45[46000] -> 44[7000] via P2P/IPC/read +gpua057:3734969:3735042 [1] NCCL INFO Connected all trees +gpua057:3734969:3735042 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3734969:3735042 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3734969:3735042 [1] NCCL INFO comm 0x5655642af9e0 rank 45 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua055:4159894:4159950 [3] NCCL INFO Connected all rings +gpua055:4159894:4159950 [3] NCCL INFO Channel 00/0 : 43[c7000] -> 42[85000] via P2P/IPC/read +gpua055:4159894:4159950 [3] NCCL INFO Channel 01/0 : 43[c7000] -> 42[85000] via P2P/IPC/read +gpua055:4159894:4159950 [3] NCCL INFO Connected all trees +gpua055:4159894:4159950 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:4159894:4159950 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:4159894:4159950 [3] NCCL INFO comm 0x563dd5fbfc30 rank 43 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua080:3124463:3124463 [3] NCCL INFO cudaDriverVersion 12020 +gpua080:3124463:3124463 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua080:3124463:3124463 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3124463:3124463 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3124463:3124525 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3124463:3124525 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3124463:3124525 [3] NCCL INFO Using network AWS Libfabric +gpua080:3124463:3124525 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua080:3124463:3124525 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3124463:3124525 [3] NCCL INFO Trees [0] -1/-1/-1->55->54 [1] -1/-1/-1->55->54 +gpua080:3124463:3124525 [3] NCCL INFO Channel 00/0 : 55[c7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua080:3124463:3124525 [3] NCCL INFO Channel 01/0 : 55[c7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua033:1490878:1490878 [1] NCCL INFO cudaDriverVersion 12020 +gpua033:1490878:1490878 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua033:1490878:1490878 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1490878:1490878 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1490878:1490944 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1490878:1490944 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1490878:1490944 [1] NCCL INFO Using network AWS Libfabric +gpua033:1490878:1490944 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua033:1490878:1490944 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1490878:1490944 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/20/-1->13->12 +gpua033:1490878:1490944 [1] NCCL INFO Channel 00/0 : 13[46000] -> 14[85000] via P2P/IPC/read +gpua033:1490878:1490944 [1] NCCL INFO Channel 01/0 : 13[46000] -> 14[85000] via P2P/IPC/read +gpua006:525538:525600 [2] NCCL INFO Channel 00/0 : 2[85000] -> 1[46000] via P2P/IPC/read +gpua006:525538:525600 [2] NCCL INFO Channel 01/0 : 2[85000] -> 1[46000] via P2P/IPC/read +gpua006:525538:525600 [2] NCCL INFO Connected all trees +gpua006:525538:525600 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:525538:525600 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua006:525538:525600 [2] NCCL INFO comm 0x55cee87c1790 rank 2 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua040:3706126:3706126 [1] NCCL INFO cudaDriverVersion 12020 +gpua040:3706126:3706126 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3706126:3706126 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3706126:3706126 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3706126:3706193 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua040:3706126:3706193 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3706126:3706193 [1] NCCL INFO Using network AWS Libfabric +gpua040:3706126:3706193 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua040:3706126:3706193 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3706126:3706193 [1] NCCL INFO Trees [0] 26/20/-1->25->24 [1] 26/-1/-1->25->24 +gpua040:3706126:3706193 [1] NCCL INFO Channel 00/0 : 25[46000] -> 26[85000] via P2P/IPC/read +gpua040:3706126:3706193 [1] NCCL INFO Channel 01/0 : 25[46000] -> 26[85000] via P2P/IPC/read +gpua054:266919:266990 [1] NCCL INFO Channel 01/0 : 37[46000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua054:266919:266990 [1] NCCL INFO Channel 01/0 : 40[7000] -> 37[46000] [receive] via NET/AWS Libfabric/1 +gpua054:266919:266990 [1] NCCL INFO Channel 00/0 : 37[46000] -> 36[7000] via P2P/IPC/read +gpua054:266919:266990 [1] NCCL INFO Channel 01/0 : 37[46000] -> 36[7000] via P2P/IPC/read +gpua054:266919:266990 [1] NCCL INFO Connected all trees +gpua054:266919:266990 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:266919:266990 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:266919:266990 [1] NCCL INFO comm 0x55f9aa179450 rank 37 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua079:3539995:3539995 [0] NCCL INFO cudaDriverVersion 12020 +gpua079:3539995:3539995 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3539995:3539995 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3539995:3539995 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3539995:3540053 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3539995:3540053 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3539995:3540053 [0] NCCL INFO Using network AWS Libfabric +gpua079:3539995:3540053 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua079:3539995:3540053 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3539995:3540053 [0] NCCL INFO Trees [0] 49/56/-1->48->32 [1] 49/-1/-1->48->52 +gpua079:3539995:3540053 [0] NCCL INFO Channel 00/0 : 47[c7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3734971:3734971 [3] NCCL INFO cudaDriverVersion 12020 +gpua057:3734971:3734971 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3734971:3734971 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3734971:3734971 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3734971:3735040 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3734971:3735040 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3734971:3735040 [3] NCCL INFO Using network AWS Libfabric +gpua057:3734971:3735040 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua057:3734971:3735040 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3734971:3735040 [3] NCCL INFO Trees [0] -1/-1/-1->47->46 [1] -1/-1/-1->47->46 +gpua057:3734971:3735040 [3] NCCL INFO Channel 00/0 : 47[c7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua057:3734971:3735040 [3] NCCL INFO Channel 01/0 : 47[c7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua055:4159892:4159892 [1] NCCL INFO cudaDriverVersion 12020 +gpua055:4159892:4159892 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:4159892:4159892 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:4159892:4159892 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:4159892:4159951 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua055:4159892:4159951 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:4159892:4159951 [1] NCCL INFO Using network AWS Libfabric +gpua055:4159892:4159951 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua055:4159892:4159951 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:4159892:4159951 [1] NCCL INFO Trees [0] 42/36/-1->41->40 [1] 42/-1/-1->41->40 +gpua055:4159892:4159951 [1] NCCL INFO Channel 00/0 : 41[46000] -> 42[85000] via P2P/IPC/read +gpua055:4159892:4159951 [1] NCCL INFO Channel 01/0 : 41[46000] -> 42[85000] via P2P/IPC/read +gpua080:3124463:3124525 [3] NCCL INFO Connected all rings +gpua080:3124463:3124525 [3] NCCL INFO Channel 00/0 : 55[c7000] -> 54[85000] via P2P/IPC/read +gpua080:3124463:3124525 [3] NCCL INFO Channel 01/0 : 55[c7000] -> 54[85000] via P2P/IPC/read +gpua080:3124463:3124525 [3] NCCL INFO Connected all trees +gpua080:3124463:3124525 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua080:3124463:3124525 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3124463:3124525 [3] NCCL INFO comm 0x5629fe178b40 rank 55 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua033:1490878:1490944 [1] NCCL INFO Connected all rings +gpua033:1490878:1490944 [1] NCCL INFO Channel 01/0 : 13[46000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua033:1490878:1490944 [1] NCCL INFO Channel 01/0 : 20[7000] -> 13[46000] [receive] via NET/AWS Libfabric/1 +gpua033:1490878:1490944 [1] NCCL INFO Channel 00/0 : 13[46000] -> 12[7000] via P2P/IPC/read +gpua033:1490878:1490944 [1] NCCL INFO Channel 01/0 : 13[46000] -> 12[7000] via P2P/IPC/read +gpua033:1490878:1490944 [1] NCCL INFO Connected all trees +gpua033:1490878:1490944 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1490878:1490944 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1490878:1490944 [1] NCCL INFO comm 0x56055de17060 rank 13 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua006:525537:525537 [1] NCCL INFO cudaDriverVersion 12020 +gpua006:525537:525537 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:525537:525537 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:525537:525537 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:525537:525602 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:525537:525602 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:525537:525602 [1] NCCL INFO Using network AWS Libfabric +gpua006:525537:525602 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua006:525537:525602 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:525537:525602 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 +gpua006:525537:525602 [1] NCCL INFO Channel 00/0 : 1[46000] -> 2[85000] via P2P/IPC/read +gpua006:525537:525602 [1] NCCL INFO Channel 01/0 : 1[46000] -> 2[85000] via P2P/IPC/read +gpua006:525537:525602 [1] NCCL INFO Connected all rings +gpua038:25306:25306 [0] NCCL INFO cudaDriverVersion 12020 +gpua038:25306:25306 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:25306:25306 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:25306:25306 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:25306:25375 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:25306:25375 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:25306:25375 [0] NCCL INFO Using network AWS Libfabric +gpua038:25306:25375 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua038:25306:25375 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:25306:25375 [0] NCCL INFO Trees [0] 17/24/-1->16->33 [1] 17/-1/-1->16->20 +gpua038:25306:25375 [0] NCCL INFO Channel 00/0 : 15[c7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Channel 01/0 : 15[c7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3706126:3706193 [1] NCCL INFO Connected all rings +gpua040:3706126:3706193 [1] NCCL INFO Channel 00/0 : 20[7000] -> 25[46000] [receive] via NET/AWS Libfabric/1 +gpua040:3706126:3706193 [1] NCCL INFO Channel 00/0 : 25[46000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua040:3706126:3706193 [1] NCCL INFO Channel 00/0 : 25[46000] -> 24[7000] via P2P/IPC/read +gpua040:3706126:3706193 [1] NCCL INFO Channel 01/0 : 25[46000] -> 24[7000] via P2P/IPC/read +gpua040:3706126:3706193 [1] NCCL INFO Connected all trees +gpua040:3706126:3706193 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3706126:3706193 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3706126:3706193 [1] NCCL INFO comm 0x55bf96906ed0 rank 25 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua079:3539995:3540053 [0] NCCL INFO Channel 01/0 : 47[c7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3539995:3540053 [0] NCCL INFO Channel 00/0 : 48[7000] -> 49[46000] via P2P/IPC/read +gpua079:3539995:3540053 [0] NCCL INFO Channel 01/0 : 48[7000] -> 49[46000] via P2P/IPC/read +gpua079:3539995:3540053 [0] NCCL INFO Connected all rings +gpua079:3539995:3540053 [0] NCCL INFO Channel 01/0 : 48[7000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua079:3539995:3540053 [0] NCCL INFO Channel 00/0 : 48[7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua079:3539995:3540053 [0] NCCL INFO Channel 00/0 : 32[7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3539995:3540053 [0] NCCL INFO Channel 00/0 : 48[7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua079:3539995:3540053 [0] NCCL INFO Channel 00/0 : 56[7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3539995:3540053 [0] NCCL INFO Channel 01/0 : 52[7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua049:3914991:3914991 [0] NCCL INFO cudaDriverVersion 12020 +gpua049:3914991:3914991 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua049:3914991:3914991 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:3914991:3914991 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:3914991:3915056 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:3914991:3915056 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:3914991:3915056 [0] NCCL INFO Using network AWS Libfabric +gpua049:3914991:3915056 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua049:3914991:3915056 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:3914991:3915056 [0] NCCL INFO Trees [0] 33/48/-1->32->0 [1] 33/-1/-1->32->36 +gpua049:3914991:3915056 [0] NCCL INFO Channel 00/0 : 31[c7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3734971:3735040 [3] NCCL INFO Connected all rings +gpua057:3734971:3735040 [3] NCCL INFO Channel 00/0 : 47[c7000] -> 46[85000] via P2P/IPC/read +gpua057:3734971:3735040 [3] NCCL INFO Channel 01/0 : 47[c7000] -> 46[85000] via P2P/IPC/read +gpua057:3734971:3735040 [3] NCCL INFO Connected all trees +gpua057:3734971:3735040 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3734971:3735040 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3734971:3735040 [3] NCCL INFO comm 0x55e31e5a8260 rank 47 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua055:4159892:4159951 [1] NCCL INFO Connected all rings +gpua055:4159892:4159951 [1] NCCL INFO Channel 00/0 : 36[7000] -> 41[46000] [receive] via NET/AWS Libfabric/1 +gpua055:4159892:4159951 [1] NCCL INFO Channel 00/0 : 41[46000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua055:4159892:4159951 [1] NCCL INFO Channel 00/0 : 41[46000] -> 40[7000] via P2P/IPC/read +gpua055:4159892:4159951 [1] NCCL INFO Channel 01/0 : 41[46000] -> 40[7000] via P2P/IPC/read +gpua055:4159892:4159951 [1] NCCL INFO Connected all trees +gpua055:4159892:4159951 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:4159892:4159951 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:4159892:4159951 [1] NCCL INFO comm 0x5646e79ba420 rank 41 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua080:3124460:3124460 [0] NCCL INFO cudaDriverVersion 12020 +gpua080:3124460:3124460 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua080:3124460:3124460 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3124460:3124460 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3124460:3124522 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3124460:3124522 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3124460:3124522 [0] NCCL INFO Using network AWS Libfabric +gpua080:3124460:3124522 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua080:3124460:3124522 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3124460:3124522 [0] NCCL INFO Trees [0] 53/-1/-1->52->57 [1] 53/48/-1->52->45 +gpua080:3124460:3124522 [0] NCCL INFO Channel 00/0 : 51[c7000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2586755:2586755 [1] NCCL INFO cudaDriverVersion 12020 +gpua012:2586755:2586755 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua012:2586755:2586755 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2586755:2586755 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2586755:2586814 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2586755:2586814 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2586755:2586814 [1] NCCL INFO Using network AWS Libfabric +gpua012:2586755:2586814 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua012:2586755:2586814 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2586755:2586814 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/8/-1->5->4 +gpua012:2586755:2586814 [1] NCCL INFO Channel 00/0 : 5[46000] -> 6[85000] via P2P/IPC/read +gpua012:2586755:2586814 [1] NCCL INFO Channel 01/0 : 5[46000] -> 6[85000] via P2P/IPC/read +gpua033:1490879:1490879 [2] NCCL INFO cudaDriverVersion 12020 +gpua033:1490879:1490879 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua033:1490879:1490879 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1490879:1490879 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1490879:1490943 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1490879:1490943 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1490879:1490943 [2] NCCL INFO Using network AWS Libfabric +gpua033:1490879:1490943 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua033:1490879:1490943 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1490879:1490943 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 +gpua033:1490879:1490943 [2] NCCL INFO Channel 00/0 : 14[85000] -> 15[c7000] via P2P/IPC/read +gpua033:1490879:1490943 [2] NCCL INFO Channel 01/0 : 14[85000] -> 15[c7000] via P2P/IPC/read +gpua006:525537:525602 [1] NCCL INFO Channel 00/0 : 1[46000] -> 0[7000] via P2P/IPC/read +gpua006:525537:525602 [1] NCCL INFO Channel 01/0 : 1[46000] -> 0[7000] via P2P/IPC/read +gpua006:525537:525602 [1] NCCL INFO Connected all trees +gpua006:525537:525602 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:525537:525602 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua006:525537:525602 [1] NCCL INFO comm 0x56468c613260 rank 1 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua006:525539:525539 [3] NCCL INFO cudaDriverVersion 12020 +gpua006:525539:525539 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:525539:525539 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:525539:525539 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:525539:525601 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:525539:525601 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3508613:3508613 [3] NCCL INFO cudaDriverVersion 12020 +gpua039:3508613:3508613 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3508613:3508613 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3508613:3508613 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3508613:3508679 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3508613:3508679 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3508613:3508679 [3] NCCL INFO Using network AWS Libfabric +gpua039:3508613:3508679 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua039:3508613:3508679 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3508613:3508679 [3] NCCL INFO Trees [0] -1/-1/-1->23->22 [1] -1/-1/-1->23->22 +gpua039:3508613:3508679 [3] NCCL INFO Channel 00/0 : 23[c7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua039:3508613:3508679 [3] NCCL INFO Channel 01/0 : 23[c7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Channel 00/0 : 16[7000] -> 17[46000] via P2P/IPC/read +gpua038:25306:25375 [0] NCCL INFO Channel 01/0 : 16[7000] -> 17[46000] via P2P/IPC/read +gpua038:25306:25375 [0] NCCL INFO Connected all rings +gpua038:25306:25375 [0] NCCL INFO Channel 01/0 : 16[7000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Channel 00/0 : 16[7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Channel 00/0 : 16[7000] -> 33[46000] [send] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Channel 00/0 : 33[46000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Channel 00/0 : 24[7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Channel 01/0 : 20[7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua038:25306:25375 [0] NCCL INFO Connected all trees +gpua038:25306:25375 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3706125:3706125 [0] NCCL INFO cudaDriverVersion 12020 +gpua040:3706125:3706125 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3706125:3706125 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3706125:3706125 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3706125:3706196 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua040:3706125:3706196 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3706125:3706196 [0] NCCL INFO Using network AWS Libfabric +gpua040:3706125:3706196 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua040:3706125:3706196 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3706125:3706196 [0] NCCL INFO Trees [0] 25/28/-1->24->16 [1] 25/-1/-1->24->21 +gpua040:3706125:3706196 [0] NCCL INFO Channel 00/0 : 23[c7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3539995:3540053 [0] NCCL INFO Connected all trees +gpua079:3539995:3540053 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3539995:3540053 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3539995:3540053 [0] NCCL INFO comm 0x565177640e70 rank 48 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua049:3914991:3915056 [0] NCCL INFO Channel 01/0 : 31[c7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:3914991:3915056 [0] NCCL INFO Channel 00/0 : 32[7000] -> 33[46000] via P2P/IPC/read +gpua049:3914991:3915056 [0] NCCL INFO Channel 01/0 : 32[7000] -> 33[46000] via P2P/IPC/read +gpua049:3914991:3915056 [0] NCCL INFO Connected all rings +gpua049:3914991:3915056 [0] NCCL INFO Channel 01/0 : 32[7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua049:3914991:3915056 [0] NCCL INFO Channel 00/0 : 32[7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua049:3914991:3915056 [0] NCCL INFO Channel 00/0 : 0[7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:3914991:3915056 [0] NCCL INFO Channel 00/0 : 32[7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua049:3914991:3915056 [0] NCCL INFO Channel 00/0 : 48[7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:3914991:3915056 [0] NCCL INFO Channel 01/0 : 36[7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3734970:3734970 [2] NCCL INFO cudaDriverVersion 12020 +gpua057:3734970:3734970 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3734970:3734970 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3734970:3734970 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3734970:3735043 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3734970:3735043 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3734970:3735043 [2] NCCL INFO Using network AWS Libfabric +gpua057:3734970:3735043 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua057:3734970:3735043 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3734970:3735043 [2] NCCL INFO Trees [0] 47/-1/-1->46->45 [1] 47/-1/-1->46->45 +gpua057:3734970:3735043 [2] NCCL INFO Channel 00/0 : 46[85000] -> 47[c7000] via P2P/IPC/read +gpua057:3734970:3735043 [2] NCCL INFO Channel 01/0 : 46[85000] -> 47[c7000] via P2P/IPC/read +gpua055:4159893:4159893 [2] NCCL INFO cudaDriverVersion 12020 +gpua055:4159893:4159893 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:4159893:4159893 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:4159893:4159893 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:4159893:4159953 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua055:4159893:4159953 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:4159893:4159953 [2] NCCL INFO Using network AWS Libfabric +gpua055:4159893:4159953 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua055:4159893:4159953 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:4159893:4159953 [2] NCCL INFO Trees [0] 43/-1/-1->42->41 [1] 43/-1/-1->42->41 +gpua055:4159893:4159953 [2] NCCL INFO Channel 00/0 : 42[85000] -> 43[c7000] via P2P/IPC/read +gpua055:4159893:4159953 [2] NCCL INFO Channel 01/0 : 42[85000] -> 43[c7000] via P2P/IPC/read +gpua080:3124460:3124522 [0] NCCL INFO Channel 01/0 : 51[c7000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3124460:3124522 [0] NCCL INFO Channel 00/0 : 52[7000] -> 53[46000] via P2P/IPC/read +gpua080:3124460:3124522 [0] NCCL INFO Channel 01/0 : 52[7000] -> 53[46000] via P2P/IPC/read +gpua080:3124460:3124522 [0] NCCL INFO Connected all rings +gpua080:3124460:3124522 [0] NCCL INFO Channel 01/0 : 48[7000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3124460:3124522 [0] NCCL INFO Channel 00/0 : 52[7000] -> 57[46000] [send] via NET/AWS Libfabric/1 +gpua080:3124460:3124522 [0] NCCL INFO Channel 01/0 : 45[46000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3124460:3124522 [0] NCCL INFO Channel 01/0 : 52[7000] -> 45[46000] [send] via NET/AWS Libfabric/1 +gpua080:3124460:3124522 [0] NCCL INFO Channel 00/0 : 57[46000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3124460:3124522 [0] NCCL INFO Channel 01/0 : 52[7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua012:2586755:2586814 [1] NCCL INFO Connected all rings +gpua012:2586755:2586814 [1] NCCL INFO Channel 01/0 : 5[46000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua012:2586755:2586814 [1] NCCL INFO Channel 01/0 : 8[7000] -> 5[46000] [receive] via NET/AWS Libfabric/1 +gpua012:2586755:2586814 [1] NCCL INFO Channel 00/0 : 5[46000] -> 4[7000] via P2P/IPC/read +gpua012:2586755:2586814 [1] NCCL INFO Channel 01/0 : 5[46000] -> 4[7000] via P2P/IPC/read +gpua012:2586755:2586814 [1] NCCL INFO Connected all trees +gpua012:2586755:2586814 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua012:2586755:2586814 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2586755:2586814 [1] NCCL INFO comm 0x55e407d7ad70 rank 5 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua012:2586756:2586756 [2] NCCL INFO cudaDriverVersion 12020 +gpua012:2586756:2586756 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua033:1490879:1490943 [2] NCCL INFO Connected all rings +gpua033:1490879:1490943 [2] NCCL INFO Channel 00/0 : 14[85000] -> 13[46000] via P2P/IPC/read +gpua033:1490879:1490943 [2] NCCL INFO Channel 01/0 : 14[85000] -> 13[46000] via P2P/IPC/read +gpua033:1490879:1490943 [2] NCCL INFO Connected all trees +gpua033:1490879:1490943 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1490879:1490943 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1490879:1490943 [2] NCCL INFO comm 0x55a2e150b9c0 rank 14 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua006:525539:525601 [3] NCCL INFO Using network AWS Libfabric +gpua006:525539:525601 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua006:525539:525601 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:525539:525601 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 +gpua006:525539:525601 [3] NCCL INFO Channel 00/0 : 3[c7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua006:525539:525601 [3] NCCL INFO Channel 01/0 : 3[c7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua006:525539:525601 [3] NCCL INFO Connected all rings +gpua006:525539:525601 [3] NCCL INFO Channel 00/0 : 3[c7000] -> 2[85000] via P2P/IPC/read +gpua006:525539:525601 [3] NCCL INFO Channel 01/0 : 3[c7000] -> 2[85000] via P2P/IPC/read +gpua006:525539:525601 [3] NCCL INFO Connected all trees +gpua006:525539:525601 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:525539:525601 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3508613:3508679 [3] NCCL INFO Connected all rings +gpua039:3508613:3508679 [3] NCCL INFO Channel 00/0 : 23[c7000] -> 22[85000] via P2P/IPC/read +gpua039:3508613:3508679 [3] NCCL INFO Channel 01/0 : 23[c7000] -> 22[85000] via P2P/IPC/read +gpua039:3508613:3508679 [3] NCCL INFO Connected all trees +gpua039:3508613:3508679 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3508613:3508679 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3508613:3508679 [3] NCCL INFO comm 0x55f42bb22550 rank 23 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua038:25306:25375 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:25306:25375 [0] NCCL INFO comm 0x55a58d6bdc10 rank 16 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua040:3706125:3706196 [0] NCCL INFO Channel 01/0 : 23[c7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3706125:3706196 [0] NCCL INFO Channel 00/0 : 24[7000] -> 25[46000] via P2P/IPC/read +gpua040:3706125:3706196 [0] NCCL INFO Channel 01/0 : 24[7000] -> 25[46000] via P2P/IPC/read +gpua040:3706125:3706196 [0] NCCL INFO Connected all rings +gpua040:3706125:3706196 [0] NCCL INFO Channel 01/0 : 21[46000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3706125:3706196 [0] NCCL INFO Channel 00/0 : 24[7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua040:3706125:3706196 [0] NCCL INFO Channel 00/0 : 16[7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3706125:3706196 [0] NCCL INFO Channel 00/0 : 24[7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua040:3706125:3706196 [0] NCCL INFO Channel 00/0 : 28[7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3706125:3706196 [0] NCCL INFO Channel 01/0 : 24[7000] -> 21[46000] [send] via NET/AWS Libfabric/1 +gpua079:3539997:3539997 [2] NCCL INFO cudaDriverVersion 12020 +gpua079:3539997:3539997 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3539997:3539997 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3539997:3539997 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3539997:3540055 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3539997:3540055 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3539997:3540055 [2] NCCL INFO Using network AWS Libfabric +gpua079:3539997:3540055 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua079:3539997:3540055 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3539997:3540055 [2] NCCL INFO Trees [0] 51/-1/-1->50->49 [1] 51/-1/-1->50->49 +gpua079:3539997:3540055 [2] NCCL INFO Channel 00/0 : 50[85000] -> 51[c7000] via P2P/IPC/read +gpua079:3539997:3540055 [2] NCCL INFO Channel 01/0 : 50[85000] -> 51[c7000] via P2P/IPC/read +gpua049:3914991:3915056 [0] NCCL INFO Connected all trees +gpua049:3914991:3915056 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:3914991:3915056 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:3914991:3915056 [0] NCCL INFO comm 0x5642b20140d0 rank 32 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua057:3734970:3735043 [2] NCCL INFO Connected all rings +gpua057:3734970:3735043 [2] NCCL INFO Channel 00/0 : 46[85000] -> 45[46000] via P2P/IPC/read +gpua057:3734970:3735043 [2] NCCL INFO Channel 01/0 : 46[85000] -> 45[46000] via P2P/IPC/read +gpua057:3734970:3735043 [2] NCCL INFO Connected all trees +gpua057:3734970:3735043 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3734970:3735043 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3734970:3735043 [2] NCCL INFO comm 0x55ac0a9c9ad0 rank 46 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua055:4159893:4159953 [2] NCCL INFO Connected all rings +gpua055:4159893:4159953 [2] NCCL INFO Channel 00/0 : 42[85000] -> 41[46000] via P2P/IPC/read +gpua055:4159893:4159953 [2] NCCL INFO Channel 01/0 : 42[85000] -> 41[46000] via P2P/IPC/read +gpua055:4159893:4159953 [2] NCCL INFO Connected all trees +gpua055:4159893:4159953 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:4159893:4159953 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:4159893:4159953 [2] NCCL INFO comm 0x55dd15014a60 rank 42 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua055:4159891:4159891 [0] NCCL INFO cudaDriverVersion 12020 +gpua055:4159891:4159891 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:4159891:4159891 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:4159891:4159891 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:4159891:4159952 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3124460:3124522 [0] NCCL INFO Connected all trees +gpua080:3124460:3124522 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua080:3124460:3124522 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3124460:3124522 [0] NCCL INFO comm 0x55697dfdf690 rank 52 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua012:2586756:2586756 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2586756:2586756 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2586756:2586815 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2586756:2586815 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2586756:2586815 [2] NCCL INFO Using network AWS Libfabric +gpua012:2586756:2586815 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua012:2586756:2586815 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2586756:2586815 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 +gpua012:2586756:2586815 [2] NCCL INFO Channel 00/0 : 6[85000] -> 7[c7000] via P2P/IPC/read +gpua012:2586756:2586815 [2] NCCL INFO Channel 01/0 : 6[85000] -> 7[c7000] via P2P/IPC/read +gpua012:2586756:2586815 [2] NCCL INFO Connected all rings +gpua012:2586756:2586815 [2] NCCL INFO Channel 00/0 : 6[85000] -> 5[46000] via P2P/IPC/read +gpua006:525539:525601 [3] NCCL INFO comm 0x55d0174dc9b0 rank 3 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua038:25309:25309 [3] NCCL INFO cudaDriverVersion 12020 +gpua038:25309:25309 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:25309:25309 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:25309:25309 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:25309:25376 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:25309:25376 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:25309:25376 [3] NCCL INFO Using network AWS Libfabric +gpua038:25309:25376 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua038:25309:25376 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:25309:25376 [3] NCCL INFO Trees [0] -1/-1/-1->19->18 [1] -1/-1/-1->19->18 +gpua038:25309:25376 [3] NCCL INFO Channel 00/0 : 19[c7000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua038:25309:25376 [3] NCCL INFO Channel 01/0 : 19[c7000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua038:25309:25376 [3] NCCL INFO Connected all rings +gpua040:3706125:3706196 [0] NCCL INFO Connected all trees +gpua040:3706125:3706196 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3706125:3706196 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3706125:3706196 [0] NCCL INFO comm 0x55b658387aa0 rank 24 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua079:3539997:3540055 [2] NCCL INFO Connected all rings +gpua079:3539997:3540055 [2] NCCL INFO Channel 00/0 : 50[85000] -> 49[46000] via P2P/IPC/read +gpua079:3539997:3540055 [2] NCCL INFO Channel 01/0 : 50[85000] -> 49[46000] via P2P/IPC/read +gpua079:3539997:3540055 [2] NCCL INFO Connected all trees +gpua079:3539997:3540055 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3539997:3540055 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3539997:3540055 [2] NCCL INFO comm 0x5651ab258a70 rank 50 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua049:3914994:3914994 [3] NCCL INFO cudaDriverVersion 12020 +gpua049:3914994:3914994 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua049:3914994:3914994 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:3914994:3914994 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:3914994:3915057 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:3914994:3915057 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:3914994:3915057 [3] NCCL INFO Using network AWS Libfabric +gpua049:3914994:3915057 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua049:3914994:3915057 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:3914994:3915057 [3] NCCL INFO Trees [0] -1/-1/-1->35->34 [1] -1/-1/-1->35->34 +gpua049:3914994:3915057 [3] NCCL INFO Channel 00/0 : 35[c7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua049:3914994:3915057 [3] NCCL INFO Channel 01/0 : 35[c7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:4159891:4159952 [0] NCCL INFO Using network AWS Libfabric +gpua055:4159891:4159952 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua055:4159891:4159952 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:4159891:4159952 [0] NCCL INFO Trees [0] 41/44/-1->40->49 [1] 41/-1/-1->40->37 +gpua055:4159891:4159952 [0] NCCL INFO Channel 00/0 : 39[c7000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO Channel 01/0 : 39[c7000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO Channel 00/0 : 40[7000] -> 41[46000] via P2P/IPC/read +gpua055:4159891:4159952 [0] NCCL INFO Channel 01/0 : 40[7000] -> 41[46000] via P2P/IPC/read +gpua055:4159891:4159952 [0] NCCL INFO Connected all rings +gpua055:4159891:4159952 [0] NCCL INFO Channel 01/0 : 37[46000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3124461:3124461 [1] NCCL INFO cudaDriverVersion 12020 +gpua080:3124461:3124461 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua080:3124461:3124461 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3124461:3124461 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3124461:3124524 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3124461:3124524 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3124461:3124524 [1] NCCL INFO Using network AWS Libfabric +gpua080:3124461:3124524 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua080:3124461:3124524 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3124461:3124524 [1] NCCL INFO Trees [0] 54/-1/-1->53->52 [1] 54/56/-1->53->52 +gpua080:3124461:3124524 [1] NCCL INFO Channel 00/0 : 53[46000] -> 54[85000] via P2P/IPC/read +gpua080:3124461:3124524 [1] NCCL INFO Channel 01/0 : 53[46000] -> 54[85000] via P2P/IPC/read +gpua012:2586756:2586815 [2] NCCL INFO Channel 01/0 : 6[85000] -> 5[46000] via P2P/IPC/read +gpua012:2586756:2586815 [2] NCCL INFO Connected all trees +gpua012:2586756:2586815 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua012:2586756:2586815 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2586756:2586815 [2] NCCL INFO comm 0x5566187280d0 rank 6 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua038:25309:25376 [3] NCCL INFO Channel 00/0 : 19[c7000] -> 18[85000] via P2P/IPC/read +gpua038:25309:25376 [3] NCCL INFO Channel 01/0 : 19[c7000] -> 18[85000] via P2P/IPC/read +gpua038:25309:25376 [3] NCCL INFO Connected all trees +gpua038:25309:25376 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua038:25309:25376 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:25309:25376 [3] NCCL INFO comm 0x55f0c716b2a0 rank 19 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua049:3914994:3915057 [3] NCCL INFO Connected all rings +gpua049:3914994:3915057 [3] NCCL INFO Channel 00/0 : 35[c7000] -> 34[85000] via P2P/IPC/read +gpua049:3914994:3915057 [3] NCCL INFO Channel 01/0 : 35[c7000] -> 34[85000] via P2P/IPC/read +gpua049:3914994:3915057 [3] NCCL INFO Connected all trees +gpua049:3914994:3915057 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:3914994:3915057 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:3914994:3915057 [3] NCCL INFO comm 0x556528b90790 rank 35 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua055:4159891:4159952 [0] NCCL INFO Channel 00/0 : 40[7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO Channel 00/0 : 40[7000] -> 49[46000] [send] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO Channel 00/0 : 49[46000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO Channel 00/0 : 44[7000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO Channel 01/0 : 40[7000] -> 37[46000] [send] via NET/AWS Libfabric/1 +gpua055:4159891:4159952 [0] NCCL INFO Connected all trees +gpua055:4159891:4159952 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:4159891:4159952 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:4159891:4159952 [0] NCCL INFO comm 0x560174973560 rank 40 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua080:3124461:3124524 [1] NCCL INFO Connected all rings +gpua080:3124461:3124524 [1] NCCL INFO Channel 01/0 : 53[46000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua080:3124461:3124524 [1] NCCL INFO Channel 01/0 : 56[7000] -> 53[46000] [receive] via NET/AWS Libfabric/1 +gpua080:3124461:3124524 [1] NCCL INFO Channel 00/0 : 53[46000] -> 52[7000] via P2P/IPC/read +gpua080:3124461:3124524 [1] NCCL INFO Channel 01/0 : 53[46000] -> 52[7000] via P2P/IPC/read +gpua080:3124461:3124524 [1] NCCL INFO Connected all trees +gpua080:3124461:3124524 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua080:3124461:3124524 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3124461:3124524 [1] NCCL INFO comm 0x5581f0e78ff0 rank 53 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua012:2586754:2586754 [0] NCCL INFO cudaDriverVersion 12020 +gpua012:2586754:2586754 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua012:2586754:2586754 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2586754:2586754 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2586754:2586816 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2586754:2586816 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2586754:2586816 [0] NCCL INFO Using network AWS Libfabric +gpua012:2586754:2586816 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua012:2586754:2586816 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2586754:2586816 [0] NCCL INFO Trees [0] 5/-1/-1->4->9 [1] 5/0/-1->4->12 +gpua012:2586754:2586816 [0] NCCL INFO Channel 00/0 : 3[c7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2586754:2586816 [0] NCCL INFO Channel 01/0 : 3[c7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua038:25308:25308 [2] NCCL INFO cudaDriverVersion 12020 +gpua038:25308:25308 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:25308:25308 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:25308:25308 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:25308:25374 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:25308:25374 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:25308:25374 [2] NCCL INFO Using network AWS Libfabric +gpua038:25308:25374 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua038:25308:25374 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:25308:25374 [2] NCCL INFO Trees [0] 19/-1/-1->18->17 [1] 19/-1/-1->18->17 +gpua038:25308:25374 [2] NCCL INFO Channel 00/0 : 18[85000] -> 19[c7000] via P2P/IPC/read +gpua038:25308:25374 [2] NCCL INFO Channel 01/0 : 18[85000] -> 19[c7000] via P2P/IPC/read +gpua038:25308:25374 [2] NCCL INFO Connected all rings +gpua080:3124462:3124462 [2] NCCL INFO cudaDriverVersion 12020 +gpua080:3124462:3124462 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua080:3124462:3124462 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3124462:3124462 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3124462:3124523 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3124462:3124523 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3124462:3124523 [2] NCCL INFO Using network AWS Libfabric +gpua080:3124462:3124523 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua080:3124462:3124523 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3124462:3124523 [2] NCCL INFO Trees [0] 55/-1/-1->54->53 [1] 55/-1/-1->54->53 +gpua080:3124462:3124523 [2] NCCL INFO Channel 00/0 : 54[85000] -> 55[c7000] via P2P/IPC/read +gpua080:3124462:3124523 [2] NCCL INFO Channel 01/0 : 54[85000] -> 55[c7000] via P2P/IPC/read +gpua012:2586754:2586816 [0] NCCL INFO Channel 00/0 : 4[7000] -> 5[46000] via P2P/IPC/read +gpua012:2586754:2586816 [0] NCCL INFO Channel 01/0 : 4[7000] -> 5[46000] via P2P/IPC/read +gpua012:2586754:2586816 [0] NCCL INFO Connected all rings +gpua012:2586754:2586816 [0] NCCL INFO Channel 01/0 : 0[7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2586754:2586816 [0] NCCL INFO Channel 00/0 : 4[7000] -> 9[46000] [send] via NET/AWS Libfabric/1 +gpua012:2586754:2586816 [0] NCCL INFO Channel 01/0 : 4[7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua012:2586754:2586816 [0] NCCL INFO Channel 01/0 : 12[7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2586754:2586816 [0] NCCL INFO Channel 00/0 : 9[46000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2586754:2586816 [0] NCCL INFO Channel 01/0 : 4[7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua012:2586754:2586816 [0] NCCL INFO Connected all trees +gpua012:2586754:2586816 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua038:25308:25374 [2] NCCL INFO Channel 00/0 : 18[85000] -> 17[46000] via P2P/IPC/read +gpua038:25308:25374 [2] NCCL INFO Channel 01/0 : 18[85000] -> 17[46000] via P2P/IPC/read +gpua038:25308:25374 [2] NCCL INFO Connected all trees +gpua038:25308:25374 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua038:25308:25374 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:25308:25374 [2] NCCL INFO comm 0x55a4ae470ea0 rank 18 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua080:3124462:3124523 [2] NCCL INFO Connected all rings +gpua080:3124462:3124523 [2] NCCL INFO Channel 00/0 : 54[85000] -> 53[46000] via P2P/IPC/read +gpua080:3124462:3124523 [2] NCCL INFO Channel 01/0 : 54[85000] -> 53[46000] via P2P/IPC/read +gpua080:3124462:3124523 [2] NCCL INFO Connected all trees +gpua080:3124462:3124523 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua080:3124462:3124523 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3124462:3124523 [2] NCCL INFO comm 0x562090a02b70 rank 54 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua012:2586754:2586816 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2586754:2586816 [0] NCCL INFO comm 0x55bd123206a0 rank 4 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua012:2586757:2586757 [3] NCCL INFO cudaDriverVersion 12020 +gpua012:2586757:2586757 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua012:2586757:2586757 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2586757:2586757 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2586757:2586813 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2586757:2586813 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2586757:2586813 [3] NCCL INFO Using network AWS Libfabric +gpua012:2586757:2586813 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua012:2586757:2586813 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2586757:2586813 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 +gpua012:2586757:2586813 [3] NCCL INFO Channel 00/0 : 7[c7000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua012:2586757:2586813 [3] NCCL INFO Channel 01/0 : 7[c7000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua012:2586757:2586813 [3] NCCL INFO Connected all rings +gpua012:2586757:2586813 [3] NCCL INFO Channel 00/0 : 7[c7000] -> 6[85000] via P2P/IPC/read +gpua012:2586757:2586813 [3] NCCL INFO Channel 01/0 : 7[c7000] -> 6[85000] via P2P/IPC/read +gpua012:2586757:2586813 [3] NCCL INFO Connected all trees +gpua012:2586757:2586813 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua012:2586757:2586813 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2586757:2586813 [3] NCCL INFO comm 0x5641970329e0 rank 7 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua049:3914993:3914993 [2] NCCL INFO cudaDriverVersion 12020 +gpua049:3914993:3914993 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua049:3914993:3914993 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:3914993:3914993 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:3914993:3915059 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:3914993:3915059 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:3914993:3915059 [2] NCCL INFO Using network AWS Libfabric +gpua049:3914993:3915059 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua049:3914993:3915059 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:3914993:3915059 [2] NCCL INFO Trees [0] 35/-1/-1->34->33 [1] 35/-1/-1->34->33 +gpua049:3914993:3915059 [2] NCCL INFO Channel 00/0 : 34[85000] -> 35[c7000] via P2P/IPC/read +gpua049:3914993:3915059 [2] NCCL INFO Channel 01/0 : 34[85000] -> 35[c7000] via P2P/IPC/read +gpua049:3914993:3915059 [2] NCCL INFO Connected all rings +gpua049:3914993:3915059 [2] NCCL INFO Channel 00/0 : 34[85000] -> 33[46000] via P2P/IPC/read +gpua049:3914993:3915059 [2] NCCL INFO Channel 01/0 : 34[85000] -> 33[46000] via P2P/IPC/read +gpua049:3914993:3915059 [2] NCCL INFO Connected all trees +gpua049:3914993:3915059 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:3914993:3915059 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:3914993:3915059 [2] NCCL INFO comm 0x563ec6af02a0 rank 34 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua042:92809:92809 [2] NCCL INFO cudaDriverVersion 12020 +gpua042:92809:92809 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.42<0> +gpua042:92809:92809 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua042:92809:92809 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua042:92809:92873 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua042:92809:92873 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua042:92809:92873 [2] NCCL INFO Using network AWS Libfabric +gpua042:92809:92873 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua042:92809:92873 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua042:92809:92873 [2] NCCL INFO Trees [0] 31/-1/-1->30->29 [1] 31/-1/-1->30->29 +gpua042:92809:92873 [2] NCCL INFO Channel 00/0 : 30[85000] -> 31[c7000] via P2P/IPC/read +gpua042:92809:92873 [2] NCCL INFO Channel 01/0 : 30[85000] -> 31[c7000] via P2P/IPC/read +gpua042:92809:92873 [2] NCCL INFO Connected all rings +gpua006:525536:525599 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:525536:525599 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:525536:525599 [0] NCCL INFO Using network AWS Libfabric +gpua006:525536:525599 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua006:525536:525599 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:525536:525599 [0] NCCL INFO Channel 00/02 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 +gpua006:525536:525599 [0] NCCL INFO Channel 01/02 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 +gpua006:525536:525599 [0] NCCL INFO Trees [0] 1/32/-1->0->-1 [1] 1/-1/-1->0->4 +gpua006:525536:525599 [0] NCCL INFO Channel 00/0 : 63[c7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:525536:525599 [0] NCCL INFO Channel 01/0 : 63[c7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:525536:525599 [0] NCCL INFO Channel 00/0 : 0[7000] -> 1[46000] via P2P/IPC/read +gpua042:92809:92873 [2] NCCL INFO Channel 00/0 : 30[85000] -> 29[46000] via P2P/IPC/read +gpua042:92809:92873 [2] NCCL INFO Channel 01/0 : 30[85000] -> 29[46000] via P2P/IPC/read +gpua042:92809:92873 [2] NCCL INFO Connected all trees +gpua042:92809:92873 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua042:92809:92873 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua042:92809:92873 [2] NCCL INFO comm 0x55e7082c30d0 rank 30 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua006:525536:525599 [0] NCCL INFO Channel 01/0 : 0[7000] -> 1[46000] via P2P/IPC/read +gpua006:525536:525599 [0] NCCL INFO Connected all rings +gpua006:525536:525599 [0] NCCL INFO Channel 01/0 : 0[7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua006:525536:525599 [0] NCCL INFO Channel 00/0 : 32[7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:525536:525599 [0] NCCL INFO Channel 00/0 : 0[7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua006:525536:525599 [0] NCCL INFO Channel 01/0 : 4[7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:525536:525599 [0] NCCL INFO Connected all trees +gpua006:525536:525599 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:525536:525599 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua006:525536:525599 [0] NCCL INFO comm 0x55dc09701eb0 rank 0 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua042:92808:92808 [1] NCCL INFO cudaDriverVersion 12020 +gpua042:92808:92808 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.42<0> +gpua042:92808:92808 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua042:92808:92808 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua042:92808:92874 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua042:92808:92874 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua042:92808:92874 [1] NCCL INFO Using network AWS Libfabric +gpua042:92808:92874 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua042:92808:92874 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua042:92808:92874 [1] NCCL INFO Trees [0] 30/-1/-1->29->28 [1] 30/44/-1->29->28 +gpua042:92808:92874 [1] NCCL INFO Channel 00/0 : 29[46000] -> 30[85000] via P2P/IPC/read +gpua042:92808:92874 [1] NCCL INFO Channel 01/0 : 29[46000] -> 30[85000] via P2P/IPC/read +gpua042:92808:92874 [1] NCCL INFO Connected all rings +gpua039:3508612:3508612 [2] NCCL INFO cudaDriverVersion 12020 +gpua039:3508612:3508612 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3508612:3508612 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3508612:3508612 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3508612:3508677 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3508612:3508677 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3508612:3508677 [2] NCCL INFO Using network AWS Libfabric +gpua039:3508612:3508677 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua039:3508612:3508677 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3508612:3508677 [2] NCCL INFO Trees [0] 23/-1/-1->22->21 [1] 23/-1/-1->22->21 +gpua039:3508612:3508677 [2] NCCL INFO Channel 00/0 : 22[85000] -> 23[c7000] via P2P/IPC/read +gpua039:3508612:3508677 [2] NCCL INFO Channel 01/0 : 22[85000] -> 23[c7000] via P2P/IPC/read +gpua042:92808:92874 [1] NCCL INFO Channel 01/0 : 29[46000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua042:92808:92874 [1] NCCL INFO Channel 01/0 : 44[7000] -> 29[46000] [receive] via NET/AWS Libfabric/1 +gpua042:92808:92874 [1] NCCL INFO Channel 00/0 : 29[46000] -> 28[7000] via P2P/IPC/read +gpua042:92808:92874 [1] NCCL INFO Channel 01/0 : 29[46000] -> 28[7000] via P2P/IPC/read +gpua042:92808:92874 [1] NCCL INFO Connected all trees +gpua042:92808:92874 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua042:92808:92874 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua042:92808:92874 [1] NCCL INFO comm 0x56104bc4b250 rank 29 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua039:3508612:3508677 [2] NCCL INFO Connected all rings +gpua039:3508612:3508677 [2] NCCL INFO Channel 00/0 : 22[85000] -> 21[46000] via P2P/IPC/read +gpua039:3508612:3508677 [2] NCCL INFO Channel 01/0 : 22[85000] -> 21[46000] via P2P/IPC/read +gpua039:3508612:3508677 [2] NCCL INFO Connected all trees +gpua039:3508612:3508677 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3508612:3508677 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3508612:3508677 [2] NCCL INFO comm 0x56111c8d3be0 rank 22 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua054:266918:266918 [0] NCCL INFO cudaDriverVersion 12020 +gpua054:266918:266918 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:266918:266918 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua054:266918:266918 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:266918:266991 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:266918:266991 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:266918:266991 [0] NCCL INFO Using network AWS Libfabric +gpua054:266918:266991 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua054:266918:266991 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:266918:266991 [0] NCCL INFO Trees [0] 37/-1/-1->36->41 [1] 37/32/-1->36->44 +gpua054:266918:266991 [0] NCCL INFO Channel 00/0 : 35[c7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Channel 01/0 : 35[c7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Channel 00/0 : 36[7000] -> 37[46000] via P2P/IPC/read +gpua054:266918:266991 [0] NCCL INFO Channel 01/0 : 36[7000] -> 37[46000] via P2P/IPC/read +gpua054:266918:266991 [0] NCCL INFO Connected all rings +gpua054:266918:266991 [0] NCCL INFO Channel 01/0 : 32[7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Channel 00/0 : 36[7000] -> 41[46000] [send] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Channel 01/0 : 36[7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Channel 01/0 : 44[7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Channel 00/0 : 41[46000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Channel 01/0 : 36[7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua054:266918:266991 [0] NCCL INFO Connected all trees +gpua054:266918:266991 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:266918:266991 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:266918:266991 [0] NCCL INFO comm 0x563e53b4b210 rank 36 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua039:3508611:3508611 [1] NCCL INFO cudaDriverVersion 12020 +gpua039:3508611:3508611 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3508611:3508611 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3508611:3508611 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3508611:3508678 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3508611:3508678 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3508611:3508678 [1] NCCL INFO Using network AWS Libfabric +gpua039:3508611:3508678 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua039:3508611:3508678 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3508611:3508678 [1] NCCL INFO Trees [0] 22/-1/-1->21->20 [1] 22/24/-1->21->20 +gpua039:3508611:3508678 [1] NCCL INFO Channel 00/0 : 21[46000] -> 22[85000] via P2P/IPC/read +gpua039:3508611:3508678 [1] NCCL INFO Channel 01/0 : 21[46000] -> 22[85000] via P2P/IPC/read +gpua039:3508611:3508678 [1] NCCL INFO Connected all rings +gpua039:3508611:3508678 [1] NCCL INFO Channel 01/0 : 21[46000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua039:3508611:3508678 [1] NCCL INFO Channel 01/0 : 24[7000] -> 21[46000] [receive] via NET/AWS Libfabric/1 +gpua039:3508611:3508678 [1] NCCL INFO Channel 00/0 : 21[46000] -> 20[7000] via P2P/IPC/read +gpua039:3508611:3508678 [1] NCCL INFO Channel 01/0 : 21[46000] -> 20[7000] via P2P/IPC/read +gpua039:3508611:3508678 [1] NCCL INFO Connected all trees +gpua039:3508611:3508678 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3508611:3508678 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3508611:3508678 [1] NCCL INFO comm 0x56324b0720d0 rank 21 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua039:3508610:3508610 [0] NCCL INFO cudaDriverVersion 12020 +gpua039:3508610:3508610 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3508610:3508610 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3508610:3508610 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3508610:3508680 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3508610:3508680 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3508610:3508680 [0] NCCL INFO Using network AWS Libfabric +gpua039:3508610:3508680 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua039:3508610:3508680 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3508610:3508680 [0] NCCL INFO Trees [0] 21/-1/-1->20->25 [1] 21/16/-1->20->13 +gpua039:3508610:3508680 [0] NCCL INFO Channel 00/0 : 19[c7000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Channel 01/0 : 19[c7000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Channel 00/0 : 20[7000] -> 21[46000] via P2P/IPC/read +gpua039:3508610:3508680 [0] NCCL INFO Channel 01/0 : 20[7000] -> 21[46000] via P2P/IPC/read +gpua039:3508610:3508680 [0] NCCL INFO Connected all rings +gpua039:3508610:3508680 [0] NCCL INFO Channel 01/0 : 16[7000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Channel 00/0 : 20[7000] -> 25[46000] [send] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Channel 01/0 : 13[46000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Channel 01/0 : 20[7000] -> 13[46000] [send] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Channel 00/0 : 25[46000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Channel 01/0 : 20[7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua039:3508610:3508680 [0] NCCL INFO Connected all trees +gpua039:3508610:3508680 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3508610:3508680 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3508610:3508680 [0] NCCL INFO comm 0x5604af6e8ce0 rank 20 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua054:266920:266920 [2] NCCL INFO cudaDriverVersion 12020 +gpua054:266920:266920 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:266920:266920 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua054:266920:266920 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:266920:266989 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:266920:266989 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:266920:266989 [2] NCCL INFO Using network AWS Libfabric +gpua054:266920:266989 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua054:266920:266989 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:266920:266989 [2] NCCL INFO Trees [0] 39/-1/-1->38->37 [1] 39/-1/-1->38->37 +gpua054:266920:266989 [2] NCCL INFO Channel 00/0 : 38[85000] -> 39[c7000] via P2P/IPC/read +gpua054:266920:266989 [2] NCCL INFO Channel 01/0 : 38[85000] -> 39[c7000] via P2P/IPC/read +gpua054:266920:266989 [2] NCCL INFO Connected all rings +gpua054:266920:266989 [2] NCCL INFO Channel 00/0 : 38[85000] -> 37[46000] via P2P/IPC/read +gpua054:266920:266989 [2] NCCL INFO Channel 01/0 : 38[85000] -> 37[46000] via P2P/IPC/read +gpua054:266920:266989 [2] NCCL INFO Connected all trees +gpua054:266920:266989 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:266920:266989 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:266920:266989 [2] NCCL INFO comm 0x55937e44cc50 rank 38 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua085:3864150:3864150 [0] NCCL INFO cudaDriverVersion 12020 +gpua085:3864150:3864150 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:3864150:3864150 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:3864150:3864150 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:3864150:3864209 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua085:3864150:3864209 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:3864150:3864209 [0] NCCL INFO Using network AWS Libfabric +gpua085:3864150:3864209 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua085:3864150:3864209 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:3864150:3864209 [0] NCCL INFO Trees [0] 57/60/-1->56->48 [1] 57/-1/-1->56->53 +gpua085:3864150:3864209 [0] NCCL INFO Channel 00/0 : 55[c7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Channel 01/0 : 55[c7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Channel 00/0 : 56[7000] -> 57[46000] via P2P/IPC/read +gpua085:3864150:3864209 [0] NCCL INFO Channel 01/0 : 56[7000] -> 57[46000] via P2P/IPC/read +gpua085:3864150:3864209 [0] NCCL INFO Connected all rings +gpua085:3864150:3864209 [0] NCCL INFO Channel 01/0 : 53[46000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Channel 00/0 : 56[7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Channel 00/0 : 48[7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Channel 00/0 : 56[7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Channel 00/0 : 60[7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Channel 01/0 : 56[7000] -> 53[46000] [send] via NET/AWS Libfabric/1 +gpua085:3864150:3864209 [0] NCCL INFO Connected all trees +gpua085:3864150:3864209 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:3864150:3864209 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:3864150:3864209 [0] NCCL INFO comm 0x55f8c00a4150 rank 56 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua085:3864153:3864153 [3] NCCL INFO cudaDriverVersion 12020 +gpua085:3864153:3864153 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:3864153:3864153 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:3864153:3864153 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:3864153:3864210 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua085:3864153:3864210 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:3864153:3864210 [3] NCCL INFO Using network AWS Libfabric +gpua085:3864153:3864210 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua085:3864153:3864210 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:3864153:3864210 [3] NCCL INFO Trees [0] -1/-1/-1->59->58 [1] -1/-1/-1->59->58 +gpua085:3864153:3864210 [3] NCCL INFO Channel 00/0 : 59[c7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua085:3864153:3864210 [3] NCCL INFO Channel 01/0 : 59[c7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua085:3864153:3864210 [3] NCCL INFO Connected all rings +gpua085:3864153:3864210 [3] NCCL INFO Channel 00/0 : 59[c7000] -> 58[85000] via P2P/IPC/read +gpua085:3864153:3864210 [3] NCCL INFO Channel 01/0 : 59[c7000] -> 58[85000] via P2P/IPC/read +gpua085:3864153:3864210 [3] NCCL INFO Connected all trees +gpua085:3864153:3864210 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:3864153:3864210 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:3864153:3864210 [3] NCCL INFO comm 0x558869845020 rank 59 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua085:3864152:3864152 [2] NCCL INFO cudaDriverVersion 12020 +gpua085:3864152:3864152 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:3864152:3864152 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:3864152:3864152 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:3864152:3864211 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua085:3864152:3864211 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:3864152:3864211 [2] NCCL INFO Using network AWS Libfabric +gpua085:3864152:3864211 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua085:3864152:3864211 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:3864152:3864211 [2] NCCL INFO Trees [0] 59/-1/-1->58->57 [1] 59/-1/-1->58->57 +gpua085:3864152:3864211 [2] NCCL INFO Channel 00/0 : 58[85000] -> 59[c7000] via P2P/IPC/read +gpua085:3864152:3864211 [2] NCCL INFO Channel 01/0 : 58[85000] -> 59[c7000] via P2P/IPC/read +gpua085:3864152:3864211 [2] NCCL INFO Connected all rings +gpua085:3864152:3864211 [2] NCCL INFO Channel 00/0 : 58[85000] -> 57[46000] via P2P/IPC/read +gpua085:3864152:3864211 [2] NCCL INFO Channel 01/0 : 58[85000] -> 57[46000] via P2P/IPC/read +gpua085:3864152:3864211 [2] NCCL INFO Connected all trees +gpua085:3864152:3864211 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:3864152:3864211 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:3864152:3864211 [2] NCCL INFO comm 0x55849f6e8860 rank 58 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua085:3864151:3864151 [1] NCCL INFO cudaDriverVersion 12020 +gpua085:3864151:3864151 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:3864151:3864151 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:3864151:3864151 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:3864151:3864208 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua085:3864151:3864208 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:3864151:3864208 [1] NCCL INFO Using network AWS Libfabric +gpua085:3864151:3864208 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua085:3864151:3864208 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:3864151:3864208 [1] NCCL INFO Trees [0] 58/52/-1->57->56 [1] 58/-1/-1->57->56 +gpua085:3864151:3864208 [1] NCCL INFO Channel 00/0 : 57[46000] -> 58[85000] via P2P/IPC/read +gpua085:3864151:3864208 [1] NCCL INFO Channel 01/0 : 57[46000] -> 58[85000] via P2P/IPC/read +gpua085:3864151:3864208 [1] NCCL INFO Connected all rings +gpua085:3864151:3864208 [1] NCCL INFO Channel 00/0 : 52[7000] -> 57[46000] [receive] via NET/AWS Libfabric/1 +gpua085:3864151:3864208 [1] NCCL INFO Channel 00/0 : 57[46000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua085:3864151:3864208 [1] NCCL INFO Channel 00/0 : 57[46000] -> 56[7000] via P2P/IPC/read +gpua085:3864151:3864208 [1] NCCL INFO Channel 01/0 : 57[46000] -> 56[7000] via P2P/IPC/read +gpua085:3864151:3864208 [1] NCCL INFO Connected all trees +gpua085:3864151:3864208 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:3864151:3864208 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:3864151:3864208 [1] NCCL INFO comm 0x5574617dfac0 rank 57 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +[gpua006:0/64] 2024-02-08 14:36:30,084 (distributed:1027) INFO: Reducer buckets have been rebuilt in this iteration. +[gpua006:0/64] 2024-02-08 14:44:18,405 (trainer:756) INFO: 22epoch:train:1-100batch: iter_time=4.263, forward_time=0.312, loss_ctc=75.171, loss_interctc_layer6=83.764, loss_interctc_layer12=69.869, loss_interctc_layer15=64.245, loss_interctc_layer21=77.349, loss=74.079, backward_time=0.267, grad_norm=65.202, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.143, optim0_lr0=8.729e-05, train_time=9.524 +[gpua006:0/64] 2024-02-08 14:52:44,492 (trainer:756) INFO: 22epoch:train:101-200batch: iter_time=0.002, forward_time=0.380, loss_ctc=70.721, loss_interctc_layer6=77.387, loss_interctc_layer12=64.732, loss_interctc_layer15=59.725, loss_interctc_layer21=72.651, loss=69.043, backward_time=0.304, grad_norm=61.953, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.152, optim0_lr0=8.727e-05, train_time=5.072 +[gpua006:0/64] 2024-02-08 14:58:59,663 (trainer:756) INFO: 22epoch:train:201-300batch: iter_time=0.001, forward_time=0.411, loss_ctc=81.563, loss_interctc_layer6=90.646, loss_interctc_layer12=76.633, loss_interctc_layer15=70.958, loss_interctc_layer21=83.949, loss=80.750, backward_time=0.305, grad_norm=62.375, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.149, optim0_lr0=8.726e-05, train_time=3.750 +[gpua006:0/64] 2024-02-08 15:03:21,216 (trainer:756) INFO: 22epoch:train:301-400batch: iter_time=4.125e-04, forward_time=0.373, loss_ctc=86.194, loss_interctc_layer6=87.494, loss_interctc_layer12=72.863, loss_interctc_layer15=66.856, loss_interctc_layer21=89.206, loss=80.523, backward_time=0.305, grad_norm=76.991, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.149, optim0_lr0=8.725e-05, train_time=2.617 +[gpua006:0/64] 2024-02-08 15:07:20,798 (trainer:756) INFO: 22epoch:train:401-500batch: iter_time=0.001, forward_time=0.306, loss_ctc=69.304, loss_interctc_layer6=81.863, loss_interctc_layer12=68.001, loss_interctc_layer15=62.466, loss_interctc_layer21=71.464, loss=70.620, backward_time=0.342, grad_norm=155.018, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.147, optim0_lr0=8.723e-05, train_time=2.395 +[gpua006:0/64] 2024-02-08 15:11:25,788 (trainer:756) INFO: 22epoch:train:501-600batch: iter_time=0.001, forward_time=0.488, loss_ctc=93.383, loss_interctc_layer6=97.488, loss_interctc_layer12=82.553, loss_interctc_layer15=76.672, loss_interctc_layer21=96.659, loss=89.351, backward_time=0.382, grad_norm=81.054, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.155, optim0_lr0=8.722e-05, train_time=2.450 +[gpua006:0/64] 2024-02-08 15:15:33,196 (trainer:756) INFO: 22epoch:train:601-700batch: iter_time=5.794e-04, forward_time=0.661, loss_ctc=77.459, loss_interctc_layer6=88.820, loss_interctc_layer12=74.482, loss_interctc_layer15=68.725, loss_interctc_layer21=79.842, loss=77.866, backward_time=0.358, grad_norm=64.822, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.162, optim0_lr0=8.720e-05, train_time=2.474 +[gpua006:0/64] 2024-02-08 15:19:45,398 (trainer:756) INFO: 22epoch:train:701-800batch: iter_time=0.003, forward_time=0.657, loss_ctc=80.404, loss_interctc_layer6=85.661, loss_interctc_layer12=72.335, loss_interctc_layer15=67.050, loss_interctc_layer21=82.794, loss=77.649, backward_time=0.457, grad_norm=81.206, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.164, optim0_lr0=8.719e-05, train_time=2.521 +[gpua006:0/64] 2024-02-08 15:24:20,896 (trainer:756) INFO: 22epoch:train:801-900batch: iter_time=0.002, forward_time=0.812, loss_ctc=73.531, loss_interctc_layer6=90.738, loss_interctc_layer12=76.091, loss_interctc_layer15=70.271, loss_interctc_layer21=75.770, loss=77.280, backward_time=0.609, grad_norm=74.154, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.174, optim0_lr0=8.718e-05, train_time=2.754 +[gpua006:0/64] 2024-02-08 15:29:35,122 (trainer:756) INFO: 22epoch:train:901-1000batch: iter_time=0.002, forward_time=0.893, loss_ctc=79.656, loss_interctc_layer6=85.092, loss_interctc_layer12=71.002, loss_interctc_layer15=65.278, loss_interctc_layer21=82.381, loss=76.682, backward_time=0.463, grad_norm=131.770, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.167, optim0_lr0=8.716e-05, train_time=3.143 +[gpua006:0/64] 2024-02-08 15:34:37,025 (trainer:756) INFO: 22epoch:train:1001-1100batch: iter_time=6.517e-04, forward_time=1.008, loss_ctc=87.544, loss_interctc_layer6=92.802, loss_interctc_layer12=78.260, loss_interctc_layer15=72.758, loss_interctc_layer21=89.925, loss=84.258, backward_time=0.561, grad_norm=97.317, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.182, optim0_lr0=8.715e-05, train_time=3.018 +[gpua006:0/64] 2024-02-08 15:38:00,342 (trainer:756) INFO: 22epoch:train:1101-1200batch: iter_time=9.036e-05, forward_time=0.162, loss_ctc=78.921, loss_interctc_layer6=83.825, loss_interctc_layer12=70.544, loss_interctc_layer15=65.031, loss_interctc_layer21=81.270, loss=75.918, backward_time=0.203, grad_norm=84.305, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=8.714e-05, train_time=2.034 +[gpua006:0/64] 2024-02-08 15:39:56,263 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-08 15:40:14,870 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 15:40:18,323 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 15:40:18,323 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-08 15:40:18,326 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 15:48:15,907 (trainer:756) INFO: 22epoch:train:1201-1300batch: iter_time=4.636, forward_time=0.318, loss_ctc=84.632, loss_interctc_layer6=85.426, loss_interctc_layer12=72.433, loss_interctc_layer15=67.310, loss_interctc_layer21=87.252, loss=79.411, backward_time=0.248, grad_norm=74.612, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.143, optim0_lr0=8.712e-05, train_time=6.155 +[gpua006:0/64] 2024-02-08 15:51:09,943 (trainer:756) INFO: 22epoch:train:1301-1400batch: iter_time=8.984e-05, forward_time=0.408, loss_ctc=62.489, loss_interctc_layer6=76.649, loss_interctc_layer12=63.813, loss_interctc_layer15=58.607, loss_interctc_layer21=64.335, loss=65.178, backward_time=0.366, grad_norm=62.598, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.153, optim0_lr0=8.711e-05, train_time=1.740 +[gpua006:0/64] 2024-02-08 15:54:08,611 (trainer:756) INFO: 22epoch:train:1401-1500batch: iter_time=0.001, forward_time=0.455, loss_ctc=73.623, loss_interctc_layer6=79.198, loss_interctc_layer12=66.184, loss_interctc_layer15=61.024, loss_interctc_layer21=75.723, loss=71.150, backward_time=0.392, grad_norm=61.258, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.154, optim0_lr0=8.709e-05, train_time=1.783 +[gpua006:0/64] 2024-02-08 15:56:43,826 (trainer:756) INFO: 22epoch:train:1501-1600batch: iter_time=3.135e-04, forward_time=0.243, loss_ctc=81.239, loss_interctc_layer6=87.713, loss_interctc_layer12=73.512, loss_interctc_layer15=67.850, loss_interctc_layer21=84.007, loss=78.864, backward_time=0.255, grad_norm=69.228, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.141, optim0_lr0=8.708e-05, train_time=1.554 +[gpua006:0/64] 2024-02-08 15:59:06,831 (trainer:756) INFO: 22epoch:train:1601-1700batch: iter_time=3.231e-04, forward_time=0.248, loss_ctc=83.154, loss_interctc_layer6=87.008, loss_interctc_layer12=72.217, loss_interctc_layer15=66.353, loss_interctc_layer21=85.753, loss=78.897, backward_time=0.251, grad_norm=65.349, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.144, optim0_lr0=8.707e-05, train_time=1.429 +[gpua006:0/64] 2024-02-08 16:01:42,117 (trainer:756) INFO: 22epoch:train:1701-1800batch: iter_time=0.006, forward_time=0.295, loss_ctc=75.246, loss_interctc_layer6=84.552, loss_interctc_layer12=70.468, loss_interctc_layer15=64.794, loss_interctc_layer21=77.796, loss=74.571, backward_time=0.279, grad_norm=58.035, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.146, optim0_lr0=8.705e-05, train_time=1.554 +[gpua006:0/64] 2024-02-08 16:04:07,159 (trainer:756) INFO: 22epoch:train:1801-1900batch: iter_time=6.177e-04, forward_time=0.354, loss_ctc=85.165, loss_interctc_layer6=95.138, loss_interctc_layer12=79.600, loss_interctc_layer15=73.483, loss_interctc_layer21=87.974, loss=84.272, backward_time=0.251, grad_norm=73.061, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.143, optim0_lr0=8.704e-05, train_time=1.449 +[gpua006:0/64] 2024-02-08 16:06:15,599 (trainer:756) INFO: 22epoch:train:1901-2000batch: iter_time=9.479e-04, forward_time=0.233, loss_ctc=89.039, loss_interctc_layer6=94.857, loss_interctc_layer12=79.730, loss_interctc_layer15=73.955, loss_interctc_layer21=92.080, loss=85.932, backward_time=0.275, grad_norm=75.496, clip=100.000, loss_scale=8.924e+30, optim_step_time=0.143, optim0_lr0=8.703e-05, train_time=1.286 +[gpua006:0/64] 2024-02-08 16:08:45,631 (trainer:756) INFO: 22epoch:train:2001-2100batch: iter_time=1.834e-04, forward_time=0.277, loss_ctc=71.245, loss_interctc_layer6=83.093, loss_interctc_layer12=69.206, loss_interctc_layer15=63.749, loss_interctc_layer21=73.517, loss=72.162, backward_time=0.242, grad_norm=65.755, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=8.701e-05, train_time=1.499 +[gpua006:0/64] 2024-02-08 16:11:34,324 (trainer:756) INFO: 22epoch:train:2101-2200batch: iter_time=3.509e-04, forward_time=0.238, loss_ctc=65.554, loss_interctc_layer6=80.182, loss_interctc_layer12=66.684, loss_interctc_layer15=61.388, loss_interctc_layer21=67.594, loss=68.281, backward_time=0.243, grad_norm=78.009, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=8.700e-05, train_time=1.687 +[gpua006:0/64] 2024-02-08 16:13:33,025 (trainer:756) INFO: 22epoch:train:2201-2300batch: iter_time=3.391e-04, forward_time=0.258, loss_ctc=83.994, loss_interctc_layer6=89.747, loss_interctc_layer12=75.316, loss_interctc_layer15=69.569, loss_interctc_layer21=86.523, loss=81.030, backward_time=0.257, grad_norm=91.525, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=8.698e-05, train_time=1.186 +[gpua006:0/64] 2024-02-08 16:16:01,703 (trainer:756) INFO: 22epoch:train:2301-2400batch: iter_time=6.359e-04, forward_time=0.235, loss_ctc=85.368, loss_interctc_layer6=88.277, loss_interctc_layer12=74.190, loss_interctc_layer15=68.789, loss_interctc_layer21=87.920, loss=80.909, backward_time=0.269, grad_norm=69.354, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.146, optim0_lr0=8.697e-05, train_time=1.485 +[gpua006:0/64] 2024-02-08 16:18:29,719 (trainer:756) INFO: 22epoch:train:2401-2500batch: iter_time=4.441e-04, forward_time=0.280, loss_ctc=82.901, loss_interctc_layer6=84.650, loss_interctc_layer12=71.204, loss_interctc_layer15=66.016, loss_interctc_layer21=85.855, loss=78.125, backward_time=0.308, grad_norm=70.892, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.146, optim0_lr0=8.696e-05, train_time=1.482 +[gpua006:0/64] 2024-02-08 16:18:49,834 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-08 16:19:08,343 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 16:19:11,869 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 16:19:11,869 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-08 16:19:11,872 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 16:29:03,158 (trainer:756) INFO: 22epoch:train:2501-2600batch: iter_time=3.102, forward_time=0.203, loss_ctc=73.632, loss_interctc_layer6=82.756, loss_interctc_layer12=68.588, loss_interctc_layer15=62.897, loss_interctc_layer21=76.035, loss=72.782, backward_time=0.219, grad_norm=74.063, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=8.694e-05, train_time=6.335 +[gpua006:0/64] 2024-02-08 16:30:43,355 (trainer:756) INFO: 22epoch:train:2601-2700batch: iter_time=8.706e-05, forward_time=0.142, loss_ctc=69.840, loss_interctc_layer6=76.727, loss_interctc_layer12=63.949, loss_interctc_layer15=58.909, loss_interctc_layer21=71.816, loss=68.248, backward_time=0.210, grad_norm=64.228, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.693e-05, train_time=1.002 +[gpua006:0/64] 2024-02-08 16:32:42,347 (trainer:756) INFO: 22epoch:train:2701-2800batch: iter_time=8.343e-05, forward_time=0.143, loss_ctc=79.825, loss_interctc_layer6=88.999, loss_interctc_layer12=74.910, loss_interctc_layer15=69.212, loss_interctc_layer21=82.419, loss=79.073, backward_time=0.209, grad_norm=62.973, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.692e-05, train_time=1.190 +[gpua006:0/64] 2024-02-08 16:34:52,538 (trainer:756) INFO: 22epoch:train:2801-2900batch: iter_time=8.639e-05, forward_time=0.141, loss_ctc=85.088, loss_interctc_layer6=86.404, loss_interctc_layer12=71.624, loss_interctc_layer15=65.648, loss_interctc_layer21=88.113, loss=79.375, backward_time=0.207, grad_norm=70.598, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.690e-05, train_time=1.302 +[gpua006:0/64] 2024-02-08 16:36:29,963 (trainer:756) INFO: 22epoch:train:2901-3000batch: iter_time=8.986e-05, forward_time=0.141, loss_ctc=67.980, loss_interctc_layer6=81.184, loss_interctc_layer12=67.211, loss_interctc_layer15=61.568, loss_interctc_layer21=70.036, loss=69.596, backward_time=0.209, grad_norm=58.583, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.689e-05, train_time=0.974 +[gpua006:0/64] 2024-02-08 16:38:24,799 (trainer:756) INFO: 22epoch:train:3001-3100batch: iter_time=9.304e-05, forward_time=0.142, loss_ctc=91.641, loss_interctc_layer6=95.512, loss_interctc_layer12=80.009, loss_interctc_layer15=74.034, loss_interctc_layer21=95.108, loss=87.261, backward_time=0.208, grad_norm=129.384, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.687e-05, train_time=1.148 +[gpua006:0/64] 2024-02-08 16:40:31,132 (trainer:756) INFO: 22epoch:train:3101-3200batch: iter_time=9.013e-05, forward_time=0.143, loss_ctc=76.048, loss_interctc_layer6=87.656, loss_interctc_layer12=73.227, loss_interctc_layer15=67.523, loss_interctc_layer21=78.413, loss=76.573, backward_time=0.209, grad_norm=72.255, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.686e-05, train_time=1.263 +[gpua006:0/64] 2024-02-08 16:42:42,113 (trainer:756) INFO: 22epoch:train:3201-3300batch: iter_time=9.144e-05, forward_time=0.143, loss_ctc=77.846, loss_interctc_layer6=83.886, loss_interctc_layer12=70.016, loss_interctc_layer15=64.652, loss_interctc_layer21=80.344, loss=75.349, backward_time=0.206, grad_norm=100.544, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.685e-05, train_time=1.310 +[gpua006:0/64] 2024-02-08 16:44:52,344 (trainer:756) INFO: 22epoch:train:3301-3400batch: iter_time=9.157e-05, forward_time=0.142, loss_ctc=71.811, loss_interctc_layer6=88.818, loss_interctc_layer12=74.094, loss_interctc_layer15=68.327, loss_interctc_layer21=73.860, loss=75.382, backward_time=0.206, grad_norm=64.236, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.683e-05, train_time=1.302 +[gpua006:0/64] 2024-02-08 16:46:41,416 (trainer:756) INFO: 22epoch:train:3401-3500batch: iter_time=9.452e-05, forward_time=0.142, loss_ctc=78.429, loss_interctc_layer6=84.819, loss_interctc_layer12=70.880, loss_interctc_layer15=64.947, loss_interctc_layer21=81.051, loss=76.025, backward_time=0.208, grad_norm=71.993, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.682e-05, train_time=1.090 +[gpua006:0/64] 2024-02-08 16:48:49,157 (trainer:756) INFO: 22epoch:train:3501-3600batch: iter_time=1.038e-04, forward_time=0.141, loss_ctc=85.417, loss_interctc_layer6=90.701, loss_interctc_layer12=76.190, loss_interctc_layer15=70.723, loss_interctc_layer21=87.941, loss=82.195, backward_time=0.208, grad_norm=71.755, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.681e-05, train_time=1.277 +[gpua006:0/64] 2024-02-08 16:50:49,554 (trainer:756) INFO: 22epoch:train:3601-3700batch: iter_time=1.023e-04, forward_time=0.142, loss_ctc=78.379, loss_interctc_layer6=83.155, loss_interctc_layer12=69.540, loss_interctc_layer15=64.056, loss_interctc_layer21=80.782, loss=75.182, backward_time=0.209, grad_norm=78.965, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.679e-05, train_time=1.204 +[gpua006:0/64] 2024-02-08 16:52:01,565 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-08 16:52:20,017 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 16:52:23,580 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 16:52:23,580 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-08 16:52:23,583 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 17:00:44,565 (trainer:756) INFO: 22epoch:train:3701-3800batch: iter_time=4.852, forward_time=0.211, loss_ctc=80.996, loss_interctc_layer6=84.011, loss_interctc_layer12=70.051, loss_interctc_layer15=64.406, loss_interctc_layer21=83.544, loss=76.601, backward_time=0.221, grad_norm=70.444, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=8.678e-05, train_time=5.950 +[gpua006:0/64] 2024-02-08 17:09:39,207 (trainer:756) INFO: 22epoch:train:3801-3900batch: iter_time=8.695e-05, forward_time=0.143, loss_ctc=62.267, loss_interctc_layer6=76.453, loss_interctc_layer12=63.657, loss_interctc_layer15=58.402, loss_interctc_layer21=64.123, loss=64.981, backward_time=0.207, grad_norm=76.919, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.677e-05, train_time=5.346 +[gpua006:0/64] 2024-02-08 17:15:24,739 (trainer:756) INFO: 22epoch:train:3901-4000batch: iter_time=0.002, forward_time=0.234, loss_ctc=72.539, loss_interctc_layer6=79.481, loss_interctc_layer12=66.250, loss_interctc_layer15=60.903, loss_interctc_layer21=74.606, loss=70.756, backward_time=0.264, grad_norm=89.339, clip=100.000, loss_scale=1.785e+31, optim_step_time=0.147, optim0_lr0=8.675e-05, train_time=3.455 +[gpua006:0/64] 2024-02-08 17:18:15,737 (trainer:756) INFO: 22epoch:train:4001-4100batch: iter_time=8.330e-05, forward_time=0.143, loss_ctc=80.158, loss_interctc_layer6=87.286, loss_interctc_layer12=72.905, loss_interctc_layer15=67.233, loss_interctc_layer21=82.930, loss=78.102, backward_time=0.210, grad_norm=71.826, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.674e-05, train_time=1.710 +[gpua006:0/64] 2024-02-08 17:20:54,405 (trainer:756) INFO: 22epoch:train:4101-4200batch: iter_time=9.038e-05, forward_time=0.143, loss_ctc=82.415, loss_interctc_layer6=86.095, loss_interctc_layer12=71.262, loss_interctc_layer15=65.579, loss_interctc_layer21=85.087, loss=78.088, backward_time=0.208, grad_norm=63.599, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.672e-05, train_time=1.587 +[gpua006:0/64] 2024-02-08 17:23:53,588 (trainer:756) INFO: 22epoch:train:4201-4300batch: iter_time=9.099e-05, forward_time=0.143, loss_ctc=74.992, loss_interctc_layer6=84.376, loss_interctc_layer12=70.178, loss_interctc_layer15=64.555, loss_interctc_layer21=77.492, loss=74.319, backward_time=0.209, grad_norm=55.964, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.671e-05, train_time=1.792 +[gpua006:0/64] 2024-02-08 17:27:31,347 (trainer:756) INFO: 22epoch:train:4301-4400batch: iter_time=9.617e-05, forward_time=0.301, loss_ctc=84.560, loss_interctc_layer6=94.984, loss_interctc_layer12=79.495, loss_interctc_layer15=73.368, loss_interctc_layer21=87.394, loss=83.960, backward_time=0.244, grad_norm=76.373, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=8.670e-05, train_time=2.176 +[gpua006:0/64] 2024-02-08 17:30:11,905 (trainer:756) INFO: 22epoch:train:4401-4500batch: iter_time=8.963e-05, forward_time=0.145, loss_ctc=88.354, loss_interctc_layer6=93.267, loss_interctc_layer12=78.191, loss_interctc_layer15=72.390, loss_interctc_layer21=91.371, loss=84.715, backward_time=0.209, grad_norm=72.645, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.668e-05, train_time=1.606 +[gpua006:0/64] 2024-02-08 17:32:36,635 (trainer:756) INFO: 22epoch:train:4501-4600batch: iter_time=8.849e-05, forward_time=0.144, loss_ctc=71.036, loss_interctc_layer6=82.978, loss_interctc_layer12=69.156, loss_interctc_layer15=63.570, loss_interctc_layer21=73.358, loss=72.020, backward_time=0.210, grad_norm=98.857, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.667e-05, train_time=1.447 +[gpua006:0/64] 2024-02-08 17:34:50,087 (trainer:756) INFO: 22epoch:train:4601-4700batch: iter_time=8.142e-05, forward_time=0.163, loss_ctc=64.957, loss_interctc_layer6=78.488, loss_interctc_layer12=65.001, loss_interctc_layer15=59.751, loss_interctc_layer21=67.119, loss=67.063, backward_time=0.210, grad_norm=55.982, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.666e-05, train_time=1.334 +[gpua006:0/64] 2024-02-08 17:37:25,300 (trainer:756) INFO: 22epoch:train:4701-4800batch: iter_time=2.388e-04, forward_time=0.247, loss_ctc=82.560, loss_interctc_layer6=89.535, loss_interctc_layer12=75.132, loss_interctc_layer15=69.580, loss_interctc_layer21=85.221, loss=80.406, backward_time=0.266, grad_norm=69.161, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.147, optim0_lr0=8.664e-05, train_time=1.551 +[gpua006:0/64] 2024-02-08 17:39:42,496 (trainer:756) INFO: 22epoch:train:4801-4900batch: iter_time=8.565e-05, forward_time=0.143, loss_ctc=81.902, loss_interctc_layer6=86.967, loss_interctc_layer12=72.541, loss_interctc_layer15=67.122, loss_interctc_layer21=84.676, loss=78.642, backward_time=0.210, grad_norm=70.186, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.663e-05, train_time=1.371 +[gpua006:0/64] 2024-02-08 17:42:11,131 (trainer:756) INFO: 22epoch:train:4901-5000batch: iter_time=8.685e-05, forward_time=0.144, loss_ctc=82.163, loss_interctc_layer6=84.610, loss_interctc_layer12=70.999, loss_interctc_layer15=65.541, loss_interctc_layer21=84.812, loss=77.625, backward_time=0.209, grad_norm=83.285, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.662e-05, train_time=1.488 +[gpua006:0/64] 2024-02-08 17:42:31,207 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-08 17:42:49,927 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 17:42:53,439 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 17:42:53,439 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-08 17:42:53,443 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 17:52:29,215 (trainer:756) INFO: 22epoch:train:5001-5100batch: iter_time=2.997, forward_time=0.181, loss_ctc=72.562, loss_interctc_layer6=81.867, loss_interctc_layer12=67.795, loss_interctc_layer15=62.164, loss_interctc_layer21=74.987, loss=71.875, backward_time=0.219, grad_norm=57.508, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.660e-05, train_time=6.180 +[gpua006:0/64] 2024-02-08 17:54:35,062 (trainer:756) INFO: 22epoch:train:5101-5200batch: iter_time=8.590e-05, forward_time=0.143, loss_ctc=70.111, loss_interctc_layer6=76.209, loss_interctc_layer12=63.433, loss_interctc_layer15=58.422, loss_interctc_layer21=72.037, loss=68.042, backward_time=0.208, grad_norm=56.959, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.659e-05, train_time=1.259 +[gpua006:0/64] 2024-02-08 17:57:24,545 (trainer:756) INFO: 22epoch:train:5201-5300batch: iter_time=8.775e-05, forward_time=0.181, loss_ctc=79.765, loss_interctc_layer6=87.758, loss_interctc_layer12=73.709, loss_interctc_layer15=68.002, loss_interctc_layer21=82.253, loss=78.297, backward_time=0.217, grad_norm=73.594, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.658e-05, train_time=1.695 +[gpua006:0/64] 2024-02-08 17:59:38,494 (trainer:756) INFO: 22epoch:train:5301-5400batch: iter_time=9.566e-05, forward_time=0.207, loss_ctc=83.163, loss_interctc_layer6=85.682, loss_interctc_layer12=70.961, loss_interctc_layer15=65.017, loss_interctc_layer21=86.024, loss=78.169, backward_time=0.235, grad_norm=62.063, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.656e-05, train_time=1.339 +[gpua006:0/64] 2024-02-08 18:01:35,120 (trainer:756) INFO: 22epoch:train:5401-5500batch: iter_time=8.999e-05, forward_time=0.141, loss_ctc=68.193, loss_interctc_layer6=81.370, loss_interctc_layer12=67.115, loss_interctc_layer15=61.539, loss_interctc_layer21=70.399, loss=69.723, backward_time=0.206, grad_norm=59.074, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.655e-05, train_time=1.166 +[gpua006:0/64] 2024-02-08 18:03:38,706 (trainer:756) INFO: 22epoch:train:5501-5600batch: iter_time=8.347e-05, forward_time=0.142, loss_ctc=90.145, loss_interctc_layer6=94.648, loss_interctc_layer12=79.187, loss_interctc_layer15=73.269, loss_interctc_layer21=93.555, loss=86.161, backward_time=0.207, grad_norm=130.722, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.654e-05, train_time=1.235 +[gpua006:0/64] 2024-02-08 18:05:49,448 (trainer:756) INFO: 22epoch:train:5601-5700batch: iter_time=8.146e-05, forward_time=0.142, loss_ctc=75.556, loss_interctc_layer6=87.472, loss_interctc_layer12=73.109, loss_interctc_layer15=67.331, loss_interctc_layer21=77.904, loss=76.274, backward_time=0.212, grad_norm=135.116, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.652e-05, train_time=1.308 +[gpua006:0/64] 2024-02-08 18:08:00,611 (trainer:756) INFO: 22epoch:train:5701-5800batch: iter_time=8.724e-05, forward_time=0.209, loss_ctc=78.623, loss_interctc_layer6=83.973, loss_interctc_layer12=70.175, loss_interctc_layer15=64.687, loss_interctc_layer21=81.099, loss=75.711, backward_time=0.253, grad_norm=70.852, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.651e-05, train_time=1.311 +[gpua006:0/64] 2024-02-08 18:10:30,161 (trainer:756) INFO: 22epoch:train:5801-5900batch: iter_time=8.717e-05, forward_time=0.161, loss_ctc=71.982, loss_interctc_layer6=89.290, loss_interctc_layer12=74.716, loss_interctc_layer15=68.814, loss_interctc_layer21=74.136, loss=75.787, backward_time=0.224, grad_norm=70.444, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.649e-05, train_time=1.495 +[gpua006:0/64] 2024-02-08 18:13:02,479 (trainer:756) INFO: 22epoch:train:5901-6000batch: iter_time=8.515e-05, forward_time=0.142, loss_ctc=76.899, loss_interctc_layer6=83.702, loss_interctc_layer12=69.545, loss_interctc_layer15=63.717, loss_interctc_layer21=79.320, loss=74.637, backward_time=0.207, grad_norm=65.608, clip=100.000, loss_scale=3.570e+31, optim_step_time=0.137, optim0_lr0=8.648e-05, train_time=1.523 +[gpua006:0/64] 2024-02-08 18:14:43,262 (trainer:756) INFO: 22epoch:train:6001-6100batch: iter_time=8.691e-05, forward_time=0.154, loss_ctc=84.627, loss_interctc_layer6=89.127, loss_interctc_layer12=74.821, loss_interctc_layer15=69.101, loss_interctc_layer21=87.027, loss=80.941, backward_time=0.209, grad_norm=73.319, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.647e-05, train_time=1.008 +[gpua006:0/64] 2024-02-08 18:16:55,750 (trainer:756) INFO: 22epoch:train:6101-6200batch: iter_time=8.173e-05, forward_time=0.143, loss_ctc=78.177, loss_interctc_layer6=82.319, loss_interctc_layer12=68.797, loss_interctc_layer15=63.365, loss_interctc_layer21=80.653, loss=74.662, backward_time=0.206, grad_norm=51.734, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.645e-05, train_time=1.324 +[gpua006:0/64] 2024-02-08 18:18:21,954 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-08 18:18:40,632 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 18:18:44,202 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 18:18:44,202 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-08 18:18:44,205 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 18:27:56,442 (trainer:756) INFO: 22epoch:train:6201-6300batch: iter_time=3.415, forward_time=0.205, loss_ctc=83.350, loss_interctc_layer6=83.681, loss_interctc_layer12=69.536, loss_interctc_layer15=63.872, loss_interctc_layer21=86.436, loss=77.375, backward_time=0.224, grad_norm=74.705, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.644e-05, train_time=6.607 +[gpua006:0/64] 2024-02-08 18:29:31,442 (trainer:756) INFO: 22epoch:train:6301-6400batch: iter_time=9.724e-05, forward_time=0.144, loss_ctc=64.827, loss_interctc_layer6=76.265, loss_interctc_layer12=63.351, loss_interctc_layer15=58.109, loss_interctc_layer21=66.766, loss=65.864, backward_time=0.211, grad_norm=62.123, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.643e-05, train_time=0.950 +[gpua006:0/64] 2024-02-08 18:31:49,575 (trainer:756) INFO: 22epoch:train:6401-6500batch: iter_time=9.087e-05, forward_time=0.288, loss_ctc=76.364, loss_interctc_layer6=78.958, loss_interctc_layer12=65.801, loss_interctc_layer15=60.491, loss_interctc_layer21=78.826, loss=72.088, backward_time=0.261, grad_norm=63.916, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.641e-05, train_time=1.380 +[gpua006:0/64] 2024-02-08 18:33:49,809 (trainer:756) INFO: 22epoch:train:6501-6600batch: iter_time=9.810e-05, forward_time=0.143, loss_ctc=83.791, loss_interctc_layer6=86.671, loss_interctc_layer12=72.617, loss_interctc_layer15=66.866, loss_interctc_layer21=86.845, loss=79.358, backward_time=0.210, grad_norm=72.597, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.640e-05, train_time=1.203 +[gpua006:0/64] 2024-02-08 18:35:59,561 (trainer:756) INFO: 22epoch:train:6601-6700batch: iter_time=1.147e-04, forward_time=0.142, loss_ctc=86.676, loss_interctc_layer6=86.056, loss_interctc_layer12=71.266, loss_interctc_layer15=65.467, loss_interctc_layer21=89.673, loss=79.827, backward_time=0.209, grad_norm=68.899, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.639e-05, train_time=1.297 +[gpua006:0/64] 2024-02-08 18:38:47,787 (trainer:756) INFO: 22epoch:train:6701-6800batch: iter_time=1.177e-04, forward_time=0.288, loss_ctc=81.797, loss_interctc_layer6=84.009, loss_interctc_layer12=69.892, loss_interctc_layer15=64.169, loss_interctc_layer21=84.690, loss=76.911, backward_time=0.259, grad_norm=69.687, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.144, optim0_lr0=8.637e-05, train_time=1.681 +[gpua006:0/64] 2024-02-08 18:41:21,283 (trainer:756) INFO: 22epoch:train:6801-6900batch: iter_time=1.147e-04, forward_time=0.144, loss_ctc=84.921, loss_interctc_layer6=93.725, loss_interctc_layer12=78.626, loss_interctc_layer15=72.665, loss_interctc_layer21=87.829, loss=83.553, backward_time=0.208, grad_norm=76.738, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.636e-05, train_time=1.536 +[gpua006:0/64] 2024-02-08 18:43:58,880 (trainer:756) INFO: 22epoch:train:6901-7000batch: iter_time=1.111e-04, forward_time=0.147, loss_ctc=95.327, loss_interctc_layer6=92.133, loss_interctc_layer12=76.904, loss_interctc_layer15=70.858, loss_interctc_layer21=98.657, loss=86.776, backward_time=0.208, grad_norm=80.044, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.635e-05, train_time=1.576 +[gpua006:0/64] 2024-02-08 18:46:36,118 (trainer:756) INFO: 22epoch:train:7001-7100batch: iter_time=6.143e-04, forward_time=0.257, loss_ctc=73.350, loss_interctc_layer6=81.557, loss_interctc_layer12=67.680, loss_interctc_layer15=62.184, loss_interctc_layer21=75.728, loss=72.100, backward_time=0.268, grad_norm=65.266, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.633e-05, train_time=1.571 +[gpua006:0/64] 2024-02-08 18:48:21,275 (trainer:756) INFO: 22epoch:train:7101-7200batch: iter_time=1.027e-04, forward_time=0.143, loss_ctc=65.947, loss_interctc_layer6=77.962, loss_interctc_layer12=64.531, loss_interctc_layer15=59.127, loss_interctc_layer21=68.155, loss=67.144, backward_time=0.208, grad_norm=69.161, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.632e-05, train_time=1.053 +[gpua006:0/64] 2024-02-08 18:50:34,475 (trainer:756) INFO: 22epoch:train:7201-7300batch: iter_time=9.270e-05, forward_time=0.256, loss_ctc=87.391, loss_interctc_layer6=89.200, loss_interctc_layer12=74.888, loss_interctc_layer15=69.147, loss_interctc_layer21=90.140, loss=82.153, backward_time=0.266, grad_norm=107.836, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.631e-05, train_time=1.332 +[gpua006:0/64] 2024-02-08 18:52:55,597 (trainer:756) INFO: 22epoch:train:7301-7400batch: iter_time=9.850e-05, forward_time=0.152, loss_ctc=87.783, loss_interctc_layer6=86.388, loss_interctc_layer12=72.117, loss_interctc_layer15=66.438, loss_interctc_layer21=90.821, loss=80.710, backward_time=0.211, grad_norm=72.186, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.629e-05, train_time=1.409 +[gpua006:0/64] 2024-02-08 18:55:00,826 (trainer:756) INFO: 22epoch:train:7401-7500batch: iter_time=1.002e-04, forward_time=0.144, loss_ctc=86.485, loss_interctc_layer6=84.236, loss_interctc_layer12=70.544, loss_interctc_layer15=65.223, loss_interctc_layer21=89.366, loss=79.171, backward_time=0.209, grad_norm=108.205, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.628e-05, train_time=1.254 +[gpua006:0/64] 2024-02-08 18:55:20,856 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-08 18:55:39,151 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 18:55:42,602 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 18:55:42,602 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-08 18:55:42,605 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 19:07:33,175 (trainer:756) INFO: 22epoch:train:7501-7600batch: iter_time=3.146, forward_time=0.770, loss_ctc=75.784, loss_interctc_layer6=82.349, loss_interctc_layer12=68.172, loss_interctc_layer15=62.475, loss_interctc_layer21=78.304, loss=73.417, backward_time=0.578, grad_norm=98.727, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.212, optim0_lr0=8.627e-05, train_time=7.522 +[gpua006:0/64] 2024-02-08 19:10:59,517 (trainer:756) INFO: 22epoch:train:7601-7700batch: iter_time=0.001, forward_time=0.703, loss_ctc=70.870, loss_interctc_layer6=75.494, loss_interctc_layer12=62.763, loss_interctc_layer15=57.610, loss_interctc_layer21=72.907, loss=67.929, backward_time=0.504, grad_norm=78.424, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.172, optim0_lr0=8.625e-05, train_time=2.063 +[gpua006:0/64] 2024-02-08 19:15:42,785 (trainer:756) INFO: 22epoch:train:7701-7800batch: iter_time=0.003, forward_time=1.107, loss_ctc=82.277, loss_interctc_layer6=87.019, loss_interctc_layer12=73.012, loss_interctc_layer15=67.276, loss_interctc_layer21=85.011, loss=78.919, backward_time=0.525, grad_norm=66.802, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.208, optim0_lr0=8.624e-05, train_time=2.833 +[gpua006:0/64] 2024-02-08 19:23:55,686 (trainer:756) INFO: 22epoch:train:7801-7900batch: iter_time=0.005, forward_time=1.759, loss_ctc=87.112, loss_interctc_layer6=85.658, loss_interctc_layer12=70.772, loss_interctc_layer15=64.815, loss_interctc_layer21=90.228, loss=79.717, backward_time=1.366, grad_norm=69.729, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.251, optim0_lr0=8.623e-05, train_time=4.928 +[gpua006:0/64] 2024-02-08 19:31:39,775 (trainer:756) INFO: 22epoch:train:7901-8000batch: iter_time=0.005, forward_time=1.652, loss_ctc=73.395, loss_interctc_layer6=80.717, loss_interctc_layer12=66.542, loss_interctc_layer15=60.932, loss_interctc_layer21=75.790, loss=71.475, backward_time=1.382, grad_norm=63.379, clip=100.000, loss_scale=7.139e+31, optim_step_time=0.246, optim0_lr0=8.621e-05, train_time=4.641 +[gpua006:0/64] 2024-02-08 19:40:02,125 (trainer:756) INFO: 22epoch:train:8001-8100batch: iter_time=0.005, forward_time=1.793, loss_ctc=96.301, loss_interctc_layer6=94.453, loss_interctc_layer12=79.064, loss_interctc_layer15=72.807, loss_interctc_layer21=99.983, loss=88.522, backward_time=1.665, grad_norm=96.276, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.262, optim0_lr0=8.620e-05, train_time=5.024 +[gpua006:0/64] 2024-02-08 19:52:19,353 (trainer:756) INFO: 22epoch:train:8101-8200batch: iter_time=0.005, forward_time=3.392, loss_ctc=79.429, loss_interctc_layer6=87.452, loss_interctc_layer12=73.014, loss_interctc_layer15=67.183, loss_interctc_layer21=82.020, loss=77.820, backward_time=1.997, grad_norm=56.535, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.299, optim0_lr0=8.619e-05, train_time=7.372 +[gpua006:0/64] 2024-02-08 20:03:35,219 (trainer:756) INFO: 22epoch:train:8201-8300batch: iter_time=0.004, forward_time=3.280, loss_ctc=83.666, loss_interctc_layer6=83.061, loss_interctc_layer12=69.074, loss_interctc_layer15=63.606, loss_interctc_layer21=86.445, loss=77.171, backward_time=1.744, grad_norm=82.620, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.295, optim0_lr0=8.617e-05, train_time=6.758 +[gpua006:0/64] 2024-02-08 20:09:59,152 (trainer:756) INFO: 22epoch:train:8301-8400batch: iter_time=0.002, forward_time=1.210, loss_ctc=75.269, loss_interctc_layer6=88.719, loss_interctc_layer12=73.876, loss_interctc_layer15=68.023, loss_interctc_layer21=77.692, loss=76.716, backward_time=1.039, grad_norm=65.625, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.209, optim0_lr0=8.616e-05, train_time=3.839 +[gpua006:0/64] 2024-02-08 20:15:34,291 (trainer:756) INFO: 22epoch:train:8401-8500batch: iter_time=0.002, forward_time=1.164, loss_ctc=82.691, loss_interctc_layer6=84.111, loss_interctc_layer12=69.834, loss_interctc_layer15=63.984, loss_interctc_layer21=85.741, loss=77.272, backward_time=1.038, grad_norm=74.637, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.217, optim0_lr0=8.615e-05, train_time=3.352 +[gpua006:0/64] 2024-02-08 20:21:53,194 (trainer:756) INFO: 22epoch:train:8501-8600batch: iter_time=0.002, forward_time=1.416, loss_ctc=88.216, loss_interctc_layer6=88.919, loss_interctc_layer12=74.634, loss_interctc_layer15=68.935, loss_interctc_layer21=90.600, loss=82.261, backward_time=1.042, grad_norm=84.802, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.233, optim0_lr0=8.613e-05, train_time=3.788 +[gpua006:0/64] 2024-02-08 20:31:30,012 (trainer:756) INFO: 22epoch:train:8601-8700batch: iter_time=0.004, forward_time=2.430, loss_ctc=80.877, loss_interctc_layer6=82.479, loss_interctc_layer12=68.907, loss_interctc_layer15=63.301, loss_interctc_layer21=83.509, loss=75.815, backward_time=1.682, grad_norm=66.547, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.282, optim0_lr0=8.612e-05, train_time=5.767 +[gpua006:0/64] 2024-02-08 20:35:38,073 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-08 20:35:56,239 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 20:35:59,719 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 20:35:59,719 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-08 20:35:59,722 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 20:45:23,118 (trainer:756) INFO: 22epoch:train:8701-8800batch: iter_time=5.431, forward_time=1.066, loss_ctc=83.047, loss_interctc_layer6=82.878, loss_interctc_layer12=68.695, loss_interctc_layer15=62.909, loss_interctc_layer21=85.878, loss=76.681, backward_time=0.787, grad_norm=74.541, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.186, optim0_lr0=8.611e-05, train_time=8.333 +[gpua006:0/64] 2024-02-08 20:47:50,664 (trainer:756) INFO: 22epoch:train:8801-8900batch: iter_time=1.315e-04, forward_time=0.259, loss_ctc=64.595, loss_interctc_layer6=76.523, loss_interctc_layer12=63.650, loss_interctc_layer15=58.345, loss_interctc_layer21=66.455, loss=65.914, backward_time=0.253, grad_norm=64.865, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.150, optim0_lr0=8.609e-05, train_time=1.474 +[gpua006:0/64] 2024-02-08 20:50:30,582 (trainer:756) INFO: 22epoch:train:8901-9000batch: iter_time=2.697e-04, forward_time=0.302, loss_ctc=75.786, loss_interctc_layer6=79.085, loss_interctc_layer12=65.912, loss_interctc_layer15=60.578, loss_interctc_layer21=78.033, loss=71.879, backward_time=0.274, grad_norm=67.151, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.141, optim0_lr0=8.608e-05, train_time=1.600 +[gpua006:0/64] 2024-02-08 20:53:08,069 (trainer:756) INFO: 22epoch:train:9001-9100batch: iter_time=8.947e-04, forward_time=0.213, loss_ctc=82.686, loss_interctc_layer6=86.413, loss_interctc_layer12=72.297, loss_interctc_layer15=66.549, loss_interctc_layer21=85.613, loss=78.712, backward_time=0.315, grad_norm=69.728, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.143, optim0_lr0=8.607e-05, train_time=1.574 +[gpua006:0/64] 2024-02-08 20:55:25,767 (trainer:756) INFO: 22epoch:train:9101-9200batch: iter_time=3.546e-04, forward_time=0.232, loss_ctc=85.374, loss_interctc_layer6=85.565, loss_interctc_layer12=70.803, loss_interctc_layer15=64.943, loss_interctc_layer21=88.481, loss=79.033, backward_time=0.260, grad_norm=72.557, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.141, optim0_lr0=8.605e-05, train_time=1.375 +[gpua006:0/64] 2024-02-08 20:57:39,194 (trainer:756) INFO: 22epoch:train:9201-9300batch: iter_time=1.053e-04, forward_time=0.163, loss_ctc=80.375, loss_interctc_layer6=83.670, loss_interctc_layer12=69.494, loss_interctc_layer15=63.689, loss_interctc_layer21=83.063, loss=76.058, backward_time=0.215, grad_norm=73.611, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.138, optim0_lr0=8.604e-05, train_time=1.337 +[gpua006:0/64] 2024-02-08 21:00:56,177 (trainer:756) INFO: 22epoch:train:9301-9400batch: iter_time=4.172e-04, forward_time=0.278, loss_ctc=84.598, loss_interctc_layer6=93.334, loss_interctc_layer12=78.214, loss_interctc_layer15=71.960, loss_interctc_layer21=87.307, loss=83.082, backward_time=0.330, grad_norm=71.060, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.145, optim0_lr0=8.603e-05, train_time=1.968 +[gpua006:0/64] 2024-02-08 21:03:17,233 (trainer:756) INFO: 22epoch:train:9401-9500batch: iter_time=2.457e-04, forward_time=0.244, loss_ctc=96.518, loss_interctc_layer6=93.200, loss_interctc_layer12=77.912, loss_interctc_layer15=72.068, loss_interctc_layer21=99.817, loss=87.903, backward_time=0.254, grad_norm=73.881, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.142, optim0_lr0=8.601e-05, train_time=1.412 +[gpua006:0/64] 2024-02-08 21:06:13,418 (trainer:756) INFO: 22epoch:train:9501-9600batch: iter_time=3.072e-04, forward_time=0.265, loss_ctc=73.756, loss_interctc_layer6=81.952, loss_interctc_layer12=68.086, loss_interctc_layer15=62.493, loss_interctc_layer21=76.000, loss=72.458, backward_time=0.236, grad_norm=57.337, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.141, optim0_lr0=8.600e-05, train_time=1.760 +[gpua006:0/64] 2024-02-08 21:08:19,329 (trainer:756) INFO: 22epoch:train:9601-9700batch: iter_time=2.684e-04, forward_time=0.217, loss_ctc=66.136, loss_interctc_layer6=78.215, loss_interctc_layer12=64.788, loss_interctc_layer15=59.325, loss_interctc_layer21=68.256, loss=67.344, backward_time=0.268, grad_norm=55.486, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.142, optim0_lr0=8.599e-05, train_time=1.259 +[gpua006:0/64] 2024-02-08 21:10:31,646 (trainer:756) INFO: 22epoch:train:9701-9800batch: iter_time=3.045e-04, forward_time=0.332, loss_ctc=85.959, loss_interctc_layer6=88.625, loss_interctc_layer12=74.351, loss_interctc_layer15=68.556, loss_interctc_layer21=88.642, loss=81.227, backward_time=0.250, grad_norm=105.516, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.142, optim0_lr0=8.597e-05, train_time=1.323 +[gpua006:0/64] 2024-02-08 21:13:04,041 (trainer:756) INFO: 22epoch:train:9801-9900batch: iter_time=9.720e-05, forward_time=0.238, loss_ctc=87.439, loss_interctc_layer6=86.110, loss_interctc_layer12=72.155, loss_interctc_layer15=66.174, loss_interctc_layer21=90.384, loss=80.452, backward_time=0.255, grad_norm=71.738, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.141, optim0_lr0=8.596e-05, train_time=1.524 +[gpua006:0/64] 2024-02-08 21:15:37,322 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-08 21:15:48,702 (trainer:756) INFO: 22epoch:train:9901-10000batch: iter_time=3.313e-04, forward_time=0.241, loss_ctc=84.877, loss_interctc_layer6=83.264, loss_interctc_layer12=69.381, loss_interctc_layer15=63.844, loss_interctc_layer21=87.891, loss=77.851, backward_time=0.245, grad_norm=61.254, clip=100.000, loss_scale=1.336e+32, optim_step_time=0.141, optim0_lr0=8.595e-05, train_time=1.647 +[gpua006:0/64] 2024-02-08 21:16:08,837 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-08 21:16:27,567 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 21:16:31,096 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 21:16:31,096 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-08 21:16:31,101 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 21:33:20,504 (trainer:756) INFO: 22epoch:train:10001-10100batch: iter_time=8.160, forward_time=0.175, loss_ctc=73.029, loss_interctc_layer6=81.805, loss_interctc_layer12=67.747, loss_interctc_layer15=62.054, loss_interctc_layer21=75.677, loss=72.063, backward_time=0.218, grad_norm=67.475, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.139, optim0_lr0=8.593e-05, train_time=10.518 +[gpua006:0/64] 2024-02-08 21:36:22,909 (trainer:756) INFO: 22epoch:train:10101-10200batch: iter_time=8.936e-05, forward_time=0.142, loss_ctc=69.926, loss_interctc_layer6=76.055, loss_interctc_layer12=63.277, loss_interctc_layer15=58.148, loss_interctc_layer21=72.026, loss=67.886, backward_time=0.210, grad_norm=61.134, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.137, optim0_lr0=8.592e-05, train_time=1.825 +[gpua006:0/64] 2024-02-08 21:36:23,936 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-08 21:42:21,809 (trainer:756) INFO: 22epoch:train:10201-10300batch: iter_time=1.712e-04, forward_time=0.277, loss_ctc=79.795, loss_interctc_layer6=86.880, loss_interctc_layer12=73.037, loss_interctc_layer15=67.336, loss_interctc_layer21=82.311, loss=77.872, backward_time=0.235, grad_norm=70.254, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.152, optim0_lr0=8.591e-05, train_time=3.588 +[gpua006:0/64] 2024-02-08 21:49:54,178 (trainer:756) INFO: 22epoch:train:10301-10400batch: iter_time=9.636e-05, forward_time=0.143, loss_ctc=83.330, loss_interctc_layer6=85.586, loss_interctc_layer12=70.804, loss_interctc_layer15=64.842, loss_interctc_layer21=86.350, loss=78.182, backward_time=0.209, grad_norm=53.502, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.589e-05, train_time=4.524 +[gpua006:0/64] 2024-02-08 21:58:03,361 (trainer:756) INFO: 22epoch:train:10401-10500batch: iter_time=9.226e-05, forward_time=0.142, loss_ctc=68.082, loss_interctc_layer6=80.462, loss_interctc_layer12=66.416, loss_interctc_layer15=60.833, loss_interctc_layer21=70.315, loss=69.222, backward_time=0.207, grad_norm=66.356, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.588e-05, train_time=4.891 +[gpua006:0/64] 2024-02-08 22:01:54,072 (trainer:756) INFO: 22epoch:train:10501-10600batch: iter_time=9.980e-05, forward_time=0.143, loss_ctc=90.188, loss_interctc_layer6=94.431, loss_interctc_layer12=78.992, loss_interctc_layer15=72.730, loss_interctc_layer21=93.649, loss=85.998, backward_time=0.210, grad_norm=77.793, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.587e-05, train_time=2.308 +[gpua006:0/64] 2024-02-08 22:07:00,326 (trainer:756) INFO: 22epoch:train:10601-10700batch: iter_time=9.584e-05, forward_time=0.142, loss_ctc=75.264, loss_interctc_layer6=86.886, loss_interctc_layer12=72.379, loss_interctc_layer15=66.548, loss_interctc_layer21=77.738, loss=75.763, backward_time=0.208, grad_norm=61.115, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.585e-05, train_time=3.062 +[gpua006:0/64] 2024-02-08 22:14:10,541 (trainer:756) INFO: 22epoch:train:10701-10800batch: iter_time=9.951e-05, forward_time=0.278, loss_ctc=76.637, loss_interctc_layer6=82.477, loss_interctc_layer12=68.564, loss_interctc_layer15=63.137, loss_interctc_layer21=79.135, loss=73.990, backward_time=0.256, grad_norm=84.303, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.144, optim0_lr0=8.584e-05, train_time=4.302 +[gpua006:0/64] 2024-02-08 22:27:11,184 (trainer:756) INFO: 22epoch:train:10801-10900batch: iter_time=9.436e-05, forward_time=0.143, loss_ctc=70.612, loss_interctc_layer6=87.655, loss_interctc_layer12=72.995, loss_interctc_layer15=67.182, loss_interctc_layer21=72.810, loss=74.251, backward_time=0.207, grad_norm=66.044, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.583e-05, train_time=7.806 +[gpua006:0/64] 2024-02-08 22:32:03,007 (trainer:756) INFO: 22epoch:train:10901-11000batch: iter_time=9.653e-05, forward_time=0.141, loss_ctc=77.506, loss_interctc_layer6=83.641, loss_interctc_layer12=69.355, loss_interctc_layer15=63.582, loss_interctc_layer21=80.179, loss=74.852, backward_time=0.207, grad_norm=65.585, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.582e-05, train_time=2.917 +[gpua006:0/64] 2024-02-08 22:37:36,923 (trainer:756) INFO: 22epoch:train:11001-11100batch: iter_time=1.072e-04, forward_time=0.142, loss_ctc=84.227, loss_interctc_layer6=88.101, loss_interctc_layer12=73.758, loss_interctc_layer15=68.305, loss_interctc_layer21=87.636, loss=80.405, backward_time=0.206, grad_norm=79.528, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.580e-05, train_time=3.340 +[gpua006:0/64] 2024-02-08 22:51:47,261 (trainer:756) INFO: 22epoch:train:11101-11200batch: iter_time=1.109e-04, forward_time=0.168, loss_ctc=77.645, loss_interctc_layer6=81.540, loss_interctc_layer12=68.081, loss_interctc_layer15=62.687, loss_interctc_layer21=80.229, loss=74.036, backward_time=0.204, grad_norm=53.074, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.579e-05, train_time=8.503 +[gpua006:0/64] 2024-02-08 22:54:12,025 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-08 22:54:30,283 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 22:54:33,856 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 22:54:33,856 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-08 22:54:33,860 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 23:04:08,425 (trainer:756) INFO: 22epoch:train:11201-11300batch: iter_time=5.245, forward_time=0.315, loss_ctc=82.880, loss_interctc_layer6=82.583, loss_interctc_layer12=68.766, loss_interctc_layer15=63.116, loss_interctc_layer21=85.552, loss=76.579, backward_time=0.252, grad_norm=56.080, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.578e-05, train_time=7.411 +[gpua006:0/64] 2024-02-08 23:06:07,846 (trainer:756) INFO: 22epoch:train:11301-11400batch: iter_time=8.885e-05, forward_time=0.143, loss_ctc=63.897, loss_interctc_layer6=75.276, loss_interctc_layer12=62.384, loss_interctc_layer15=57.277, loss_interctc_layer21=65.750, loss=64.917, backward_time=0.211, grad_norm=95.768, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.576e-05, train_time=1.194 +[gpua006:0/64] 2024-02-08 23:08:49,248 (trainer:756) INFO: 22epoch:train:11401-11500batch: iter_time=6.241e-04, forward_time=0.257, loss_ctc=74.819, loss_interctc_layer6=78.690, loss_interctc_layer12=65.547, loss_interctc_layer15=60.317, loss_interctc_layer21=77.067, loss=71.288, backward_time=0.256, grad_norm=75.288, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.144, optim0_lr0=8.575e-05, train_time=1.613 +[gpua006:0/64] 2024-02-08 23:10:49,155 (trainer:756) INFO: 22epoch:train:11501-11600batch: iter_time=8.964e-05, forward_time=0.143, loss_ctc=82.458, loss_interctc_layer6=85.685, loss_interctc_layer12=71.533, loss_interctc_layer15=65.816, loss_interctc_layer21=85.210, loss=78.140, backward_time=0.210, grad_norm=67.716, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.574e-05, train_time=1.200 +[gpua006:0/64] 2024-02-08 23:13:34,133 (trainer:756) INFO: 22epoch:train:11601-11700batch: iter_time=7.174e-04, forward_time=0.165, loss_ctc=85.281, loss_interctc_layer6=85.132, loss_interctc_layer12=70.362, loss_interctc_layer15=64.646, loss_interctc_layer21=88.123, loss=78.709, backward_time=0.261, grad_norm=64.852, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.572e-05, train_time=1.648 +[gpua006:0/64] 2024-02-08 23:16:12,530 (trainer:756) INFO: 22epoch:train:11701-11800batch: iter_time=8.499e-05, forward_time=0.195, loss_ctc=80.711, loss_interctc_layer6=83.247, loss_interctc_layer12=69.155, loss_interctc_layer15=63.475, loss_interctc_layer21=83.539, loss=76.025, backward_time=0.215, grad_norm=79.078, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.571e-05, train_time=1.584 +[gpua006:0/64] 2024-02-08 23:18:18,477 (trainer:756) INFO: 22epoch:train:11801-11900batch: iter_time=8.069e-05, forward_time=0.143, loss_ctc=85.840, loss_interctc_layer6=93.279, loss_interctc_layer12=78.404, loss_interctc_layer15=72.228, loss_interctc_layer21=88.850, loss=83.720, backward_time=0.208, grad_norm=64.026, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.570e-05, train_time=1.260 +[gpua006:0/64] 2024-02-08 23:20:56,422 (trainer:756) INFO: 22epoch:train:11901-12000batch: iter_time=3.643e-04, forward_time=0.246, loss_ctc=95.788, loss_interctc_layer6=92.426, loss_interctc_layer12=77.351, loss_interctc_layer15=71.312, loss_interctc_layer21=99.108, loss=87.197, backward_time=0.249, grad_norm=67.161, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=8.568e-05, train_time=1.578 +[gpua006:0/64] 2024-02-08 23:23:12,007 (trainer:756) INFO: 22epoch:train:12001-12100batch: iter_time=8.541e-05, forward_time=0.144, loss_ctc=72.865, loss_interctc_layer6=80.829, loss_interctc_layer12=67.010, loss_interctc_layer15=61.405, loss_interctc_layer21=75.119, loss=71.446, backward_time=0.209, grad_norm=75.030, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.567e-05, train_time=1.356 +[gpua006:0/64] 2024-02-08 23:25:33,141 (trainer:756) INFO: 22epoch:train:12101-12200batch: iter_time=3.455e-04, forward_time=0.191, loss_ctc=64.747, loss_interctc_layer6=77.528, loss_interctc_layer12=64.088, loss_interctc_layer15=58.642, loss_interctc_layer21=66.777, loss=66.356, backward_time=0.279, grad_norm=50.976, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.143, optim0_lr0=8.566e-05, train_time=1.412 +[gpua006:0/64] 2024-02-08 23:27:06,311 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-08 23:28:16,164 (trainer:756) INFO: 22epoch:train:12201-12300batch: iter_time=9.312e-05, forward_time=0.143, loss_ctc=86.291, loss_interctc_layer6=88.406, loss_interctc_layer12=74.116, loss_interctc_layer15=68.428, loss_interctc_layer21=89.022, loss=81.252, backward_time=0.209, grad_norm=81.635, clip=100.000, loss_scale=6.433e+31, optim_step_time=0.137, optim0_lr0=8.564e-05, train_time=1.630 +[gpua006:0/64] 2024-02-08 23:30:36,404 (trainer:756) INFO: 22epoch:train:12301-12400batch: iter_time=9.227e-05, forward_time=0.143, loss_ctc=87.166, loss_interctc_layer6=85.519, loss_interctc_layer12=71.620, loss_interctc_layer15=65.888, loss_interctc_layer21=89.919, loss=80.022, backward_time=0.210, grad_norm=73.918, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.563e-05, train_time=1.402 +[gpua006:0/64] 2024-02-08 23:33:12,523 (trainer:756) INFO: 22epoch:train:12401-12500batch: iter_time=1.025e-04, forward_time=0.233, loss_ctc=84.666, loss_interctc_layer6=83.310, loss_interctc_layer12=69.444, loss_interctc_layer15=64.021, loss_interctc_layer21=87.636, loss=77.815, backward_time=0.248, grad_norm=71.390, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.562e-05, train_time=1.562 +[gpua006:0/64] 2024-02-08 23:33:32,668 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-08 23:33:51,362 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-08 23:33:54,837 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-08 23:33:54,838 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-08 23:33:54,841 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-08 23:43:59,763 (trainer:756) INFO: 22epoch:train:12501-12600batch: iter_time=5.352, forward_time=0.187, loss_ctc=74.492, loss_interctc_layer6=81.520, loss_interctc_layer12=67.417, loss_interctc_layer15=61.747, loss_interctc_layer21=76.907, loss=72.417, backward_time=0.216, grad_norm=57.549, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.561e-05, train_time=6.472 +[gpua006:0/64] 2024-02-08 23:45:44,819 (trainer:756) INFO: 22epoch:train:12601-12700batch: iter_time=9.208e-05, forward_time=0.142, loss_ctc=70.242, loss_interctc_layer6=75.015, loss_interctc_layer12=62.196, loss_interctc_layer15=57.063, loss_interctc_layer21=72.353, loss=67.374, backward_time=0.209, grad_norm=93.376, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.559e-05, train_time=1.051 +[gpua006:0/64] 2024-02-08 23:48:06,965 (trainer:756) INFO: 22epoch:train:12701-12800batch: iter_time=9.198e-05, forward_time=0.191, loss_ctc=82.823, loss_interctc_layer6=87.496, loss_interctc_layer12=73.321, loss_interctc_layer15=67.528, loss_interctc_layer21=85.173, loss=79.268, backward_time=0.297, grad_norm=80.096, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.143, optim0_lr0=8.558e-05, train_time=1.421 +[gpua006:0/64] 2024-02-08 23:50:20,203 (trainer:756) INFO: 22epoch:train:12801-12900batch: iter_time=9.542e-05, forward_time=0.142, loss_ctc=86.583, loss_interctc_layer6=85.477, loss_interctc_layer12=70.674, loss_interctc_layer15=64.691, loss_interctc_layer21=89.741, loss=79.433, backward_time=0.208, grad_norm=95.821, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.557e-05, train_time=1.333 +[gpua006:0/64] 2024-02-08 23:52:38,461 (trainer:756) INFO: 22epoch:train:12901-13000batch: iter_time=1.613e-04, forward_time=0.322, loss_ctc=72.722, loss_interctc_layer6=80.587, loss_interctc_layer12=66.620, loss_interctc_layer15=61.128, loss_interctc_layer21=75.145, loss=71.241, backward_time=0.237, grad_norm=58.941, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.555e-05, train_time=1.381 +[gpua006:0/64] 2024-02-08 23:53:12,778 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-08 23:54:50,645 (trainer:756) INFO: 22epoch:train:13001-13100batch: iter_time=9.734e-05, forward_time=0.143, loss_ctc=95.899, loss_interctc_layer6=93.124, loss_interctc_layer12=77.600, loss_interctc_layer15=71.338, loss_interctc_layer21=99.533, loss=87.499, backward_time=0.208, grad_norm=78.477, clip=100.000, loss_scale=2.540e+31, optim_step_time=0.137, optim0_lr0=8.554e-05, train_time=1.322 +[gpua006:0/64] 2024-02-08 23:57:13,775 (trainer:756) INFO: 22epoch:train:13101-13200batch: iter_time=9.936e-05, forward_time=0.249, loss_ctc=78.001, loss_interctc_layer6=86.590, loss_interctc_layer12=72.168, loss_interctc_layer15=66.239, loss_interctc_layer21=80.450, loss=76.690, backward_time=0.246, grad_norm=66.622, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.553e-05, train_time=1.431 +[gpua006:0/64] 2024-02-08 23:59:31,058 (trainer:756) INFO: 22epoch:train:13201-13300batch: iter_time=1.011e-04, forward_time=0.168, loss_ctc=82.752, loss_interctc_layer6=82.272, loss_interctc_layer12=68.323, loss_interctc_layer15=63.063, loss_interctc_layer21=85.472, loss=76.376, backward_time=0.208, grad_norm=76.492, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.551e-05, train_time=1.372 +[gpua006:0/64] 2024-02-09 00:01:52,366 (trainer:756) INFO: 22epoch:train:13301-13400batch: iter_time=9.380e-05, forward_time=0.147, loss_ctc=74.004, loss_interctc_layer6=87.368, loss_interctc_layer12=72.844, loss_interctc_layer15=67.049, loss_interctc_layer21=76.259, loss=75.505, backward_time=0.214, grad_norm=61.711, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.550e-05, train_time=1.413 +[gpua006:0/64] 2024-02-09 00:04:08,878 (trainer:756) INFO: 22epoch:train:13401-13500batch: iter_time=6.625e-04, forward_time=0.281, loss_ctc=82.079, loss_interctc_layer6=83.528, loss_interctc_layer12=69.214, loss_interctc_layer15=63.427, loss_interctc_layer21=84.743, loss=76.598, backward_time=0.235, grad_norm=74.024, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=8.549e-05, train_time=1.364 +[gpua006:0/64] 2024-02-09 00:06:27,344 (trainer:756) INFO: 22epoch:train:13501-13600batch: iter_time=1.046e-04, forward_time=0.164, loss_ctc=87.831, loss_interctc_layer6=88.269, loss_interctc_layer12=74.277, loss_interctc_layer15=68.704, loss_interctc_layer21=90.632, loss=81.943, backward_time=0.221, grad_norm=67.259, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.548e-05, train_time=1.385 +[gpua006:0/64] 2024-02-09 00:08:35,041 (trainer:756) INFO: 22epoch:train:13601-13700batch: iter_time=2.577e-04, forward_time=0.234, loss_ctc=80.350, loss_interctc_layer6=82.022, loss_interctc_layer12=68.292, loss_interctc_layer15=62.720, loss_interctc_layer21=83.018, loss=75.281, backward_time=0.240, grad_norm=157.430, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.546e-05, train_time=1.277 +[gpua006:0/64] 2024-02-09 00:09:57,261 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-09 00:10:15,576 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 00:10:19,098 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 00:10:19,098 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-09 00:10:19,123 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 00:20:51,187 (trainer:756) INFO: 22epoch:train:13701-13800batch: iter_time=6.177, forward_time=0.196, loss_ctc=83.579, loss_interctc_layer6=82.887, loss_interctc_layer12=68.780, loss_interctc_layer15=63.074, loss_interctc_layer21=86.549, loss=76.974, backward_time=0.220, grad_norm=74.816, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.545e-05, train_time=7.361 +[gpua006:0/64] 2024-02-09 00:23:11,630 (trainer:756) INFO: 22epoch:train:13801-13900batch: iter_time=8.418e-05, forward_time=0.143, loss_ctc=63.796, loss_interctc_layer6=75.612, loss_interctc_layer12=62.735, loss_interctc_layer15=57.468, loss_interctc_layer21=65.815, loss=65.085, backward_time=0.209, grad_norm=60.243, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.544e-05, train_time=1.404 +[gpua006:0/64] 2024-02-09 00:25:17,284 (trainer:756) INFO: 22epoch:train:13901-14000batch: iter_time=8.731e-05, forward_time=0.143, loss_ctc=74.790, loss_interctc_layer6=78.605, loss_interctc_layer12=65.446, loss_interctc_layer15=60.082, loss_interctc_layer21=77.130, loss=71.211, backward_time=0.208, grad_norm=64.627, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.542e-05, train_time=1.256 +[gpua006:0/64] 2024-02-09 00:27:43,791 (trainer:756) INFO: 22epoch:train:14001-14100batch: iter_time=5.233e-04, forward_time=0.193, loss_ctc=82.539, loss_interctc_layer6=86.240, loss_interctc_layer12=72.174, loss_interctc_layer15=66.222, loss_interctc_layer21=85.204, loss=78.476, backward_time=0.289, grad_norm=81.931, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.147, optim0_lr0=8.541e-05, train_time=1.464 +[gpua006:0/64] 2024-02-09 00:30:07,468 (trainer:756) INFO: 22epoch:train:14101-14200batch: iter_time=1.055e-04, forward_time=0.143, loss_ctc=84.818, loss_interctc_layer6=84.853, loss_interctc_layer12=70.054, loss_interctc_layer15=64.218, loss_interctc_layer21=87.902, loss=78.369, backward_time=0.207, grad_norm=62.126, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.540e-05, train_time=1.436 +[gpua006:0/64] 2024-02-09 00:31:14,763 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 00:31:48,778 (trainer:756) INFO: 22epoch:train:14201-14300batch: iter_time=1.050e-04, forward_time=0.144, loss_ctc=80.106, loss_interctc_layer6=83.491, loss_interctc_layer12=69.302, loss_interctc_layer15=63.500, loss_interctc_layer21=82.982, loss=75.876, backward_time=0.210, grad_norm=64.527, clip=100.000, loss_scale=1.659e+31, optim_step_time=0.136, optim0_lr0=8.538e-05, train_time=1.014 +[gpua006:0/64] 2024-02-09 00:33:57,145 (trainer:756) INFO: 22epoch:train:14301-14400batch: iter_time=4.556e-04, forward_time=0.231, loss_ctc=85.835, loss_interctc_layer6=93.351, loss_interctc_layer12=78.150, loss_interctc_layer15=71.993, loss_interctc_layer21=88.807, loss=83.627, backward_time=0.254, grad_norm=91.521, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.146, optim0_lr0=8.537e-05, train_time=1.281 +[gpua006:0/64] 2024-02-09 00:36:53,292 (trainer:756) INFO: 22epoch:train:14401-14500batch: iter_time=9.809e-05, forward_time=0.182, loss_ctc=95.277, loss_interctc_layer6=91.965, loss_interctc_layer12=76.776, loss_interctc_layer15=70.743, loss_interctc_layer21=98.413, loss=86.635, backward_time=0.206, grad_norm=80.784, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.536e-05, train_time=1.763 +[gpua006:0/64] 2024-02-09 00:39:12,324 (trainer:756) INFO: 22epoch:train:14501-14600batch: iter_time=9.964e-05, forward_time=0.143, loss_ctc=73.334, loss_interctc_layer6=81.382, loss_interctc_layer12=67.499, loss_interctc_layer15=61.894, loss_interctc_layer21=75.728, loss=71.967, backward_time=0.208, grad_norm=77.616, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.535e-05, train_time=1.390 +[gpua006:0/64] 2024-02-09 00:41:55,633 (trainer:756) INFO: 22epoch:train:14601-14700batch: iter_time=1.129e-04, forward_time=0.224, loss_ctc=64.577, loss_interctc_layer6=77.538, loss_interctc_layer12=64.032, loss_interctc_layer15=58.576, loss_interctc_layer21=66.592, loss=66.263, backward_time=0.226, grad_norm=65.534, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.144, optim0_lr0=8.533e-05, train_time=1.633 +[gpua006:0/64] 2024-02-09 00:44:06,259 (trainer:756) INFO: 22epoch:train:14701-14800batch: iter_time=1.020e-04, forward_time=0.143, loss_ctc=85.415, loss_interctc_layer6=88.226, loss_interctc_layer12=73.977, loss_interctc_layer15=68.468, loss_interctc_layer21=87.942, loss=80.806, backward_time=0.208, grad_norm=77.712, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.532e-05, train_time=1.306 +[gpua006:0/64] 2024-02-09 00:46:09,897 (trainer:756) INFO: 22epoch:train:14801-14900batch: iter_time=9.420e-05, forward_time=0.143, loss_ctc=86.584, loss_interctc_layer6=85.525, loss_interctc_layer12=71.135, loss_interctc_layer15=65.578, loss_interctc_layer21=89.416, loss=79.648, backward_time=0.209, grad_norm=67.066, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.531e-05, train_time=1.236 +[gpua006:0/64] 2024-02-09 00:48:12,091 (trainer:756) INFO: 22epoch:train:14901-15000batch: iter_time=1.935e-04, forward_time=0.307, loss_ctc=84.908, loss_interctc_layer6=83.570, loss_interctc_layer12=69.660, loss_interctc_layer15=64.056, loss_interctc_layer21=88.080, loss=78.055, backward_time=0.258, grad_norm=58.060, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=8.529e-05, train_time=1.221 +[gpua006:0/64] 2024-02-09 01:20:47,344 (trainer:355) INFO: 22epoch results: [train] iter_time=0.379, forward_time=0.351, loss_ctc=79.494, loss_interctc_layer6=85.058, loss_interctc_layer12=70.942, loss_interctc_layer15=65.298, loss_interctc_layer21=82.095, loss=76.578, backward_time=0.326, grad_norm=74.469, clip=100.000, loss_scale=3.274e+31, optim_step_time=0.149, optim0_lr0=8.628e-05, train_time=2.479, time=10 hours, 20 minutes and 11.71 seconds, total_count=330000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=47.608, cer_ctc=0.216, loss_interctc_layer6=52.559, cer_interctc_layer6=0.232, loss_interctc_layer12=39.619, cer_interctc_layer12=0.164, loss_interctc_layer15=35.411, cer_interctc_layer15=0.139, loss_interctc_layer21=50.076, cer_interctc_layer21=0.227, loss=45.054, time=32 minutes and 10.87 seconds, total_count=102762, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-09 01:21:16,563 (trainer:410) INFO: The best model has been updated: valid.total_count +[gpua006:0/64] 2024-02-09 01:21:16,630 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/17epoch.pth +[gpua006:0/64] 2024-02-09 01:21:16,631 (trainer:289) INFO: 23/45epoch started. Estimated time to finish: 1 week, 3 days and 10 hours +[gpua006:0/64] 2024-02-09 01:21:16,645 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-09 01:21:34,325 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 01:21:37,607 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 01:21:37,608 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-09 01:21:37,611 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 01:28:26,080 (trainer:756) INFO: 23epoch:train:1-100batch: iter_time=3.261, forward_time=0.177, loss_ctc=85.516, loss_interctc_layer6=87.365, loss_interctc_layer12=73.096, loss_interctc_layer15=67.275, loss_interctc_layer21=88.254, loss=80.301, backward_time=0.222, grad_norm=65.093, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=8.528e-05, train_time=4.294 +[gpua006:0/64] 2024-02-09 01:30:02,323 (trainer:756) INFO: 23epoch:train:101-200batch: iter_time=9.504e-05, forward_time=0.143, loss_ctc=89.191, loss_interctc_layer6=97.096, loss_interctc_layer12=82.319, loss_interctc_layer15=76.612, loss_interctc_layer21=91.619, loss=87.368, backward_time=0.212, grad_norm=75.542, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.527e-05, train_time=0.962 +[gpua006:0/64] 2024-02-09 01:32:06,043 (trainer:756) INFO: 23epoch:train:201-300batch: iter_time=5.259e-04, forward_time=0.215, loss_ctc=77.552, loss_interctc_layer6=83.326, loss_interctc_layer12=69.828, loss_interctc_layer15=64.472, loss_interctc_layer21=79.916, loss=75.019, backward_time=0.260, grad_norm=82.143, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.146, optim0_lr0=8.526e-05, train_time=1.236 +[gpua006:0/64] 2024-02-09 01:34:06,067 (trainer:756) INFO: 23epoch:train:301-400batch: iter_time=9.766e-05, forward_time=0.143, loss_ctc=77.688, loss_interctc_layer6=88.737, loss_interctc_layer12=75.040, loss_interctc_layer15=69.327, loss_interctc_layer21=79.861, loss=78.130, backward_time=0.209, grad_norm=70.310, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.524e-05, train_time=1.201 +[gpua006:0/64] 2024-02-09 01:36:10,366 (trainer:756) INFO: 23epoch:train:401-500batch: iter_time=9.546e-05, forward_time=0.146, loss_ctc=77.004, loss_interctc_layer6=82.205, loss_interctc_layer12=68.693, loss_interctc_layer15=63.295, loss_interctc_layer21=79.350, loss=74.109, backward_time=0.207, grad_norm=60.332, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=8.523e-05, train_time=1.243 +[gpua006:0/64] 2024-02-09 01:38:17,730 (trainer:756) INFO: 23epoch:train:501-600batch: iter_time=3.538e-04, forward_time=0.236, loss_ctc=77.090, loss_interctc_layer6=78.333, loss_interctc_layer12=65.904, loss_interctc_layer15=61.088, loss_interctc_layer21=79.475, loss=72.378, backward_time=0.256, grad_norm=64.687, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=8.522e-05, train_time=1.271 +[gpua006:0/64] 2024-02-09 01:40:40,381 (trainer:756) INFO: 23epoch:train:601-700batch: iter_time=9.906e-05, forward_time=0.144, loss_ctc=87.985, loss_interctc_layer6=89.440, loss_interctc_layer12=74.388, loss_interctc_layer15=68.514, loss_interctc_layer21=91.111, loss=82.288, backward_time=0.208, grad_norm=76.430, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.520e-05, train_time=1.428 +[gpua006:0/64] 2024-02-09 01:43:04,104 (trainer:756) INFO: 23epoch:train:701-800batch: iter_time=9.967e-05, forward_time=0.153, loss_ctc=92.391, loss_interctc_layer6=83.862, loss_interctc_layer12=69.755, loss_interctc_layer15=64.153, loss_interctc_layer21=95.602, loss=81.153, backward_time=0.208, grad_norm=69.585, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=8.519e-05, train_time=1.438 +[gpua006:0/64] 2024-02-09 01:45:36,243 (trainer:756) INFO: 23epoch:train:801-900batch: iter_time=2.173e-04, forward_time=0.265, loss_ctc=85.007, loss_interctc_layer6=85.159, loss_interctc_layer12=71.418, loss_interctc_layer15=65.946, loss_interctc_layer21=87.773, loss=79.061, backward_time=0.239, grad_norm=66.428, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=8.518e-05, train_time=1.519 +[gpua006:0/64] 2024-02-09 01:48:20,219 (trainer:756) INFO: 23epoch:train:901-1000batch: iter_time=9.008e-05, forward_time=0.143, loss_ctc=76.295, loss_interctc_layer6=77.851, loss_interctc_layer12=64.653, loss_interctc_layer15=59.477, loss_interctc_layer21=78.515, loss=71.358, backward_time=0.207, grad_norm=75.184, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.516e-05, train_time=1.641 +[gpua006:0/64] 2024-02-09 01:50:24,727 (trainer:756) INFO: 23epoch:train:1001-1100batch: iter_time=9.093e-05, forward_time=0.155, loss_ctc=78.138, loss_interctc_layer6=84.271, loss_interctc_layer12=70.326, loss_interctc_layer15=64.721, loss_interctc_layer21=80.600, loss=75.611, backward_time=0.208, grad_norm=58.024, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.515e-05, train_time=1.245 +[gpua006:0/64] 2024-02-09 01:52:30,306 (trainer:756) INFO: 23epoch:train:1101-1200batch: iter_time=2.155e-04, forward_time=0.240, loss_ctc=84.210, loss_interctc_layer6=92.209, loss_interctc_layer12=78.251, loss_interctc_layer15=72.566, loss_interctc_layer21=86.523, loss=82.752, backward_time=0.246, grad_norm=71.381, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=8.514e-05, train_time=1.252 +[gpua006:0/64] 2024-02-09 01:53:41,961 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-09 01:54:00,510 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 01:54:04,124 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 01:54:04,124 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-09 01:54:04,128 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 02:01:32,789 (trainer:756) INFO: 23epoch:train:1201-1300batch: iter_time=3.079, forward_time=0.175, loss_ctc=72.381, loss_interctc_layer6=78.016, loss_interctc_layer12=65.034, loss_interctc_layer15=59.902, loss_interctc_layer21=74.902, loss=70.047, backward_time=0.217, grad_norm=68.108, clip=100.000, loss_scale=1.379e+31, optim_step_time=0.138, optim0_lr0=8.513e-05, train_time=5.428 +[gpua006:0/64] 2024-02-09 02:03:15,453 (trainer:756) INFO: 23epoch:train:1301-1400batch: iter_time=8.425e-05, forward_time=0.144, loss_ctc=91.654, loss_interctc_layer6=101.421, loss_interctc_layer12=85.201, loss_interctc_layer15=78.790, loss_interctc_layer21=94.628, loss=90.339, backward_time=0.210, grad_norm=72.507, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.511e-05, train_time=1.027 +[gpua006:0/64] 2024-02-09 02:04:55,867 (trainer:756) INFO: 23epoch:train:1401-1500batch: iter_time=8.241e-05, forward_time=0.144, loss_ctc=72.954, loss_interctc_layer6=87.062, loss_interctc_layer12=73.490, loss_interctc_layer15=67.986, loss_interctc_layer21=75.157, loss=75.330, backward_time=0.210, grad_norm=70.290, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.510e-05, train_time=1.004 +[gpua006:0/64] 2024-02-09 02:07:23,691 (trainer:756) INFO: 23epoch:train:1501-1600batch: iter_time=0.009, forward_time=0.267, loss_ctc=73.800, loss_interctc_layer6=85.225, loss_interctc_layer12=71.251, loss_interctc_layer15=65.665, loss_interctc_layer21=76.000, loss=74.388, backward_time=0.236, grad_norm=67.520, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.509e-05, train_time=1.477 +[gpua006:0/64] 2024-02-09 02:09:28,718 (trainer:756) INFO: 23epoch:train:1601-1700batch: iter_time=8.727e-05, forward_time=0.164, loss_ctc=75.543, loss_interctc_layer6=86.762, loss_interctc_layer12=72.981, loss_interctc_layer15=67.430, loss_interctc_layer21=77.926, loss=76.128, backward_time=0.209, grad_norm=70.536, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.508e-05, train_time=1.251 +[gpua006:0/64] 2024-02-09 02:12:09,887 (trainer:756) INFO: 23epoch:train:1701-1800batch: iter_time=8.323e-05, forward_time=0.141, loss_ctc=72.406, loss_interctc_layer6=79.662, loss_interctc_layer12=66.417, loss_interctc_layer15=61.119, loss_interctc_layer21=74.684, loss=70.858, backward_time=0.207, grad_norm=56.722, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.506e-05, train_time=1.611 +[gpua006:0/64] 2024-02-09 02:14:18,419 (trainer:756) INFO: 23epoch:train:1801-1900batch: iter_time=8.506e-05, forward_time=0.141, loss_ctc=77.766, loss_interctc_layer6=79.790, loss_interctc_layer12=66.644, loss_interctc_layer15=61.354, loss_interctc_layer21=80.526, loss=73.216, backward_time=0.207, grad_norm=71.523, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.505e-05, train_time=1.285 +[gpua006:0/64] 2024-02-09 02:17:09,173 (trainer:756) INFO: 23epoch:train:1901-2000batch: iter_time=0.002, forward_time=0.203, loss_ctc=93.747, loss_interctc_layer6=94.044, loss_interctc_layer12=78.381, loss_interctc_layer15=71.980, loss_interctc_layer21=96.767, loss=86.984, backward_time=0.286, grad_norm=71.069, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=8.504e-05, train_time=1.706 +[gpua006:0/64] 2024-02-09 02:19:14,054 (trainer:756) INFO: 23epoch:train:2001-2100batch: iter_time=8.881e-05, forward_time=0.141, loss_ctc=74.549, loss_interctc_layer6=75.481, loss_interctc_layer12=62.517, loss_interctc_layer15=57.475, loss_interctc_layer21=77.243, loss=69.453, backward_time=0.208, grad_norm=55.252, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.502e-05, train_time=1.249 +[gpua006:0/64] 2024-02-09 02:21:14,885 (trainer:756) INFO: 23epoch:train:2101-2200batch: iter_time=8.793e-05, forward_time=0.141, loss_ctc=70.550, loss_interctc_layer6=78.439, loss_interctc_layer12=65.734, loss_interctc_layer15=60.613, loss_interctc_layer21=72.574, loss=69.582, backward_time=0.208, grad_norm=55.467, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.501e-05, train_time=1.209 +[gpua006:0/64] 2024-02-09 02:23:07,704 (trainer:756) INFO: 23epoch:train:2201-2300batch: iter_time=8.531e-05, forward_time=0.143, loss_ctc=81.894, loss_interctc_layer6=85.045, loss_interctc_layer12=71.011, loss_interctc_layer15=65.321, loss_interctc_layer21=84.555, loss=77.565, backward_time=0.209, grad_norm=64.213, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.500e-05, train_time=1.128 +[gpua006:0/64] 2024-02-09 02:25:22,840 (trainer:756) INFO: 23epoch:train:2301-2400batch: iter_time=8.535e-05, forward_time=0.141, loss_ctc=74.254, loss_interctc_layer6=84.793, loss_interctc_layer12=70.832, loss_interctc_layer15=65.305, loss_interctc_layer21=76.381, loss=74.313, backward_time=0.208, grad_norm=102.402, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.499e-05, train_time=1.351 +[gpua006:0/64] 2024-02-09 02:27:36,285 (trainer:756) INFO: 23epoch:train:2401-2500batch: iter_time=9.323e-05, forward_time=0.233, loss_ctc=80.451, loss_interctc_layer6=91.708, loss_interctc_layer12=77.174, loss_interctc_layer15=71.475, loss_interctc_layer21=82.842, loss=80.730, backward_time=0.256, grad_norm=71.333, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.497e-05, train_time=1.333 +[gpua006:0/64] 2024-02-09 02:27:53,632 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-09 02:28:12,086 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 02:28:15,692 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 02:28:15,692 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-09 02:28:15,695 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 02:34:44,273 (trainer:756) INFO: 23epoch:train:2501-2600batch: iter_time=3.206, forward_time=0.183, loss_ctc=84.687, loss_interctc_layer6=86.789, loss_interctc_layer12=72.291, loss_interctc_layer15=66.431, loss_interctc_layer21=87.534, loss=79.546, backward_time=0.223, grad_norm=74.948, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.496e-05, train_time=4.281 +[gpua006:0/64] 2024-02-09 02:36:55,660 (trainer:756) INFO: 23epoch:train:2601-2700batch: iter_time=9.273e-05, forward_time=0.172, loss_ctc=87.280, loss_interctc_layer6=95.315, loss_interctc_layer12=80.556, loss_interctc_layer15=74.876, loss_interctc_layer21=90.102, loss=85.626, backward_time=0.249, grad_norm=88.722, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.495e-05, train_time=1.313 +[gpua006:0/64] 2024-02-09 02:39:18,026 (trainer:756) INFO: 23epoch:train:2701-2800batch: iter_time=8.962e-05, forward_time=0.143, loss_ctc=76.727, loss_interctc_layer6=82.477, loss_interctc_layer12=68.927, loss_interctc_layer15=63.578, loss_interctc_layer21=79.001, loss=74.142, backward_time=0.209, grad_norm=66.595, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.493e-05, train_time=1.424 +[gpua006:0/64] 2024-02-09 02:41:16,147 (trainer:756) INFO: 23epoch:train:2801-2900batch: iter_time=9.359e-05, forward_time=0.143, loss_ctc=75.279, loss_interctc_layer6=87.170, loss_interctc_layer12=73.104, loss_interctc_layer15=67.482, loss_interctc_layer21=77.486, loss=76.105, backward_time=0.208, grad_norm=71.119, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.492e-05, train_time=1.181 +[gpua006:0/64] 2024-02-09 02:43:55,596 (trainer:756) INFO: 23epoch:train:2901-3000batch: iter_time=9.285e-05, forward_time=0.142, loss_ctc=75.867, loss_interctc_layer6=81.135, loss_interctc_layer12=67.608, loss_interctc_layer15=62.227, loss_interctc_layer21=78.232, loss=73.014, backward_time=0.207, grad_norm=58.759, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.491e-05, train_time=1.594 +[gpua006:0/64] 2024-02-09 02:46:23,235 (trainer:756) INFO: 23epoch:train:3001-3100batch: iter_time=4.592e-04, forward_time=0.198, loss_ctc=75.239, loss_interctc_layer6=76.812, loss_interctc_layer12=64.545, loss_interctc_layer15=59.560, loss_interctc_layer21=77.654, loss=70.762, backward_time=0.275, grad_norm=61.163, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.144, optim0_lr0=8.490e-05, train_time=1.476 +[gpua006:0/64] 2024-02-09 02:48:33,643 (trainer:756) INFO: 23epoch:train:3101-3200batch: iter_time=8.815e-05, forward_time=0.144, loss_ctc=86.481, loss_interctc_layer6=87.563, loss_interctc_layer12=72.629, loss_interctc_layer15=66.654, loss_interctc_layer21=89.373, loss=80.540, backward_time=0.207, grad_norm=133.150, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.488e-05, train_time=1.304 +[gpua006:0/64] 2024-02-09 02:50:31,502 (trainer:756) INFO: 23epoch:train:3201-3300batch: iter_time=7.839e-05, forward_time=0.142, loss_ctc=92.028, loss_interctc_layer6=83.466, loss_interctc_layer12=69.162, loss_interctc_layer15=63.466, loss_interctc_layer21=95.351, loss=80.695, backward_time=0.209, grad_norm=61.296, clip=100.000, loss_scale=2.758e+31, optim_step_time=0.137, optim0_lr0=8.487e-05, train_time=1.179 +[gpua006:0/64] 2024-02-09 02:52:30,458 (trainer:756) INFO: 23epoch:train:3301-3400batch: iter_time=9.292e-05, forward_time=0.143, loss_ctc=85.554, loss_interctc_layer6=84.384, loss_interctc_layer12=70.600, loss_interctc_layer15=65.113, loss_interctc_layer21=88.085, loss=78.747, backward_time=0.208, grad_norm=70.155, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.486e-05, train_time=1.189 +[gpua006:0/64] 2024-02-09 02:55:03,147 (trainer:756) INFO: 23epoch:train:3401-3500batch: iter_time=1.044e-04, forward_time=0.142, loss_ctc=76.037, loss_interctc_layer6=77.830, loss_interctc_layer12=64.609, loss_interctc_layer15=59.248, loss_interctc_layer21=78.578, loss=71.260, backward_time=0.207, grad_norm=61.941, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.485e-05, train_time=1.527 +[gpua006:0/64] 2024-02-09 02:57:28,835 (trainer:756) INFO: 23epoch:train:3501-3600batch: iter_time=9.001e-05, forward_time=0.193, loss_ctc=75.058, loss_interctc_layer6=82.557, loss_interctc_layer12=68.552, loss_interctc_layer15=62.916, loss_interctc_layer21=77.425, loss=73.302, backward_time=0.281, grad_norm=63.539, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.146, optim0_lr0=8.483e-05, train_time=1.456 +[gpua006:0/64] 2024-02-09 02:59:35,425 (trainer:756) INFO: 23epoch:train:3601-3700batch: iter_time=8.790e-05, forward_time=0.142, loss_ctc=81.893, loss_interctc_layer6=91.240, loss_interctc_layer12=76.981, loss_interctc_layer15=71.349, loss_interctc_layer21=84.160, loss=81.125, backward_time=0.208, grad_norm=63.239, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.482e-05, train_time=1.266 +[gpua006:0/64] 2024-02-09 03:01:07,389 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-09 03:01:25,558 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 03:01:29,407 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 03:01:29,408 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-09 03:01:29,411 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 03:07:27,543 (trainer:756) INFO: 23epoch:train:3701-3800batch: iter_time=3.460, forward_time=0.183, loss_ctc=75.014, loss_interctc_layer6=77.697, loss_interctc_layer12=64.656, loss_interctc_layer15=59.397, loss_interctc_layer21=77.628, loss=70.879, backward_time=0.218, grad_norm=57.051, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.481e-05, train_time=4.721 +[gpua006:0/64] 2024-02-09 03:09:24,399 (trainer:756) INFO: 23epoch:train:3801-3900batch: iter_time=8.849e-05, forward_time=0.239, loss_ctc=96.899, loss_interctc_layer6=100.484, loss_interctc_layer12=84.255, loss_interctc_layer15=77.839, loss_interctc_layer21=99.961, loss=91.888, backward_time=0.259, grad_norm=68.801, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.479e-05, train_time=1.168 +[gpua006:0/64] 2024-02-09 03:11:14,049 (trainer:756) INFO: 23epoch:train:3901-4000batch: iter_time=8.854e-05, forward_time=0.153, loss_ctc=74.836, loss_interctc_layer6=86.479, loss_interctc_layer12=72.782, loss_interctc_layer15=67.251, loss_interctc_layer21=76.986, loss=75.667, backward_time=0.215, grad_norm=82.002, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=8.478e-05, train_time=1.096 +[gpua006:0/64] 2024-02-09 03:13:42,355 (trainer:756) INFO: 23epoch:train:4001-4100batch: iter_time=4.227e-04, forward_time=0.241, loss_ctc=77.993, loss_interctc_layer6=84.351, loss_interctc_layer12=70.456, loss_interctc_layer15=64.936, loss_interctc_layer21=80.272, loss=75.601, backward_time=0.248, grad_norm=86.334, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.144, optim0_lr0=8.477e-05, train_time=1.482 +[gpua006:0/64] 2024-02-09 03:15:46,588 (trainer:756) INFO: 23epoch:train:4101-4200batch: iter_time=9.360e-05, forward_time=0.164, loss_ctc=79.118, loss_interctc_layer6=86.326, loss_interctc_layer12=72.421, loss_interctc_layer15=67.183, loss_interctc_layer21=81.401, loss=77.290, backward_time=0.210, grad_norm=95.077, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.476e-05, train_time=1.243 +[gpua006:0/64] 2024-02-09 03:18:31,983 (trainer:756) INFO: 23epoch:train:4201-4300batch: iter_time=9.876e-05, forward_time=0.256, loss_ctc=72.585, loss_interctc_layer6=78.833, loss_interctc_layer12=65.380, loss_interctc_layer15=60.304, loss_interctc_layer21=74.782, loss=70.377, backward_time=0.243, grad_norm=73.513, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.474e-05, train_time=1.654 +[gpua006:0/64] 2024-02-09 03:20:44,344 (trainer:756) INFO: 23epoch:train:4301-4400batch: iter_time=9.044e-05, forward_time=0.161, loss_ctc=82.447, loss_interctc_layer6=80.238, loss_interctc_layer12=66.848, loss_interctc_layer15=61.540, loss_interctc_layer21=85.307, loss=75.276, backward_time=0.236, grad_norm=69.511, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.473e-05, train_time=1.322 +[gpua006:0/64] 2024-02-09 03:22:38,211 (trainer:756) INFO: 23epoch:train:4401-4500batch: iter_time=2.272e-04, forward_time=0.187, loss_ctc=101.701, loss_interctc_layer6=92.675, loss_interctc_layer12=77.000, loss_interctc_layer15=70.664, loss_interctc_layer21=105.225, loss=89.453, backward_time=0.220, grad_norm=72.977, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.472e-05, train_time=1.140 +[gpua006:0/64] 2024-02-09 03:24:53,289 (trainer:756) INFO: 23epoch:train:4501-4600batch: iter_time=8.439e-05, forward_time=0.201, loss_ctc=75.882, loss_interctc_layer6=74.677, loss_interctc_layer12=61.738, loss_interctc_layer15=56.575, loss_interctc_layer21=78.544, loss=69.483, backward_time=0.238, grad_norm=61.401, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=8.471e-05, train_time=1.350 +[gpua006:0/64] 2024-02-09 03:27:05,703 (trainer:756) INFO: 23epoch:train:4601-4700batch: iter_time=6.260e-04, forward_time=0.190, loss_ctc=70.548, loss_interctc_layer6=77.395, loss_interctc_layer12=64.513, loss_interctc_layer15=59.546, loss_interctc_layer21=72.736, loss=68.948, backward_time=0.227, grad_norm=82.542, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.469e-05, train_time=1.325 +[gpua006:0/64] 2024-02-09 03:29:24,784 (trainer:756) INFO: 23epoch:train:4701-4800batch: iter_time=7.903e-05, forward_time=0.218, loss_ctc=85.033, loss_interctc_layer6=83.660, loss_interctc_layer12=69.673, loss_interctc_layer15=64.037, loss_interctc_layer21=87.774, loss=78.036, backward_time=0.233, grad_norm=64.332, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.468e-05, train_time=1.389 +[gpua006:0/64] 2024-02-09 03:31:25,190 (trainer:756) INFO: 23epoch:train:4801-4900batch: iter_time=3.664e-04, forward_time=0.182, loss_ctc=74.847, loss_interctc_layer6=84.025, loss_interctc_layer12=70.102, loss_interctc_layer15=64.567, loss_interctc_layer21=77.114, loss=74.131, backward_time=0.223, grad_norm=62.761, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.467e-05, train_time=1.205 +[gpua006:0/64] 2024-02-09 03:33:48,097 (trainer:756) INFO: 23epoch:train:4901-5000batch: iter_time=8.071e-05, forward_time=0.177, loss_ctc=82.737, loss_interctc_layer6=91.278, loss_interctc_layer12=76.674, loss_interctc_layer15=70.902, loss_interctc_layer21=85.132, loss=81.345, backward_time=0.258, grad_norm=65.635, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.465e-05, train_time=1.427 +[gpua006:0/64] 2024-02-09 03:34:05,498 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-09 03:34:24,066 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 03:34:27,494 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 03:34:27,495 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-09 03:34:27,525 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 03:43:36,877 (trainer:756) INFO: 23epoch:train:5001-5100batch: iter_time=2.823, forward_time=0.486, loss_ctc=79.157, loss_interctc_layer6=86.050, loss_interctc_layer12=71.543, loss_interctc_layer15=65.617, loss_interctc_layer21=81.962, loss=76.866, backward_time=0.343, grad_norm=72.306, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.169, optim0_lr0=8.464e-05, train_time=5.887 +[gpua006:0/64] 2024-02-09 03:46:34,138 (trainer:756) INFO: 23epoch:train:5101-5200batch: iter_time=4.847e-04, forward_time=0.636, loss_ctc=81.515, loss_interctc_layer6=94.592, loss_interctc_layer12=79.687, loss_interctc_layer15=73.786, loss_interctc_layer21=84.384, loss=82.793, backward_time=0.374, grad_norm=75.277, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.159, optim0_lr0=8.463e-05, train_time=1.773 +[gpua006:0/64] 2024-02-09 03:50:05,274 (trainer:756) INFO: 23epoch:train:5201-5300batch: iter_time=7.424e-04, forward_time=0.835, loss_ctc=71.942, loss_interctc_layer6=81.550, loss_interctc_layer12=68.096, loss_interctc_layer15=62.714, loss_interctc_layer21=74.223, loss=71.705, backward_time=0.479, grad_norm=61.771, clip=100.000, loss_scale=5.517e+31, optim_step_time=0.169, optim0_lr0=8.462e-05, train_time=2.111 +[gpua006:0/64] 2024-02-09 03:53:46,569 (trainer:756) INFO: 23epoch:train:5301-5400batch: iter_time=0.001, forward_time=0.787, loss_ctc=72.418, loss_interctc_layer6=87.138, loss_interctc_layer12=72.899, loss_interctc_layer15=67.303, loss_interctc_layer21=74.758, loss=74.903, backward_time=0.437, grad_norm=69.623, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.166, optim0_lr0=8.460e-05, train_time=2.215 +[gpua006:0/64] 2024-02-09 03:57:28,188 (trainer:756) INFO: 23epoch:train:5401-5500batch: iter_time=1.039e-04, forward_time=0.706, loss_ctc=70.565, loss_interctc_layer6=81.241, loss_interctc_layer12=67.638, loss_interctc_layer15=62.356, loss_interctc_layer21=72.775, loss=70.915, backward_time=0.472, grad_norm=69.034, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.163, optim0_lr0=8.459e-05, train_time=2.216 +[gpua006:0/64] 2024-02-09 04:01:08,932 (trainer:756) INFO: 23epoch:train:5501-5600batch: iter_time=1.953e-04, forward_time=0.652, loss_ctc=71.926, loss_interctc_layer6=76.326, loss_interctc_layer12=63.913, loss_interctc_layer15=58.890, loss_interctc_layer21=74.478, loss=69.106, backward_time=0.438, grad_norm=62.290, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.161, optim0_lr0=8.458e-05, train_time=2.206 +[gpua006:0/64] 2024-02-09 04:04:35,222 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 04:04:37,092 (trainer:756) INFO: 23epoch:train:5601-5700batch: iter_time=0.004, forward_time=0.573, loss_ctc=83.900, loss_interctc_layer6=87.792, loss_interctc_layer12=72.682, loss_interctc_layer15=66.771, loss_interctc_layer21=86.916, loss=79.612, backward_time=0.440, grad_norm=81.402, clip=100.000, loss_scale=8.072e+31, optim_step_time=0.161, optim0_lr0=8.457e-05, train_time=2.082 +[gpua006:0/64] 2024-02-09 04:07:24,369 (trainer:756) INFO: 23epoch:train:5701-5800batch: iter_time=0.001, forward_time=0.442, loss_ctc=81.693, loss_interctc_layer6=83.169, loss_interctc_layer12=68.990, loss_interctc_layer15=63.267, loss_interctc_layer21=84.704, loss=76.365, backward_time=0.300, grad_norm=61.835, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.151, optim0_lr0=8.455e-05, train_time=1.672 +[gpua006:0/64] 2024-02-09 04:09:57,905 (trainer:756) INFO: 23epoch:train:5801-5900batch: iter_time=0.001, forward_time=0.375, loss_ctc=81.702, loss_interctc_layer6=83.963, loss_interctc_layer12=70.131, loss_interctc_layer15=64.655, loss_interctc_layer21=84.314, loss=76.953, backward_time=0.361, grad_norm=61.458, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.153, optim0_lr0=8.454e-05, train_time=1.535 +[gpua006:0/64] 2024-02-09 04:12:21,698 (trainer:756) INFO: 23epoch:train:5901-6000batch: iter_time=7.078e-04, forward_time=0.408, loss_ctc=73.407, loss_interctc_layer6=77.377, loss_interctc_layer12=64.124, loss_interctc_layer15=58.922, loss_interctc_layer21=75.805, loss=69.927, backward_time=0.328, grad_norm=52.534, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.151, optim0_lr0=8.453e-05, train_time=1.437 +[gpua006:0/64] 2024-02-09 04:14:52,149 (trainer:756) INFO: 23epoch:train:6001-6100batch: iter_time=5.406e-04, forward_time=0.384, loss_ctc=72.033, loss_interctc_layer6=82.020, loss_interctc_layer12=67.988, loss_interctc_layer15=62.395, loss_interctc_layer21=74.341, loss=71.755, backward_time=0.303, grad_norm=97.899, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.148, optim0_lr0=8.452e-05, train_time=1.505 +[gpua006:0/64] 2024-02-09 04:17:29,742 (trainer:756) INFO: 23epoch:train:6101-6200batch: iter_time=2.671e-04, forward_time=0.331, loss_ctc=80.807, loss_interctc_layer6=91.181, loss_interctc_layer12=76.795, loss_interctc_layer15=71.230, loss_interctc_layer21=83.202, loss=80.643, backward_time=0.336, grad_norm=67.425, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.150, optim0_lr0=8.450e-05, train_time=1.573 +[gpua006:0/64] 2024-02-09 04:19:22,414 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-09 04:19:41,019 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 04:19:44,401 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 04:19:44,401 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-09 04:19:44,409 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 04:25:19,754 (trainer:756) INFO: 23epoch:train:6201-6300batch: iter_time=3.123, forward_time=0.227, loss_ctc=69.448, loss_interctc_layer6=77.628, loss_interctc_layer12=64.539, loss_interctc_layer15=59.192, loss_interctc_layer21=71.887, loss=68.539, backward_time=0.231, grad_norm=85.596, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.449e-05, train_time=4.705 +[gpua006:0/64] 2024-02-09 04:27:09,511 (trainer:756) INFO: 23epoch:train:6301-6400batch: iter_time=5.420e-04, forward_time=0.205, loss_ctc=89.687, loss_interctc_layer6=100.287, loss_interctc_layer12=84.090, loss_interctc_layer15=77.612, loss_interctc_layer21=92.276, loss=88.790, backward_time=0.231, grad_norm=76.599, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.144, optim0_lr0=8.448e-05, train_time=1.097 +[gpua006:0/64] 2024-02-09 04:29:09,008 (trainer:756) INFO: 23epoch:train:6401-6500batch: iter_time=2.834e-04, forward_time=0.190, loss_ctc=71.806, loss_interctc_layer6=85.942, loss_interctc_layer12=72.184, loss_interctc_layer15=66.711, loss_interctc_layer21=73.826, loss=74.094, backward_time=0.221, grad_norm=64.402, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=8.447e-05, train_time=1.195 +[gpua006:0/64] 2024-02-09 04:31:08,754 (trainer:756) INFO: 23epoch:train:6501-6600batch: iter_time=8.462e-05, forward_time=0.169, loss_ctc=71.688, loss_interctc_layer6=83.088, loss_interctc_layer12=69.141, loss_interctc_layer15=63.576, loss_interctc_layer21=73.777, loss=72.254, backward_time=0.215, grad_norm=60.964, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.445e-05, train_time=1.198 +[gpua006:0/64] 2024-02-09 04:32:46,996 (trainer:756) INFO: 23epoch:train:6601-6700batch: iter_time=8.790e-05, forward_time=0.142, loss_ctc=74.100, loss_interctc_layer6=85.309, loss_interctc_layer12=71.456, loss_interctc_layer15=65.967, loss_interctc_layer21=76.242, loss=74.615, backward_time=0.210, grad_norm=58.788, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.444e-05, train_time=0.982 +[gpua006:0/64] 2024-02-09 04:34:46,237 (trainer:756) INFO: 23epoch:train:6701-6800batch: iter_time=9.162e-05, forward_time=0.141, loss_ctc=71.081, loss_interctc_layer6=78.429, loss_interctc_layer12=65.058, loss_interctc_layer15=59.778, loss_interctc_layer21=73.397, loss=69.548, backward_time=0.209, grad_norm=59.591, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.443e-05, train_time=1.192 +[gpua006:0/64] 2024-02-09 04:37:48,461 (trainer:756) INFO: 23epoch:train:6801-6900batch: iter_time=8.946e-05, forward_time=0.155, loss_ctc=77.016, loss_interctc_layer6=79.187, loss_interctc_layer12=65.681, loss_interctc_layer15=60.485, loss_interctc_layer21=79.861, loss=72.446, backward_time=0.210, grad_norm=63.354, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.442e-05, train_time=1.822 +[gpua006:0/64] 2024-02-09 04:39:55,348 (trainer:756) INFO: 23epoch:train:6901-7000batch: iter_time=8.156e-05, forward_time=0.175, loss_ctc=92.175, loss_interctc_layer6=92.380, loss_interctc_layer12=76.662, loss_interctc_layer15=70.482, loss_interctc_layer21=95.460, loss=85.432, backward_time=0.227, grad_norm=86.024, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.440e-05, train_time=1.269 +[gpua006:0/64] 2024-02-09 04:42:19,825 (trainer:756) INFO: 23epoch:train:7001-7100batch: iter_time=8.272e-05, forward_time=0.155, loss_ctc=73.346, loss_interctc_layer6=73.861, loss_interctc_layer12=61.024, loss_interctc_layer15=55.888, loss_interctc_layer21=75.952, loss=68.014, backward_time=0.209, grad_norm=58.507, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.439e-05, train_time=1.444 +[gpua006:0/64] 2024-02-09 04:44:15,487 (trainer:756) INFO: 23epoch:train:7101-7200batch: iter_time=3.564e-04, forward_time=0.191, loss_ctc=69.469, loss_interctc_layer6=77.610, loss_interctc_layer12=64.624, loss_interctc_layer15=59.522, loss_interctc_layer21=71.689, loss=68.583, backward_time=0.221, grad_norm=54.714, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.438e-05, train_time=1.154 +[gpua006:0/64] 2024-02-09 04:46:07,136 (trainer:756) INFO: 23epoch:train:7201-7300batch: iter_time=7.312e-04, forward_time=0.176, loss_ctc=79.636, loss_interctc_layer6=83.388, loss_interctc_layer12=69.305, loss_interctc_layer15=63.731, loss_interctc_layer21=82.109, loss=75.634, backward_time=0.215, grad_norm=59.064, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.437e-05, train_time=1.118 +[gpua006:0/64] 2024-02-09 04:47:51,139 (trainer:756) INFO: 23epoch:train:7301-7400batch: iter_time=8.598e-05, forward_time=0.148, loss_ctc=73.311, loss_interctc_layer6=84.003, loss_interctc_layer12=69.971, loss_interctc_layer15=64.397, loss_interctc_layer21=75.425, loss=73.421, backward_time=0.210, grad_norm=66.362, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.435e-05, train_time=1.040 +[gpua006:0/64] 2024-02-09 04:49:55,413 (trainer:756) INFO: 23epoch:train:7401-7500batch: iter_time=9.426e-05, forward_time=0.142, loss_ctc=79.407, loss_interctc_layer6=90.633, loss_interctc_layer12=76.118, loss_interctc_layer15=70.382, loss_interctc_layer21=81.880, loss=79.684, backward_time=0.208, grad_norm=64.467, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.434e-05, train_time=1.243 +[gpua006:0/64] 2024-02-09 04:50:10,849 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-09 04:50:29,030 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 04:50:32,669 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 04:50:32,669 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-09 04:50:32,672 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 04:56:28,171 (trainer:756) INFO: 23epoch:train:7501-7600batch: iter_time=2.550, forward_time=0.167, loss_ctc=84.500, loss_interctc_layer6=85.797, loss_interctc_layer12=71.281, loss_interctc_layer15=65.462, loss_interctc_layer21=87.525, loss=78.913, backward_time=0.218, grad_norm=85.013, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.433e-05, train_time=3.927 +[gpua006:0/64] 2024-02-09 04:58:32,327 (trainer:756) INFO: 23epoch:train:7601-7700batch: iter_time=4.275e-04, forward_time=0.146, loss_ctc=86.760, loss_interctc_layer6=94.593, loss_interctc_layer12=79.754, loss_interctc_layer15=74.126, loss_interctc_layer21=89.598, loss=84.966, backward_time=0.212, grad_norm=92.045, clip=100.000, loss_scale=4.097e+31, optim_step_time=0.138, optim0_lr0=8.432e-05, train_time=1.241 +[gpua006:0/64] 2024-02-09 05:00:21,273 (trainer:756) INFO: 23epoch:train:7701-7800batch: iter_time=7.921e-05, forward_time=0.192, loss_ctc=76.072, loss_interctc_layer6=81.391, loss_interctc_layer12=67.825, loss_interctc_layer15=62.405, loss_interctc_layer21=78.413, loss=73.221, backward_time=0.222, grad_norm=61.311, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.139, optim0_lr0=8.430e-05, train_time=1.088 +[gpua006:0/64] 2024-02-09 05:02:35,993 (trainer:756) INFO: 23epoch:train:7801-7900batch: iter_time=8.153e-05, forward_time=0.174, loss_ctc=74.903, loss_interctc_layer6=86.341, loss_interctc_layer12=72.334, loss_interctc_layer15=66.715, loss_interctc_layer21=77.218, loss=75.502, backward_time=0.222, grad_norm=74.702, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.138, optim0_lr0=8.429e-05, train_time=1.349 +[gpua006:0/64] 2024-02-09 05:04:44,882 (trainer:756) INFO: 23epoch:train:7901-8000batch: iter_time=8.243e-05, forward_time=0.149, loss_ctc=75.735, loss_interctc_layer6=80.857, loss_interctc_layer12=67.234, loss_interctc_layer15=61.737, loss_interctc_layer21=78.151, loss=72.743, backward_time=0.211, grad_norm=57.432, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.136, optim0_lr0=8.428e-05, train_time=1.288 +[gpua006:0/64] 2024-02-09 05:07:16,330 (trainer:756) INFO: 23epoch:train:8001-8100batch: iter_time=7.995e-05, forward_time=0.143, loss_ctc=74.956, loss_interctc_layer6=76.969, loss_interctc_layer12=64.390, loss_interctc_layer15=59.336, loss_interctc_layer21=77.382, loss=70.606, backward_time=0.208, grad_norm=65.205, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.136, optim0_lr0=8.427e-05, train_time=1.515 +[gpua006:0/64] 2024-02-09 05:09:26,451 (trainer:756) INFO: 23epoch:train:8101-8200batch: iter_time=7.879e-05, forward_time=0.142, loss_ctc=86.542, loss_interctc_layer6=87.190, loss_interctc_layer12=72.273, loss_interctc_layer15=66.334, loss_interctc_layer21=89.560, loss=80.380, backward_time=0.207, grad_norm=62.325, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.136, optim0_lr0=8.425e-05, train_time=1.301 +[gpua006:0/64] 2024-02-09 05:11:10,014 (trainer:756) INFO: 23epoch:train:8201-8300batch: iter_time=1.800e-04, forward_time=0.150, loss_ctc=91.836, loss_interctc_layer6=83.405, loss_interctc_layer12=69.086, loss_interctc_layer15=63.348, loss_interctc_layer21=95.118, loss=80.559, backward_time=0.216, grad_norm=61.291, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.136, optim0_lr0=8.424e-05, train_time=1.035 +[gpua006:0/64] 2024-02-09 05:13:17,110 (trainer:756) INFO: 23epoch:train:8301-8400batch: iter_time=7.914e-05, forward_time=0.163, loss_ctc=83.847, loss_interctc_layer6=83.132, loss_interctc_layer12=69.244, loss_interctc_layer15=63.713, loss_interctc_layer21=86.621, loss=77.312, backward_time=0.227, grad_norm=63.973, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.138, optim0_lr0=8.423e-05, train_time=1.271 +[gpua006:0/64] 2024-02-09 05:15:05,664 (trainer:756) INFO: 23epoch:train:8401-8500batch: iter_time=3.613e-04, forward_time=0.142, loss_ctc=75.710, loss_interctc_layer6=76.995, loss_interctc_layer12=63.831, loss_interctc_layer15=58.656, loss_interctc_layer21=78.236, loss=70.686, backward_time=0.210, grad_norm=82.231, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.137, optim0_lr0=8.422e-05, train_time=1.085 +[gpua006:0/64] 2024-02-09 05:17:01,248 (trainer:756) INFO: 23epoch:train:8501-8600batch: iter_time=8.312e-05, forward_time=0.143, loss_ctc=75.058, loss_interctc_layer6=82.229, loss_interctc_layer12=68.218, loss_interctc_layer15=62.474, loss_interctc_layer21=77.479, loss=73.092, backward_time=0.209, grad_norm=77.159, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.136, optim0_lr0=8.420e-05, train_time=1.156 +[gpua006:0/64] 2024-02-09 05:19:12,354 (trainer:756) INFO: 23epoch:train:8601-8700batch: iter_time=8.189e-05, forward_time=0.175, loss_ctc=81.766, loss_interctc_layer6=90.758, loss_interctc_layer12=76.456, loss_interctc_layer15=70.660, loss_interctc_layer21=84.150, loss=80.758, backward_time=0.217, grad_norm=88.904, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.139, optim0_lr0=8.419e-05, train_time=1.311 +[gpua006:0/64] 2024-02-09 05:20:36,590 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-09 05:20:54,999 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 05:20:58,368 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 05:20:58,368 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-09 05:20:58,392 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 05:26:17,427 (trainer:756) INFO: 23epoch:train:8701-8800batch: iter_time=2.932, forward_time=0.157, loss_ctc=71.814, loss_interctc_layer6=77.231, loss_interctc_layer12=64.142, loss_interctc_layer15=58.798, loss_interctc_layer21=74.354, loss=69.268, backward_time=0.213, grad_norm=52.672, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.137, optim0_lr0=8.418e-05, train_time=4.251 +[gpua006:0/64] 2024-02-09 05:27:30,945 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 05:27:57,878 (trainer:756) INFO: 23epoch:train:8801-8900batch: iter_time=8.047e-05, forward_time=0.153, loss_ctc=89.775, loss_interctc_layer6=99.692, loss_interctc_layer12=83.613, loss_interctc_layer15=77.166, loss_interctc_layer21=92.700, loss=88.589, backward_time=0.214, grad_norm=78.824, clip=100.000, loss_scale=7.007e+31, optim_step_time=0.137, optim0_lr0=8.417e-05, train_time=1.004 +[gpua006:0/64] 2024-02-09 05:30:06,574 (trainer:756) INFO: 23epoch:train:8901-9000batch: iter_time=8.075e-05, forward_time=0.143, loss_ctc=70.985, loss_interctc_layer6=85.381, loss_interctc_layer12=71.618, loss_interctc_layer15=66.174, loss_interctc_layer21=73.211, loss=73.474, backward_time=0.210, grad_norm=88.047, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.415e-05, train_time=1.286 +[gpua006:0/64] 2024-02-09 05:32:18,793 (trainer:756) INFO: 23epoch:train:9001-9100batch: iter_time=8.581e-05, forward_time=0.144, loss_ctc=71.666, loss_interctc_layer6=83.068, loss_interctc_layer12=69.063, loss_interctc_layer15=63.347, loss_interctc_layer21=73.809, loss=72.191, backward_time=0.210, grad_norm=60.437, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.414e-05, train_time=1.322 +[gpua006:0/64] 2024-02-09 05:34:16,841 (trainer:756) INFO: 23epoch:train:9101-9200batch: iter_time=8.293e-05, forward_time=0.143, loss_ctc=74.667, loss_interctc_layer6=86.166, loss_interctc_layer12=72.122, loss_interctc_layer15=66.638, loss_interctc_layer21=77.063, loss=75.331, backward_time=0.211, grad_norm=64.553, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.413e-05, train_time=1.180 +[gpua006:0/64] 2024-02-09 05:36:10,488 (trainer:756) INFO: 23epoch:train:9201-9300batch: iter_time=5.241e-04, forward_time=0.170, loss_ctc=70.331, loss_interctc_layer6=77.502, loss_interctc_layer12=64.117, loss_interctc_layer15=58.869, loss_interctc_layer21=72.691, loss=68.702, backward_time=0.224, grad_norm=61.575, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.412e-05, train_time=1.136 +[gpua006:0/64] 2024-02-09 05:38:20,172 (trainer:756) INFO: 23epoch:train:9301-9400batch: iter_time=6.062e-04, forward_time=0.162, loss_ctc=76.801, loss_interctc_layer6=78.936, loss_interctc_layer12=65.613, loss_interctc_layer15=60.462, loss_interctc_layer21=79.492, loss=72.261, backward_time=0.219, grad_norm=76.801, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.410e-05, train_time=1.297 +[gpua006:0/64] 2024-02-09 05:41:25,023 (trainer:756) INFO: 23epoch:train:9401-9500batch: iter_time=2.078e-04, forward_time=0.142, loss_ctc=91.336, loss_interctc_layer6=91.910, loss_interctc_layer12=76.239, loss_interctc_layer15=69.965, loss_interctc_layer21=94.625, loss=84.815, backward_time=0.207, grad_norm=67.551, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.409e-05, train_time=1.848 +[gpua006:0/64] 2024-02-09 05:43:54,014 (trainer:756) INFO: 23epoch:train:9501-9600batch: iter_time=8.482e-05, forward_time=0.153, loss_ctc=72.599, loss_interctc_layer6=73.963, loss_interctc_layer12=60.886, loss_interctc_layer15=55.711, loss_interctc_layer21=75.287, loss=67.689, backward_time=0.228, grad_norm=55.595, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.408e-05, train_time=1.490 +[gpua006:0/64] 2024-02-09 05:45:58,538 (trainer:756) INFO: 23epoch:train:9601-9700batch: iter_time=8.657e-05, forward_time=0.162, loss_ctc=69.208, loss_interctc_layer6=76.953, loss_interctc_layer12=64.038, loss_interctc_layer15=58.850, loss_interctc_layer21=71.389, loss=68.088, backward_time=0.211, grad_norm=52.493, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.407e-05, train_time=1.244 +[gpua006:0/64] 2024-02-09 05:48:02,329 (trainer:756) INFO: 23epoch:train:9701-9800batch: iter_time=8.269e-05, forward_time=0.145, loss_ctc=79.679, loss_interctc_layer6=83.275, loss_interctc_layer12=69.082, loss_interctc_layer15=63.528, loss_interctc_layer21=82.293, loss=75.571, backward_time=0.218, grad_norm=84.733, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.405e-05, train_time=1.239 +[gpua006:0/64] 2024-02-09 05:49:48,414 (trainer:756) INFO: 23epoch:train:9801-9900batch: iter_time=2.767e-04, forward_time=0.152, loss_ctc=73.184, loss_interctc_layer6=83.888, loss_interctc_layer12=69.847, loss_interctc_layer15=64.190, loss_interctc_layer21=75.385, loss=73.299, backward_time=0.214, grad_norm=63.852, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.404e-05, train_time=1.061 +[gpua006:0/64] 2024-02-09 05:51:40,318 (trainer:756) INFO: 23epoch:train:9901-10000batch: iter_time=7.648e-05, forward_time=0.142, loss_ctc=79.471, loss_interctc_layer6=90.459, loss_interctc_layer12=76.046, loss_interctc_layer15=70.271, loss_interctc_layer21=81.878, loss=79.625, backward_time=0.210, grad_norm=73.829, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.403e-05, train_time=1.119 +[gpua006:0/64] 2024-02-09 05:52:00,348 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-09 05:52:18,751 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 05:52:22,144 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 05:52:22,144 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-09 05:52:22,181 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 05:58:08,740 (trainer:756) INFO: 23epoch:train:10001-10100batch: iter_time=2.800, forward_time=0.144, loss_ctc=83.679, loss_interctc_layer6=84.975, loss_interctc_layer12=70.621, loss_interctc_layer15=64.757, loss_interctc_layer21=86.501, loss=78.106, backward_time=0.211, grad_norm=63.742, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.402e-05, train_time=3.883 +[gpua006:0/64] 2024-02-09 06:00:03,108 (trainer:756) INFO: 23epoch:train:10101-10200batch: iter_time=7.834e-05, forward_time=0.143, loss_ctc=85.895, loss_interctc_layer6=94.444, loss_interctc_layer12=79.564, loss_interctc_layer15=73.699, loss_interctc_layer21=88.449, loss=84.410, backward_time=0.210, grad_norm=121.299, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.401e-05, train_time=1.144 +[gpua006:0/64] 2024-02-09 06:02:23,094 (trainer:756) INFO: 23epoch:train:10201-10300batch: iter_time=7.895e-05, forward_time=0.175, loss_ctc=75.780, loss_interctc_layer6=81.373, loss_interctc_layer12=67.755, loss_interctc_layer15=62.249, loss_interctc_layer21=78.186, loss=73.069, backward_time=0.243, grad_norm=89.879, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=8.399e-05, train_time=1.400 +[gpua006:0/64] 2024-02-09 06:03:55,249 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 06:04:24,737 (trainer:756) INFO: 23epoch:train:10301-10400batch: iter_time=3.640e-04, forward_time=0.162, loss_ctc=75.025, loss_interctc_layer6=86.160, loss_interctc_layer12=72.106, loss_interctc_layer15=66.452, loss_interctc_layer21=77.134, loss=75.375, backward_time=0.212, grad_norm=77.187, clip=100.000, loss_scale=3.524e+31, optim_step_time=0.138, optim0_lr0=8.398e-05, train_time=1.216 +[gpua006:0/64] 2024-02-09 06:06:31,105 (trainer:756) INFO: 23epoch:train:10401-10500batch: iter_time=8.140e-05, forward_time=0.153, loss_ctc=75.088, loss_interctc_layer6=79.912, loss_interctc_layer12=66.208, loss_interctc_layer15=60.719, loss_interctc_layer21=77.701, loss=71.926, backward_time=0.223, grad_norm=66.424, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.397e-05, train_time=1.263 +[gpua006:0/64] 2024-02-09 06:08:29,317 (trainer:756) INFO: 23epoch:train:10501-10600batch: iter_time=8.373e-05, forward_time=0.151, loss_ctc=74.285, loss_interctc_layer6=75.790, loss_interctc_layer12=63.233, loss_interctc_layer15=58.342, loss_interctc_layer21=76.831, loss=69.696, backward_time=0.216, grad_norm=57.895, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.396e-05, train_time=1.182 +[gpua006:0/64] 2024-02-09 06:10:15,728 (trainer:756) INFO: 23epoch:train:10601-10700batch: iter_time=8.680e-05, forward_time=0.155, loss_ctc=86.271, loss_interctc_layer6=87.256, loss_interctc_layer12=72.193, loss_interctc_layer15=66.143, loss_interctc_layer21=89.342, loss=80.241, backward_time=0.213, grad_norm=71.059, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.394e-05, train_time=1.063 +[gpua006:0/64] 2024-02-09 06:12:27,867 (trainer:756) INFO: 23epoch:train:10701-10800batch: iter_time=8.320e-05, forward_time=0.142, loss_ctc=91.499, loss_interctc_layer6=82.689, loss_interctc_layer12=68.478, loss_interctc_layer15=62.718, loss_interctc_layer21=94.878, loss=80.052, backward_time=0.213, grad_norm=67.497, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.393e-05, train_time=1.322 +[gpua006:0/64] 2024-02-09 06:14:42,188 (trainer:756) INFO: 23epoch:train:10801-10900batch: iter_time=8.262e-05, forward_time=0.147, loss_ctc=82.963, loss_interctc_layer6=83.115, loss_interctc_layer12=69.162, loss_interctc_layer15=63.601, loss_interctc_layer21=85.734, loss=76.915, backward_time=0.212, grad_norm=54.771, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.392e-05, train_time=1.343 +[gpua006:0/64] 2024-02-09 06:16:56,747 (trainer:756) INFO: 23epoch:train:10901-11000batch: iter_time=8.107e-05, forward_time=0.142, loss_ctc=75.794, loss_interctc_layer6=76.791, loss_interctc_layer12=63.621, loss_interctc_layer15=58.468, loss_interctc_layer21=78.240, loss=70.583, backward_time=0.209, grad_norm=71.590, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.391e-05, train_time=1.345 +[gpua006:0/64] 2024-02-09 06:19:12,188 (trainer:756) INFO: 23epoch:train:11001-11100batch: iter_time=7.955e-05, forward_time=0.142, loss_ctc=75.251, loss_interctc_layer6=81.569, loss_interctc_layer12=67.489, loss_interctc_layer15=61.900, loss_interctc_layer21=77.808, loss=72.803, backward_time=0.208, grad_norm=72.954, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.389e-05, train_time=1.354 +[gpua006:0/64] 2024-02-09 06:21:34,490 (trainer:756) INFO: 23epoch:train:11101-11200batch: iter_time=7.956e-05, forward_time=0.143, loss_ctc=82.167, loss_interctc_layer6=90.635, loss_interctc_layer12=76.258, loss_interctc_layer15=70.652, loss_interctc_layer21=84.545, loss=80.852, backward_time=0.210, grad_norm=86.304, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.388e-05, train_time=1.423 +[gpua006:0/64] 2024-02-09 06:22:46,099 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-09 06:23:04,527 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 06:23:07,989 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 06:23:07,989 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-09 06:23:07,993 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 06:32:19,682 (trainer:756) INFO: 23epoch:train:11201-11300batch: iter_time=3.165, forward_time=0.174, loss_ctc=73.957, loss_interctc_layer6=77.354, loss_interctc_layer12=64.173, loss_interctc_layer15=58.845, loss_interctc_layer21=76.382, loss=70.142, backward_time=0.218, grad_norm=68.473, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.387e-05, train_time=6.451 +[gpua006:0/64] 2024-02-09 06:34:01,988 (trainer:756) INFO: 23epoch:train:11301-11400batch: iter_time=7.138e-05, forward_time=0.143, loss_ctc=95.964, loss_interctc_layer6=99.661, loss_interctc_layer12=83.417, loss_interctc_layer15=77.017, loss_interctc_layer21=98.984, loss=91.009, backward_time=0.209, grad_norm=90.610, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.386e-05, train_time=1.023 +[gpua006:0/64] 2024-02-09 06:35:49,654 (trainer:756) INFO: 23epoch:train:11401-11500batch: iter_time=7.823e-05, forward_time=0.142, loss_ctc=73.180, loss_interctc_layer6=85.603, loss_interctc_layer12=71.734, loss_interctc_layer15=66.289, loss_interctc_layer21=75.334, loss=74.428, backward_time=0.209, grad_norm=85.706, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.385e-05, train_time=1.076 +[gpua006:0/64] 2024-02-09 06:38:03,217 (trainer:756) INFO: 23epoch:train:11501-11600batch: iter_time=8.023e-05, forward_time=0.145, loss_ctc=76.864, loss_interctc_layer6=82.956, loss_interctc_layer12=68.986, loss_interctc_layer15=63.398, loss_interctc_layer21=79.137, loss=74.268, backward_time=0.209, grad_norm=60.498, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.383e-05, train_time=1.335 +[gpua006:0/64] 2024-02-09 06:40:12,666 (trainer:756) INFO: 23epoch:train:11601-11700batch: iter_time=8.053e-05, forward_time=0.141, loss_ctc=78.754, loss_interctc_layer6=85.670, loss_interctc_layer12=71.703, loss_interctc_layer15=66.039, loss_interctc_layer21=81.232, loss=76.680, backward_time=0.208, grad_norm=63.184, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.382e-05, train_time=1.294 +[gpua006:0/64] 2024-02-09 06:42:26,756 (trainer:756) INFO: 23epoch:train:11701-11800batch: iter_time=7.996e-05, forward_time=0.140, loss_ctc=71.673, loss_interctc_layer6=77.831, loss_interctc_layer12=64.484, loss_interctc_layer15=59.125, loss_interctc_layer21=74.068, loss=69.436, backward_time=0.207, grad_norm=64.395, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.381e-05, train_time=1.341 +[gpua006:0/64] 2024-02-09 06:44:44,727 (trainer:756) INFO: 23epoch:train:11801-11900batch: iter_time=7.916e-05, forward_time=0.141, loss_ctc=82.062, loss_interctc_layer6=79.129, loss_interctc_layer12=65.756, loss_interctc_layer15=60.466, loss_interctc_layer21=85.029, loss=74.488, backward_time=0.207, grad_norm=68.592, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.380e-05, train_time=1.379 +[gpua006:0/64] 2024-02-09 06:46:40,516 (trainer:756) INFO: 23epoch:train:11901-12000batch: iter_time=7.996e-05, forward_time=0.141, loss_ctc=100.864, loss_interctc_layer6=91.541, loss_interctc_layer12=75.860, loss_interctc_layer15=69.736, loss_interctc_layer21=104.359, loss=88.472, backward_time=0.207, grad_norm=71.437, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.378e-05, train_time=1.157 +[gpua006:0/64] 2024-02-09 06:49:05,638 (trainer:756) INFO: 23epoch:train:12001-12100batch: iter_time=8.287e-05, forward_time=0.235, loss_ctc=75.973, loss_interctc_layer6=74.035, loss_interctc_layer12=61.145, loss_interctc_layer15=56.023, loss_interctc_layer21=78.743, loss=69.184, backward_time=0.240, grad_norm=61.951, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.377e-05, train_time=1.451 +[gpua006:0/64] 2024-02-09 06:51:04,183 (trainer:756) INFO: 23epoch:train:12101-12200batch: iter_time=8.166e-05, forward_time=0.141, loss_ctc=68.816, loss_interctc_layer6=76.640, loss_interctc_layer12=63.750, loss_interctc_layer15=58.737, loss_interctc_layer21=71.049, loss=67.799, backward_time=0.208, grad_norm=123.639, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.376e-05, train_time=1.186 +[gpua006:0/64] 2024-02-09 06:53:26,757 (trainer:756) INFO: 23epoch:train:12201-12300batch: iter_time=8.187e-05, forward_time=0.142, loss_ctc=84.946, loss_interctc_layer6=83.700, loss_interctc_layer12=69.441, loss_interctc_layer15=63.814, loss_interctc_layer21=87.839, loss=77.948, backward_time=0.207, grad_norm=88.154, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.375e-05, train_time=1.426 +[gpua006:0/64] 2024-02-09 06:55:41,801 (trainer:756) INFO: 23epoch:train:12301-12400batch: iter_time=7.809e-05, forward_time=0.142, loss_ctc=75.290, loss_interctc_layer6=83.733, loss_interctc_layer12=69.612, loss_interctc_layer15=63.981, loss_interctc_layer21=77.617, loss=74.046, backward_time=0.210, grad_norm=78.915, clip=100.000, loss_scale=2.556e+31, optim_step_time=0.137, optim0_lr0=8.374e-05, train_time=1.350 +[gpua006:0/64] 2024-02-09 06:57:59,621 (trainer:756) INFO: 23epoch:train:12401-12500batch: iter_time=7.571e-05, forward_time=0.142, loss_ctc=81.253, loss_interctc_layer6=90.640, loss_interctc_layer12=75.873, loss_interctc_layer15=69.999, loss_interctc_layer21=83.668, loss=80.287, backward_time=0.207, grad_norm=73.023, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.372e-05, train_time=1.378 +[gpua006:0/64] 2024-02-09 06:58:19,650 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-09 06:58:38,560 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 06:58:41,969 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 06:58:41,969 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-09 06:58:41,998 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 07:04:43,853 (trainer:756) INFO: 23epoch:train:12501-12600batch: iter_time=2.944, forward_time=0.179, loss_ctc=78.393, loss_interctc_layer6=85.230, loss_interctc_layer12=70.716, loss_interctc_layer15=64.815, loss_interctc_layer21=81.313, loss=76.093, backward_time=0.218, grad_norm=76.804, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.371e-05, train_time=4.042 +[gpua006:0/64] 2024-02-09 07:04:45,667 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 07:06:30,773 (trainer:756) INFO: 23epoch:train:12601-12700batch: iter_time=8.060e-05, forward_time=0.143, loss_ctc=81.714, loss_interctc_layer6=94.575, loss_interctc_layer12=79.591, loss_interctc_layer15=73.699, loss_interctc_layer21=84.103, loss=82.737, backward_time=0.210, grad_norm=82.280, clip=100.000, loss_scale=2.049e+31, optim_step_time=0.137, optim0_lr0=8.370e-05, train_time=1.069 +[gpua006:0/64] 2024-02-09 07:09:26,221 (trainer:756) INFO: 23epoch:train:12701-12800batch: iter_time=8.431e-05, forward_time=0.142, loss_ctc=70.341, loss_interctc_layer6=80.680, loss_interctc_layer12=67.194, loss_interctc_layer15=61.810, loss_interctc_layer21=72.381, loss=70.481, backward_time=0.208, grad_norm=57.867, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.369e-05, train_time=1.754 +[gpua006:0/64] 2024-02-09 07:11:25,497 (trainer:756) INFO: 23epoch:train:12801-12900batch: iter_time=8.083e-05, forward_time=0.147, loss_ctc=71.296, loss_interctc_layer6=85.952, loss_interctc_layer12=71.829, loss_interctc_layer15=66.218, loss_interctc_layer21=73.533, loss=73.766, backward_time=0.213, grad_norm=67.220, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.367e-05, train_time=1.193 +[gpua006:0/64] 2024-02-09 07:13:07,078 (trainer:756) INFO: 23epoch:train:12901-13000batch: iter_time=7.185e-05, forward_time=0.144, loss_ctc=69.254, loss_interctc_layer6=79.866, loss_interctc_layer12=66.207, loss_interctc_layer15=60.726, loss_interctc_layer21=71.417, loss=69.494, backward_time=0.211, grad_norm=50.153, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.366e-05, train_time=1.016 +[gpua006:0/64] 2024-02-09 07:15:55,285 (trainer:756) INFO: 23epoch:train:13001-13100batch: iter_time=8.392e-05, forward_time=0.141, loss_ctc=71.248, loss_interctc_layer6=75.475, loss_interctc_layer12=62.850, loss_interctc_layer15=57.841, loss_interctc_layer21=73.344, loss=68.152, backward_time=0.208, grad_norm=55.590, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.365e-05, train_time=1.682 +[gpua006:0/64] 2024-02-09 07:18:04,834 (trainer:756) INFO: 23epoch:train:13101-13200batch: iter_time=8.371e-05, forward_time=0.234, loss_ctc=82.896, loss_interctc_layer6=86.240, loss_interctc_layer12=71.159, loss_interctc_layer15=65.207, loss_interctc_layer21=85.944, loss=78.289, backward_time=0.254, grad_norm=69.602, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.364e-05, train_time=1.295 +[gpua006:0/64] 2024-02-09 07:20:10,955 (trainer:756) INFO: 23epoch:train:13201-13300batch: iter_time=8.418e-05, forward_time=0.142, loss_ctc=81.650, loss_interctc_layer6=82.328, loss_interctc_layer12=68.132, loss_interctc_layer15=62.329, loss_interctc_layer21=84.614, loss=75.811, backward_time=0.209, grad_norm=61.901, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.363e-05, train_time=1.261 +[gpua006:0/64] 2024-02-09 07:22:15,559 (trainer:756) INFO: 23epoch:train:13301-13400batch: iter_time=8.175e-05, forward_time=0.142, loss_ctc=80.597, loss_interctc_layer6=83.242, loss_interctc_layer12=69.224, loss_interctc_layer15=63.600, loss_interctc_layer21=83.243, loss=75.981, backward_time=0.209, grad_norm=81.952, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.361e-05, train_time=1.246 +[gpua006:0/64] 2024-02-09 07:24:43,975 (trainer:756) INFO: 23epoch:train:13401-13500batch: iter_time=8.015e-05, forward_time=0.145, loss_ctc=72.806, loss_interctc_layer6=76.491, loss_interctc_layer12=63.418, loss_interctc_layer15=58.082, loss_interctc_layer21=75.109, loss=69.181, backward_time=0.215, grad_norm=68.768, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.360e-05, train_time=1.484 +[gpua006:0/64] 2024-02-09 07:26:36,916 (trainer:756) INFO: 23epoch:train:13501-13600batch: iter_time=8.428e-05, forward_time=0.142, loss_ctc=72.070, loss_interctc_layer6=81.688, loss_interctc_layer12=67.449, loss_interctc_layer15=61.824, loss_interctc_layer21=74.446, loss=71.496, backward_time=0.210, grad_norm=66.267, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.359e-05, train_time=1.129 +[gpua006:0/64] 2024-02-09 07:29:09,206 (trainer:756) INFO: 23epoch:train:13601-13700batch: iter_time=8.189e-05, forward_time=0.144, loss_ctc=79.869, loss_interctc_layer6=90.295, loss_interctc_layer12=75.952, loss_interctc_layer15=70.292, loss_interctc_layer21=82.101, loss=79.702, backward_time=0.208, grad_norm=86.774, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.358e-05, train_time=1.523 +[gpua006:0/64] 2024-02-09 07:30:50,243 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-09 07:31:08,844 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 07:31:12,243 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 07:31:12,243 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-09 07:31:12,269 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 07:36:43,432 (trainer:756) INFO: 23epoch:train:13701-13800batch: iter_time=3.085, forward_time=0.142, loss_ctc=68.879, loss_interctc_layer6=76.287, loss_interctc_layer12=63.196, loss_interctc_layer15=57.909, loss_interctc_layer21=71.365, loss=67.527, backward_time=0.209, grad_norm=48.789, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.356e-05, train_time=4.542 +[gpua006:0/64] 2024-02-09 07:38:52,956 (trainer:756) INFO: 23epoch:train:13801-13900batch: iter_time=8.338e-05, forward_time=0.227, loss_ctc=88.895, loss_interctc_layer6=99.707, loss_interctc_layer12=83.540, loss_interctc_layer15=77.035, loss_interctc_layer21=91.839, loss=88.203, backward_time=0.228, grad_norm=70.596, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.176, optim0_lr0=8.355e-05, train_time=1.294 +[gpua006:0/64] 2024-02-09 07:40:39,477 (trainer:756) INFO: 23epoch:train:13901-14000batch: iter_time=7.848e-05, forward_time=0.142, loss_ctc=70.967, loss_interctc_layer6=85.182, loss_interctc_layer12=71.451, loss_interctc_layer15=65.869, loss_interctc_layer21=73.199, loss=73.334, backward_time=0.211, grad_norm=105.555, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.354e-05, train_time=1.065 +[gpua006:0/64] 2024-02-09 07:43:12,095 (trainer:756) INFO: 23epoch:train:14001-14100batch: iter_time=8.147e-05, forward_time=0.141, loss_ctc=71.240, loss_interctc_layer6=82.254, loss_interctc_layer12=68.252, loss_interctc_layer15=62.628, loss_interctc_layer21=73.359, loss=71.546, backward_time=0.210, grad_norm=67.031, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.353e-05, train_time=1.526 +[gpua006:0/64] 2024-02-09 07:45:29,167 (trainer:756) INFO: 23epoch:train:14101-14200batch: iter_time=8.180e-05, forward_time=0.142, loss_ctc=73.532, loss_interctc_layer6=85.332, loss_interctc_layer12=71.476, loss_interctc_layer15=65.900, loss_interctc_layer21=75.906, loss=74.429, backward_time=0.209, grad_norm=62.877, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.352e-05, train_time=1.371 +[gpua006:0/64] 2024-02-09 07:47:18,823 (trainer:756) INFO: 23epoch:train:14201-14300batch: iter_time=7.876e-05, forward_time=0.145, loss_ctc=69.257, loss_interctc_layer6=77.250, loss_interctc_layer12=64.004, loss_interctc_layer15=58.643, loss_interctc_layer21=71.465, loss=68.124, backward_time=0.213, grad_norm=61.456, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.350e-05, train_time=1.096 +[gpua006:0/64] 2024-02-09 07:49:37,905 (trainer:756) INFO: 23epoch:train:14301-14400batch: iter_time=8.097e-05, forward_time=0.141, loss_ctc=77.138, loss_interctc_layer6=78.801, loss_interctc_layer12=65.374, loss_interctc_layer15=60.136, loss_interctc_layer21=79.616, loss=72.213, backward_time=0.209, grad_norm=69.948, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.349e-05, train_time=1.391 +[gpua006:0/64] 2024-02-09 07:52:26,155 (trainer:756) INFO: 23epoch:train:14401-14500batch: iter_time=8.201e-05, forward_time=0.142, loss_ctc=91.146, loss_interctc_layer6=92.262, loss_interctc_layer12=76.324, loss_interctc_layer15=69.989, loss_interctc_layer21=94.460, loss=84.836, backward_time=0.209, grad_norm=82.352, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.348e-05, train_time=1.682 +[gpua006:0/64] 2024-02-09 07:54:39,548 (trainer:756) INFO: 23epoch:train:14501-14600batch: iter_time=8.178e-05, forward_time=0.141, loss_ctc=72.025, loss_interctc_layer6=73.497, loss_interctc_layer12=60.597, loss_interctc_layer15=55.425, loss_interctc_layer21=74.761, loss=67.261, backward_time=0.209, grad_norm=56.191, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.347e-05, train_time=1.334 +[gpua006:0/64] 2024-02-09 07:56:58,934 (trainer:756) INFO: 23epoch:train:14601-14700batch: iter_time=8.190e-05, forward_time=0.141, loss_ctc=68.468, loss_interctc_layer6=76.637, loss_interctc_layer12=63.534, loss_interctc_layer15=58.358, loss_interctc_layer21=70.720, loss=67.544, backward_time=0.209, grad_norm=57.552, clip=100.000, loss_scale=4.016e+31, optim_step_time=0.137, optim0_lr0=8.346e-05, train_time=1.394 +[gpua006:0/64] 2024-02-09 07:57:00,950 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 07:59:12,909 (trainer:756) INFO: 23epoch:train:14701-14800batch: iter_time=8.059e-05, forward_time=0.157, loss_ctc=78.599, loss_interctc_layer6=82.934, loss_interctc_layer12=68.810, loss_interctc_layer15=63.116, loss_interctc_layer21=81.230, loss=74.938, backward_time=0.222, grad_norm=58.056, clip=100.000, loss_scale=2.049e+31, optim_step_time=0.139, optim0_lr0=8.344e-05, train_time=1.340 +[gpua006:0/64] 2024-02-09 08:01:29,590 (trainer:756) INFO: 23epoch:train:14801-14900batch: iter_time=8.115e-05, forward_time=0.184, loss_ctc=73.051, loss_interctc_layer6=83.127, loss_interctc_layer12=69.148, loss_interctc_layer15=63.420, loss_interctc_layer21=75.309, loss=72.811, backward_time=0.240, grad_norm=66.762, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.343e-05, train_time=1.366 +[gpua006:0/64] 2024-02-09 08:03:42,537 (trainer:756) INFO: 23epoch:train:14901-15000batch: iter_time=7.980e-05, forward_time=0.144, loss_ctc=78.099, loss_interctc_layer6=90.099, loss_interctc_layer12=75.505, loss_interctc_layer15=69.600, loss_interctc_layer21=80.447, loss=78.750, backward_time=0.208, grad_norm=66.571, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.342e-05, train_time=1.329 +[gpua006:0/64] 2024-02-09 08:34:15,357 (trainer:355) INFO: 23epoch results: [train] iter_time=0.243, forward_time=0.194, loss_ctc=78.606, loss_interctc_layer6=84.220, loss_interctc_layer12=70.244, loss_interctc_layer15=64.678, loss_interctc_layer21=81.146, loss=75.779, backward_time=0.232, grad_norm=70.602, clip=100.000, loss_scale=3.383e+31, optim_step_time=0.140, optim0_lr0=8.434e-05, train_time=1.610, time=6 hours, 42 minutes and 49.84 seconds, total_count=345000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=46.277, cer_ctc=0.208, loss_interctc_layer6=51.226, cer_interctc_layer6=0.226, loss_interctc_layer12=38.343, cer_interctc_layer12=0.159, loss_interctc_layer15=34.013, cer_interctc_layer15=0.133, loss_interctc_layer21=48.694, cer_interctc_layer21=0.221, loss=43.711, time=30 minutes and 8.87 seconds, total_count=107433, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-09 08:34:34,080 (trainer:410) INFO: The best model has been updated: valid.cer_ctc, valid.loss_ctc, valid.total_count +[gpua006:0/64] 2024-02-09 08:34:34,087 (trainer:289) INFO: 24/45epoch started. Estimated time to finish: 1 week, 1 day and 7 hours +[gpua006:0/64] 2024-02-09 08:34:34,102 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-09 08:34:51,629 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 08:34:55,057 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 08:34:55,057 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-09 08:34:55,060 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 08:41:09,542 (trainer:756) INFO: 24epoch:train:1-100batch: iter_time=2.769, forward_time=0.183, loss_ctc=89.141, loss_interctc_layer6=86.145, loss_interctc_layer12=71.821, loss_interctc_layer15=66.116, loss_interctc_layer21=92.373, loss=81.119, backward_time=0.219, grad_norm=74.259, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.341e-05, train_time=3.954 +[gpua006:0/64] 2024-02-09 08:43:12,869 (trainer:756) INFO: 24epoch:train:101-200batch: iter_time=9.749e-05, forward_time=0.262, loss_ctc=96.626, loss_interctc_layer6=86.412, loss_interctc_layer12=71.782, loss_interctc_layer15=66.029, loss_interctc_layer21=100.057, loss=84.181, backward_time=0.240, grad_norm=69.208, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.155, optim0_lr0=8.339e-05, train_time=1.232 +[gpua006:0/64] 2024-02-09 08:45:05,270 (trainer:756) INFO: 24epoch:train:201-300batch: iter_time=2.043e-04, forward_time=0.145, loss_ctc=69.275, loss_interctc_layer6=76.647, loss_interctc_layer12=63.714, loss_interctc_layer15=58.754, loss_interctc_layer21=71.327, loss=67.943, backward_time=0.215, grad_norm=59.821, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=8.338e-05, train_time=1.125 +[gpua006:0/64] 2024-02-09 08:47:42,696 (trainer:756) INFO: 24epoch:train:301-400batch: iter_time=9.315e-05, forward_time=0.276, loss_ctc=76.903, loss_interctc_layer6=88.739, loss_interctc_layer12=74.519, loss_interctc_layer15=68.904, loss_interctc_layer21=79.024, loss=77.618, backward_time=0.258, grad_norm=85.842, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=8.337e-05, train_time=1.574 +[gpua006:0/64] 2024-02-09 08:49:48,265 (trainer:756) INFO: 24epoch:train:401-500batch: iter_time=5.137e-04, forward_time=0.146, loss_ctc=90.856, loss_interctc_layer6=87.566, loss_interctc_layer12=73.244, loss_interctc_layer15=67.432, loss_interctc_layer21=93.736, loss=82.567, backward_time=0.208, grad_norm=94.357, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.336e-05, train_time=1.255 +[gpua006:0/64] 2024-02-09 08:52:14,313 (trainer:756) INFO: 24epoch:train:501-600batch: iter_time=9.679e-05, forward_time=0.217, loss_ctc=75.613, loss_interctc_layer6=77.850, loss_interctc_layer12=64.589, loss_interctc_layer15=59.294, loss_interctc_layer21=78.205, loss=71.110, backward_time=0.266, grad_norm=60.633, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.335e-05, train_time=1.461 +[gpua006:0/64] 2024-02-09 08:54:44,120 (trainer:756) INFO: 24epoch:train:601-700batch: iter_time=2.902e-04, forward_time=0.157, loss_ctc=73.464, loss_interctc_layer6=81.933, loss_interctc_layer12=68.208, loss_interctc_layer15=62.699, loss_interctc_layer21=75.821, loss=72.425, backward_time=0.213, grad_norm=61.039, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.333e-05, train_time=1.497 +[gpua006:0/64] 2024-02-09 08:56:51,865 (trainer:756) INFO: 24epoch:train:701-800batch: iter_time=9.302e-05, forward_time=0.225, loss_ctc=89.013, loss_interctc_layer6=87.433, loss_interctc_layer12=73.260, loss_interctc_layer15=67.492, loss_interctc_layer21=91.850, loss=81.810, backward_time=0.233, grad_norm=64.543, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=8.332e-05, train_time=1.278 +[gpua006:0/64] 2024-02-09 08:59:12,532 (trainer:756) INFO: 24epoch:train:801-900batch: iter_time=4.279e-04, forward_time=0.156, loss_ctc=77.283, loss_interctc_layer6=88.034, loss_interctc_layer12=74.343, loss_interctc_layer15=68.992, loss_interctc_layer21=79.299, loss=77.590, backward_time=0.215, grad_norm=76.609, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=8.331e-05, train_time=1.406 +[gpua006:0/64] 2024-02-09 09:01:39,582 (trainer:756) INFO: 24epoch:train:901-1000batch: iter_time=8.823e-05, forward_time=0.143, loss_ctc=77.292, loss_interctc_layer6=89.502, loss_interctc_layer12=74.520, loss_interctc_layer15=68.582, loss_interctc_layer21=79.762, loss=77.932, backward_time=0.205, grad_norm=73.239, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.330e-05, train_time=1.471 +[gpua006:0/64] 2024-02-09 09:04:15,226 (trainer:756) INFO: 24epoch:train:1001-1100batch: iter_time=9.132e-05, forward_time=0.240, loss_ctc=95.277, loss_interctc_layer6=96.694, loss_interctc_layer12=80.590, loss_interctc_layer15=74.384, loss_interctc_layer21=98.179, loss=89.025, backward_time=0.236, grad_norm=75.683, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=8.329e-05, train_time=1.556 +[gpua006:0/64] 2024-02-09 09:06:18,364 (trainer:756) INFO: 24epoch:train:1101-1200batch: iter_time=8.512e-04, forward_time=0.165, loss_ctc=74.983, loss_interctc_layer6=85.613, loss_interctc_layer12=72.018, loss_interctc_layer15=66.613, loss_interctc_layer21=77.226, loss=75.291, backward_time=0.214, grad_norm=70.209, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.146, optim0_lr0=8.327e-05, train_time=1.231 +[gpua006:0/64] 2024-02-09 09:07:40,245 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-09 09:07:58,743 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 09:08:02,143 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 09:08:02,143 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-09 09:08:02,219 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 09:13:19,352 (trainer:756) INFO: 24epoch:train:1201-1300batch: iter_time=2.953, forward_time=0.145, loss_ctc=90.047, loss_interctc_layer6=91.178, loss_interctc_layer12=76.116, loss_interctc_layer15=69.961, loss_interctc_layer21=93.086, loss=84.078, backward_time=0.208, grad_norm=75.458, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.326e-05, train_time=4.210 +[gpua006:0/64] 2024-02-09 09:14:58,714 (trainer:756) INFO: 24epoch:train:1301-1400batch: iter_time=8.650e-05, forward_time=0.145, loss_ctc=92.988, loss_interctc_layer6=83.571, loss_interctc_layer12=69.355, loss_interctc_layer15=63.706, loss_interctc_layer21=96.249, loss=81.174, backward_time=0.211, grad_norm=62.964, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.325e-05, train_time=0.994 +[gpua006:0/64] 2024-02-09 09:17:16,595 (trainer:756) INFO: 24epoch:train:1401-1500batch: iter_time=9.068e-05, forward_time=0.144, loss_ctc=77.885, loss_interctc_layer6=81.892, loss_interctc_layer12=68.148, loss_interctc_layer15=62.909, loss_interctc_layer21=80.232, loss=74.213, backward_time=0.209, grad_norm=68.430, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.324e-05, train_time=1.379 +[gpua006:0/64] 2024-02-09 09:19:00,589 (trainer:756) INFO: 24epoch:train:1501-1600batch: iter_time=9.069e-05, forward_time=0.149, loss_ctc=76.579, loss_interctc_layer6=86.591, loss_interctc_layer12=72.268, loss_interctc_layer15=66.462, loss_interctc_layer21=78.838, loss=76.147, backward_time=0.211, grad_norm=63.009, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.323e-05, train_time=1.040 +[gpua006:0/64] 2024-02-09 09:21:26,325 (trainer:756) INFO: 24epoch:train:1601-1700batch: iter_time=8.582e-05, forward_time=0.215, loss_ctc=75.433, loss_interctc_layer6=78.733, loss_interctc_layer12=65.671, loss_interctc_layer15=60.452, loss_interctc_layer21=77.781, loss=71.614, backward_time=0.261, grad_norm=76.375, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.321e-05, train_time=1.457 +[gpua006:0/64] 2024-02-09 09:23:24,661 (trainer:756) INFO: 24epoch:train:1701-1800batch: iter_time=3.663e-04, forward_time=0.156, loss_ctc=84.442, loss_interctc_layer6=84.066, loss_interctc_layer12=69.658, loss_interctc_layer15=63.948, loss_interctc_layer21=87.448, loss=77.912, backward_time=0.235, grad_norm=73.107, clip=100.000, loss_scale=4.016e+31, optim_step_time=0.142, optim0_lr0=8.320e-05, train_time=1.183 +[gpua006:0/64] 2024-02-09 09:25:28,908 (trainer:756) INFO: 24epoch:train:1801-1900batch: iter_time=9.074e-05, forward_time=0.144, loss_ctc=67.826, loss_interctc_layer6=77.988, loss_interctc_layer12=64.615, loss_interctc_layer15=59.318, loss_interctc_layer21=70.043, loss=67.958, backward_time=0.209, grad_norm=79.275, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.319e-05, train_time=1.242 +[gpua006:0/64] 2024-02-09 09:27:46,221 (trainer:756) INFO: 24epoch:train:1901-2000batch: iter_time=9.397e-05, forward_time=0.144, loss_ctc=84.896, loss_interctc_layer6=85.994, loss_interctc_layer12=71.453, loss_interctc_layer15=65.842, loss_interctc_layer21=87.410, loss=79.119, backward_time=0.208, grad_norm=58.967, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.318e-05, train_time=1.374 +[gpua006:0/64] 2024-02-09 09:29:51,140 (trainer:756) INFO: 24epoch:train:2001-2100batch: iter_time=9.666e-05, forward_time=0.145, loss_ctc=84.041, loss_interctc_layer6=86.269, loss_interctc_layer12=72.284, loss_interctc_layer15=66.679, loss_interctc_layer21=86.821, loss=79.219, backward_time=0.210, grad_norm=63.406, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.317e-05, train_time=1.248 +[gpua006:0/64] 2024-02-09 09:31:54,297 (trainer:756) INFO: 24epoch:train:2101-2200batch: iter_time=9.977e-05, forward_time=0.145, loss_ctc=79.808, loss_interctc_layer6=93.214, loss_interctc_layer12=78.239, loss_interctc_layer15=72.296, loss_interctc_layer21=82.014, loss=81.114, backward_time=0.209, grad_norm=90.840, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.315e-05, train_time=1.233 +[gpua006:0/64] 2024-02-09 09:34:13,590 (trainer:756) INFO: 24epoch:train:2201-2300batch: iter_time=8.897e-05, forward_time=0.143, loss_ctc=86.455, loss_interctc_layer6=87.819, loss_interctc_layer12=72.838, loss_interctc_layer15=66.817, loss_interctc_layer21=89.045, loss=80.595, backward_time=0.209, grad_norm=67.286, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.314e-05, train_time=1.393 +[gpua006:0/64] 2024-02-09 09:36:49,291 (trainer:756) INFO: 24epoch:train:2301-2400batch: iter_time=8.283e-05, forward_time=0.145, loss_ctc=81.096, loss_interctc_layer6=90.110, loss_interctc_layer12=75.434, loss_interctc_layer15=69.817, loss_interctc_layer21=83.541, loss=79.999, backward_time=0.208, grad_norm=76.675, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.313e-05, train_time=1.557 +[gpua006:0/64] 2024-02-09 09:38:47,472 (trainer:756) INFO: 24epoch:train:2401-2500batch: iter_time=6.923e-05, forward_time=0.146, loss_ctc=78.951, loss_interctc_layer6=87.669, loss_interctc_layer12=73.480, loss_interctc_layer15=67.717, loss_interctc_layer21=81.442, loss=77.852, backward_time=0.209, grad_norm=71.186, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.312e-05, train_time=1.182 +[gpua006:0/64] 2024-02-09 09:39:07,503 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-09 09:39:25,988 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 09:39:29,396 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 09:39:29,396 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-09 09:39:29,513 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 09:45:52,991 (trainer:756) INFO: 24epoch:train:2501-2600batch: iter_time=2.872, forward_time=0.176, loss_ctc=82.623, loss_interctc_layer6=85.320, loss_interctc_layer12=70.832, loss_interctc_layer15=65.085, loss_interctc_layer21=85.603, loss=77.893, backward_time=0.229, grad_norm=76.103, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.311e-05, train_time=4.255 +[gpua006:0/64] 2024-02-09 09:47:35,956 (trainer:756) INFO: 24epoch:train:2601-2700batch: iter_time=7.811e-05, forward_time=0.144, loss_ctc=87.267, loss_interctc_layer6=86.798, loss_interctc_layer12=72.005, loss_interctc_layer15=66.213, loss_interctc_layer21=90.240, loss=80.505, backward_time=0.209, grad_norm=61.483, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.309e-05, train_time=1.030 +[gpua006:0/64] 2024-02-09 09:49:42,628 (trainer:756) INFO: 24epoch:train:2701-2800batch: iter_time=7.809e-05, forward_time=0.142, loss_ctc=65.943, loss_interctc_layer6=75.551, loss_interctc_layer12=62.613, loss_interctc_layer15=57.557, loss_interctc_layer21=67.954, loss=65.924, backward_time=0.209, grad_norm=75.640, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.308e-05, train_time=1.267 +[gpua006:0/64] 2024-02-09 09:51:50,189 (trainer:756) INFO: 24epoch:train:2801-2900batch: iter_time=8.222e-05, forward_time=0.143, loss_ctc=75.120, loss_interctc_layer6=88.222, loss_interctc_layer12=73.841, loss_interctc_layer15=67.966, loss_interctc_layer21=77.376, loss=76.505, backward_time=0.208, grad_norm=65.641, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.307e-05, train_time=1.275 +[gpua006:0/64] 2024-02-09 09:53:31,512 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 09:53:59,307 (trainer:756) INFO: 24epoch:train:2901-3000batch: iter_time=8.463e-05, forward_time=0.141, loss_ctc=80.127, loss_interctc_layer6=85.901, loss_interctc_layer12=71.439, loss_interctc_layer15=65.565, loss_interctc_layer21=82.770, loss=77.160, backward_time=0.207, grad_norm=66.552, clip=100.000, loss_scale=3.503e+31, optim_step_time=0.136, optim0_lr0=8.306e-05, train_time=1.292 +[gpua006:0/64] 2024-02-09 09:55:50,120 (trainer:756) INFO: 24epoch:train:3001-3100batch: iter_time=8.548e-05, forward_time=0.141, loss_ctc=70.162, loss_interctc_layer6=76.804, loss_interctc_layer12=63.627, loss_interctc_layer15=58.262, loss_interctc_layer21=72.835, loss=68.338, backward_time=0.208, grad_norm=60.845, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.305e-05, train_time=1.108 +[gpua006:0/64] 2024-02-09 09:57:54,338 (trainer:756) INFO: 24epoch:train:3101-3200batch: iter_time=8.567e-05, forward_time=0.142, loss_ctc=72.796, loss_interctc_layer6=81.658, loss_interctc_layer12=67.780, loss_interctc_layer15=62.226, loss_interctc_layer21=75.079, loss=71.908, backward_time=0.207, grad_norm=65.881, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.303e-05, train_time=1.242 +[gpua006:0/64] 2024-02-09 10:00:32,816 (trainer:756) INFO: 24epoch:train:3201-3300batch: iter_time=8.136e-05, forward_time=0.143, loss_ctc=85.622, loss_interctc_layer6=86.193, loss_interctc_layer12=72.028, loss_interctc_layer15=66.314, loss_interctc_layer21=88.459, loss=79.723, backward_time=0.207, grad_norm=58.052, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.302e-05, train_time=1.585 +[gpua006:0/64] 2024-02-09 10:02:45,443 (trainer:756) INFO: 24epoch:train:3301-3400batch: iter_time=7.939e-05, forward_time=0.146, loss_ctc=73.990, loss_interctc_layer6=86.658, loss_interctc_layer12=72.927, loss_interctc_layer15=67.447, loss_interctc_layer21=76.215, loss=75.447, backward_time=0.206, grad_norm=73.485, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.301e-05, train_time=1.326 +[gpua006:0/64] 2024-02-09 10:05:03,488 (trainer:756) INFO: 24epoch:train:3401-3500batch: iter_time=8.492e-05, forward_time=0.194, loss_ctc=73.068, loss_interctc_layer6=88.402, loss_interctc_layer12=73.577, loss_interctc_layer15=67.608, loss_interctc_layer21=75.266, loss=75.584, backward_time=0.280, grad_norm=68.807, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=8.300e-05, train_time=1.380 +[gpua006:0/64] 2024-02-09 10:07:07,332 (trainer:756) INFO: 24epoch:train:3501-3600batch: iter_time=8.137e-05, forward_time=0.142, loss_ctc=86.851, loss_interctc_layer6=95.022, loss_interctc_layer12=78.941, loss_interctc_layer15=72.650, loss_interctc_layer21=89.551, loss=84.603, backward_time=0.209, grad_norm=98.491, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.299e-05, train_time=1.239 +[gpua006:0/64] 2024-02-09 10:09:04,197 (trainer:756) INFO: 24epoch:train:3601-3700batch: iter_time=8.241e-05, forward_time=0.142, loss_ctc=72.672, loss_interctc_layer6=84.790, loss_interctc_layer12=71.439, loss_interctc_layer15=65.838, loss_interctc_layer21=74.472, loss=73.842, backward_time=0.208, grad_norm=63.891, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.298e-05, train_time=1.168 +[gpua006:0/64] 2024-02-09 10:10:22,885 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-09 10:10:41,543 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 10:10:45,216 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 10:10:45,216 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-09 10:10:45,219 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 10:15:46,107 (trainer:756) INFO: 24epoch:train:3701-3800batch: iter_time=2.774, forward_time=0.143, loss_ctc=84.786, loss_interctc_layer6=91.363, loss_interctc_layer12=76.225, loss_interctc_layer15=70.096, loss_interctc_layer21=87.663, loss=82.027, backward_time=0.209, grad_norm=70.660, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.296e-05, train_time=4.018 +[gpua006:0/64] 2024-02-09 10:18:03,701 (trainer:756) INFO: 24epoch:train:3801-3900batch: iter_time=7.977e-05, forward_time=0.143, loss_ctc=86.157, loss_interctc_layer6=83.208, loss_interctc_layer12=68.901, loss_interctc_layer15=63.209, loss_interctc_layer21=89.194, loss=78.134, backward_time=0.210, grad_norm=61.954, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.295e-05, train_time=1.377 +[gpua006:0/64] 2024-02-09 10:19:44,213 (trainer:756) INFO: 24epoch:train:3901-4000batch: iter_time=7.779e-05, forward_time=0.141, loss_ctc=70.832, loss_interctc_layer6=81.264, loss_interctc_layer12=67.336, loss_interctc_layer15=62.031, loss_interctc_layer21=73.068, loss=70.906, backward_time=0.209, grad_norm=76.544, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.294e-05, train_time=1.005 +[gpua006:0/64] 2024-02-09 10:21:40,349 (trainer:756) INFO: 24epoch:train:4001-4100batch: iter_time=8.395e-05, forward_time=0.142, loss_ctc=75.130, loss_interctc_layer6=86.465, loss_interctc_layer12=72.084, loss_interctc_layer15=66.255, loss_interctc_layer21=77.360, loss=75.459, backward_time=0.208, grad_norm=65.132, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.293e-05, train_time=1.161 +[gpua006:0/64] 2024-02-09 10:24:09,165 (trainer:756) INFO: 24epoch:train:4101-4200batch: iter_time=8.002e-05, forward_time=0.141, loss_ctc=69.979, loss_interctc_layer6=77.844, loss_interctc_layer12=64.921, loss_interctc_layer15=59.677, loss_interctc_layer21=72.265, loss=68.937, backward_time=0.208, grad_norm=65.455, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.292e-05, train_time=1.488 +[gpua006:0/64] 2024-02-09 10:26:30,917 (trainer:756) INFO: 24epoch:train:4201-4300batch: iter_time=7.524e-05, forward_time=0.146, loss_ctc=78.934, loss_interctc_layer6=84.685, loss_interctc_layer12=70.400, loss_interctc_layer15=64.622, loss_interctc_layer21=81.499, loss=76.028, backward_time=0.209, grad_norm=74.334, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.290e-05, train_time=1.417 +[gpua006:0/64] 2024-02-09 10:29:17,832 (trainer:756) INFO: 24epoch:train:4301-4400batch: iter_time=7.912e-05, forward_time=0.204, loss_ctc=64.729, loss_interctc_layer6=77.727, loss_interctc_layer12=64.369, loss_interctc_layer15=59.042, loss_interctc_layer21=66.797, loss=66.533, backward_time=0.295, grad_norm=63.739, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.289e-05, train_time=1.669 +[gpua006:0/64] 2024-02-09 10:31:17,850 (trainer:756) INFO: 24epoch:train:4401-4500batch: iter_time=8.033e-05, forward_time=0.142, loss_ctc=82.764, loss_interctc_layer6=85.640, loss_interctc_layer12=71.178, loss_interctc_layer15=65.430, loss_interctc_layer21=85.397, loss=78.082, backward_time=0.209, grad_norm=65.303, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.288e-05, train_time=1.200 +[gpua006:0/64] 2024-02-09 10:33:31,402 (trainer:756) INFO: 24epoch:train:4501-4600batch: iter_time=8.157e-05, forward_time=0.142, loss_ctc=79.511, loss_interctc_layer6=84.865, loss_interctc_layer12=71.013, loss_interctc_layer15=65.357, loss_interctc_layer21=82.391, loss=76.627, backward_time=0.208, grad_norm=65.513, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.287e-05, train_time=1.336 +[gpua006:0/64] 2024-02-09 10:35:59,079 (trainer:756) INFO: 24epoch:train:4601-4700batch: iter_time=7.862e-05, forward_time=0.142, loss_ctc=77.166, loss_interctc_layer6=92.721, loss_interctc_layer12=77.688, loss_interctc_layer15=71.709, loss_interctc_layer21=79.361, loss=79.729, backward_time=0.207, grad_norm=84.163, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.286e-05, train_time=1.477 +[gpua006:0/64] 2024-02-09 10:37:56,239 (trainer:756) INFO: 24epoch:train:4701-4800batch: iter_time=7.596e-05, forward_time=0.142, loss_ctc=81.880, loss_interctc_layer6=87.819, loss_interctc_layer12=72.890, loss_interctc_layer15=66.850, loss_interctc_layer21=84.584, loss=78.805, backward_time=0.209, grad_norm=81.500, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.284e-05, train_time=1.170 +[gpua006:0/64] 2024-02-09 10:39:51,706 (trainer:756) INFO: 24epoch:train:4801-4900batch: iter_time=7.503e-05, forward_time=0.143, loss_ctc=75.230, loss_interctc_layer6=89.429, loss_interctc_layer12=75.175, loss_interctc_layer15=69.370, loss_interctc_layer21=77.514, loss=77.344, backward_time=0.209, grad_norm=64.721, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.283e-05, train_time=1.156 +[gpua006:0/64] 2024-02-09 10:42:04,634 (trainer:756) INFO: 24epoch:train:4901-5000batch: iter_time=7.838e-05, forward_time=0.142, loss_ctc=74.344, loss_interctc_layer6=87.336, loss_interctc_layer12=73.067, loss_interctc_layer15=67.275, loss_interctc_layer21=76.789, loss=75.762, backward_time=0.208, grad_norm=69.695, clip=100.000, loss_scale=2.576e+31, optim_step_time=0.136, optim0_lr0=8.282e-05, train_time=1.329 +[gpua006:0/64] 2024-02-09 10:42:24,663 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-09 10:42:42,828 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 10:42:46,201 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 10:42:46,201 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-09 10:42:46,241 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 10:49:15,326 (trainer:756) INFO: 24epoch:train:5001-5100batch: iter_time=2.966, forward_time=0.143, loss_ctc=88.383, loss_interctc_layer6=84.958, loss_interctc_layer12=70.632, loss_interctc_layer15=64.948, loss_interctc_layer21=91.607, loss=80.106, backward_time=0.211, grad_norm=58.035, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.281e-05, train_time=4.307 +[gpua006:0/64] 2024-02-09 10:50:50,432 (trainer:756) INFO: 24epoch:train:5101-5200batch: iter_time=7.826e-05, forward_time=0.143, loss_ctc=95.535, loss_interctc_layer6=85.770, loss_interctc_layer12=71.119, loss_interctc_layer15=65.217, loss_interctc_layer21=98.981, loss=83.324, backward_time=0.211, grad_norm=79.573, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.280e-05, train_time=0.951 +[gpua006:0/64] 2024-02-09 10:52:56,456 (trainer:756) INFO: 24epoch:train:5201-5300batch: iter_time=7.807e-05, forward_time=0.143, loss_ctc=66.997, loss_interctc_layer6=74.171, loss_interctc_layer12=61.232, loss_interctc_layer15=56.244, loss_interctc_layer21=69.233, loss=65.575, backward_time=0.211, grad_norm=57.984, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.279e-05, train_time=1.260 +[gpua006:0/64] 2024-02-09 10:55:33,021 (trainer:756) INFO: 24epoch:train:5301-5400batch: iter_time=8.215e-05, forward_time=0.211, loss_ctc=75.533, loss_interctc_layer6=87.646, loss_interctc_layer12=73.145, loss_interctc_layer15=67.347, loss_interctc_layer21=77.799, loss=76.294, backward_time=0.276, grad_norm=65.549, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.146, optim0_lr0=8.277e-05, train_time=1.565 +[gpua006:0/64] 2024-02-09 10:57:55,205 (trainer:756) INFO: 24epoch:train:5401-5500batch: iter_time=8.026e-05, forward_time=0.141, loss_ctc=88.636, loss_interctc_layer6=85.900, loss_interctc_layer12=71.462, loss_interctc_layer15=65.558, loss_interctc_layer21=91.672, loss=80.645, backward_time=0.208, grad_norm=74.235, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.276e-05, train_time=1.421 +[gpua006:0/64] 2024-02-09 11:00:46,315 (trainer:756) INFO: 24epoch:train:5501-5600batch: iter_time=8.792e-05, forward_time=0.142, loss_ctc=74.374, loss_interctc_layer6=76.510, loss_interctc_layer12=63.338, loss_interctc_layer15=57.961, loss_interctc_layer21=77.046, loss=69.846, backward_time=0.207, grad_norm=63.558, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.275e-05, train_time=1.711 +[gpua006:0/64] 2024-02-09 11:02:34,640 (trainer:756) INFO: 24epoch:train:5601-5700batch: iter_time=7.658e-05, forward_time=0.142, loss_ctc=72.919, loss_interctc_layer6=81.658, loss_interctc_layer12=67.700, loss_interctc_layer15=62.107, loss_interctc_layer21=75.246, loss=71.926, backward_time=0.209, grad_norm=63.194, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.274e-05, train_time=1.083 +[gpua006:0/64] 2024-02-09 11:04:26,614 (trainer:756) INFO: 24epoch:train:5701-5800batch: iter_time=7.838e-05, forward_time=0.142, loss_ctc=87.734, loss_interctc_layer6=86.343, loss_interctc_layer12=71.954, loss_interctc_layer15=66.134, loss_interctc_layer21=90.567, loss=80.546, backward_time=0.210, grad_norm=77.429, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.273e-05, train_time=1.120 +[gpua006:0/64] 2024-02-09 11:06:47,468 (trainer:756) INFO: 24epoch:train:5801-5900batch: iter_time=8.289e-05, forward_time=0.144, loss_ctc=75.618, loss_interctc_layer6=86.225, loss_interctc_layer12=72.562, loss_interctc_layer15=67.104, loss_interctc_layer21=77.631, loss=75.828, backward_time=0.208, grad_norm=81.441, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.271e-05, train_time=1.407 +[gpua006:0/64] 2024-02-09 11:08:51,082 (trainer:756) INFO: 24epoch:train:5901-6000batch: iter_time=8.546e-05, forward_time=0.143, loss_ctc=76.281, loss_interctc_layer6=87.733, loss_interctc_layer12=72.927, loss_interctc_layer15=66.823, loss_interctc_layer21=78.696, loss=76.492, backward_time=0.209, grad_norm=78.864, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.270e-05, train_time=1.237 +[gpua006:0/64] 2024-02-09 11:10:46,463 (trainer:756) INFO: 24epoch:train:6001-6100batch: iter_time=8.097e-05, forward_time=0.142, loss_ctc=91.754, loss_interctc_layer6=93.138, loss_interctc_layer12=77.373, loss_interctc_layer15=71.256, loss_interctc_layer21=94.641, loss=85.632, backward_time=0.210, grad_norm=74.816, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.269e-05, train_time=1.154 +[gpua006:0/64] 2024-02-09 11:13:07,552 (trainer:756) INFO: 24epoch:train:6101-6200batch: iter_time=7.963e-05, forward_time=0.142, loss_ctc=74.013, loss_interctc_layer6=84.290, loss_interctc_layer12=70.794, loss_interctc_layer15=65.420, loss_interctc_layer21=76.307, loss=74.165, backward_time=0.209, grad_norm=73.074, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.268e-05, train_time=1.411 +[gpua006:0/64] 2024-02-09 11:14:33,279 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-09 11:14:51,587 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 11:14:54,968 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 11:14:54,968 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-09 11:14:55,035 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 11:20:32,445 (trainer:756) INFO: 24epoch:train:6201-6300batch: iter_time=2.674, forward_time=0.232, loss_ctc=89.359, loss_interctc_layer6=90.565, loss_interctc_layer12=75.432, loss_interctc_layer15=69.275, loss_interctc_layer21=92.411, loss=83.409, backward_time=0.221, grad_norm=68.481, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.267e-05, train_time=4.448 +[gpua006:0/64] 2024-02-09 11:22:21,242 (trainer:756) INFO: 24epoch:train:6301-6400batch: iter_time=9.235e-05, forward_time=0.144, loss_ctc=92.373, loss_interctc_layer6=82.783, loss_interctc_layer12=68.620, loss_interctc_layer15=62.824, loss_interctc_layer21=95.766, loss=80.473, backward_time=0.209, grad_norm=72.258, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.266e-05, train_time=1.089 +[gpua006:0/64] 2024-02-09 11:24:12,588 (trainer:756) INFO: 24epoch:train:6401-6500batch: iter_time=9.231e-05, forward_time=0.142, loss_ctc=76.370, loss_interctc_layer6=80.842, loss_interctc_layer12=66.888, loss_interctc_layer15=61.563, loss_interctc_layer21=78.931, loss=72.919, backward_time=0.208, grad_norm=67.119, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.264e-05, train_time=1.113 +[gpua006:0/64] 2024-02-09 11:26:23,390 (trainer:756) INFO: 24epoch:train:6501-6600batch: iter_time=9.721e-05, forward_time=0.143, loss_ctc=76.304, loss_interctc_layer6=86.405, loss_interctc_layer12=72.052, loss_interctc_layer15=66.101, loss_interctc_layer21=78.623, loss=75.897, backward_time=0.208, grad_norm=93.652, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.263e-05, train_time=1.308 +[gpua006:0/64] 2024-02-09 11:27:58,806 (trainer:756) INFO: 24epoch:train:6601-6700batch: iter_time=9.382e-05, forward_time=0.142, loss_ctc=73.792, loss_interctc_layer6=76.954, loss_interctc_layer12=64.002, loss_interctc_layer15=58.819, loss_interctc_layer21=76.173, loss=69.948, backward_time=0.209, grad_norm=68.769, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.262e-05, train_time=0.954 +[gpua006:0/64] 2024-02-09 11:30:35,631 (trainer:756) INFO: 24epoch:train:6701-6800batch: iter_time=8.443e-05, forward_time=0.142, loss_ctc=84.400, loss_interctc_layer6=83.255, loss_interctc_layer12=69.173, loss_interctc_layer15=63.341, loss_interctc_layer21=87.217, loss=77.477, backward_time=0.209, grad_norm=64.537, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.261e-05, train_time=1.568 +[gpua006:0/64] 2024-02-09 11:33:17,044 (trainer:756) INFO: 24epoch:train:6801-6900batch: iter_time=8.407e-05, forward_time=0.141, loss_ctc=66.372, loss_interctc_layer6=76.744, loss_interctc_layer12=63.374, loss_interctc_layer15=58.018, loss_interctc_layer21=68.730, loss=66.647, backward_time=0.208, grad_norm=64.518, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.260e-05, train_time=1.614 +[gpua006:0/64] 2024-02-09 11:33:58,491 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 11:35:13,524 (trainer:756) INFO: 24epoch:train:6901-7000batch: iter_time=8.067e-05, forward_time=0.144, loss_ctc=84.268, loss_interctc_layer6=85.263, loss_interctc_layer12=70.849, loss_interctc_layer15=65.051, loss_interctc_layer21=86.891, loss=78.465, backward_time=0.209, grad_norm=67.981, clip=100.000, loss_scale=2.807e+31, optim_step_time=0.136, optim0_lr0=8.259e-05, train_time=1.165 +[gpua006:0/64] 2024-02-09 11:37:27,081 (trainer:756) INFO: 24epoch:train:7001-7100batch: iter_time=0.002, forward_time=0.250, loss_ctc=82.753, loss_interctc_layer6=85.243, loss_interctc_layer12=71.062, loss_interctc_layer15=65.323, loss_interctc_layer21=85.301, loss=77.936, backward_time=0.246, grad_norm=64.995, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.154, optim0_lr0=8.257e-05, train_time=1.333 +[gpua006:0/64] 2024-02-09 11:39:27,822 (trainer:756) INFO: 24epoch:train:7101-7200batch: iter_time=8.438e-05, forward_time=0.143, loss_ctc=78.791, loss_interctc_layer6=91.940, loss_interctc_layer12=76.863, loss_interctc_layer15=70.816, loss_interctc_layer21=80.977, loss=79.878, backward_time=0.208, grad_norm=72.785, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.256e-05, train_time=1.209 +[gpua006:0/64] 2024-02-09 11:41:33,866 (trainer:756) INFO: 24epoch:train:7201-7300batch: iter_time=8.073e-05, forward_time=0.142, loss_ctc=85.927, loss_interctc_layer6=87.656, loss_interctc_layer12=72.601, loss_interctc_layer15=66.644, loss_interctc_layer21=88.750, loss=80.316, backward_time=0.208, grad_norm=67.990, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.255e-05, train_time=1.260 +[gpua006:0/64] 2024-02-09 11:44:49,369 (trainer:756) INFO: 24epoch:train:7301-7400batch: iter_time=8.097e-05, forward_time=0.142, loss_ctc=80.406, loss_interctc_layer6=89.053, loss_interctc_layer12=74.681, loss_interctc_layer15=69.007, loss_interctc_layer21=82.895, loss=79.208, backward_time=0.206, grad_norm=86.793, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.254e-05, train_time=1.955 +[gpua006:0/64] 2024-02-09 11:46:56,502 (trainer:756) INFO: 24epoch:train:7401-7500batch: iter_time=7.734e-05, forward_time=0.143, loss_ctc=77.717, loss_interctc_layer6=85.935, loss_interctc_layer12=71.657, loss_interctc_layer15=65.900, loss_interctc_layer21=80.159, loss=76.274, backward_time=0.210, grad_norm=62.742, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.253e-05, train_time=1.271 +[gpua006:0/64] 2024-02-09 11:47:16,533 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-09 11:47:35,070 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 11:47:38,433 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 11:47:38,434 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-09 11:47:38,437 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 11:54:09,089 (trainer:756) INFO: 24epoch:train:7501-7600batch: iter_time=3.036, forward_time=0.229, loss_ctc=82.586, loss_interctc_layer6=84.681, loss_interctc_layer12=70.176, loss_interctc_layer15=64.504, loss_interctc_layer21=85.417, loss=77.473, backward_time=0.228, grad_norm=80.183, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.252e-05, train_time=4.326 +[gpua006:0/64] 2024-02-09 11:54:26,908 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 11:55:48,662 (trainer:756) INFO: 24epoch:train:7601-7700batch: iter_time=8.361e-05, forward_time=0.143, loss_ctc=85.277, loss_interctc_layer6=85.141, loss_interctc_layer12=70.447, loss_interctc_layer15=64.612, loss_interctc_layer21=88.195, loss=78.734, backward_time=0.210, grad_norm=67.426, clip=100.000, loss_scale=1.188e+31, optim_step_time=0.136, optim0_lr0=8.250e-05, train_time=0.993 +[gpua006:0/64] 2024-02-09 11:58:28,600 (trainer:756) INFO: 24epoch:train:7701-7800batch: iter_time=8.611e-05, forward_time=0.142, loss_ctc=64.559, loss_interctc_layer6=74.105, loss_interctc_layer12=61.177, loss_interctc_layer15=56.078, loss_interctc_layer21=66.634, loss=64.510, backward_time=0.208, grad_norm=68.095, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.249e-05, train_time=1.602 +[gpua006:0/64] 2024-02-09 12:00:34,441 (trainer:756) INFO: 24epoch:train:7801-7900batch: iter_time=9.207e-05, forward_time=0.143, loss_ctc=73.607, loss_interctc_layer6=87.004, loss_interctc_layer12=72.595, loss_interctc_layer15=66.781, loss_interctc_layer21=75.821, loss=75.162, backward_time=0.209, grad_norm=100.556, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.248e-05, train_time=1.258 +[gpua006:0/64] 2024-02-09 12:02:40,906 (trainer:756) INFO: 24epoch:train:7901-8000batch: iter_time=8.961e-05, forward_time=0.141, loss_ctc=79.479, loss_interctc_layer6=85.503, loss_interctc_layer12=71.005, loss_interctc_layer15=65.070, loss_interctc_layer21=82.212, loss=76.654, backward_time=0.207, grad_norm=83.523, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.247e-05, train_time=1.264 +[gpua006:0/64] 2024-02-09 12:05:10,816 (trainer:756) INFO: 24epoch:train:8001-8100batch: iter_time=9.135e-05, forward_time=0.142, loss_ctc=69.822, loss_interctc_layer6=77.159, loss_interctc_layer12=63.837, loss_interctc_layer15=58.419, loss_interctc_layer21=72.393, loss=68.326, backward_time=0.207, grad_norm=84.141, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.246e-05, train_time=1.499 +[gpua006:0/64] 2024-02-09 12:07:37,469 (trainer:756) INFO: 24epoch:train:8101-8200batch: iter_time=8.832e-05, forward_time=0.150, loss_ctc=71.604, loss_interctc_layer6=81.437, loss_interctc_layer12=67.509, loss_interctc_layer15=61.993, loss_interctc_layer21=73.941, loss=71.297, backward_time=0.207, grad_norm=59.864, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.245e-05, train_time=1.466 +[gpua006:0/64] 2024-02-09 12:10:30,347 (trainer:756) INFO: 24epoch:train:8201-8300batch: iter_time=8.926e-05, forward_time=0.142, loss_ctc=84.336, loss_interctc_layer6=85.242, loss_interctc_layer12=70.988, loss_interctc_layer15=65.220, loss_interctc_layer21=87.197, loss=78.597, backward_time=0.208, grad_norm=77.456, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.243e-05, train_time=1.729 +[gpua006:0/64] 2024-02-09 12:13:13,172 (trainer:756) INFO: 24epoch:train:8301-8400batch: iter_time=9.304e-05, forward_time=0.324, loss_ctc=74.710, loss_interctc_layer6=86.083, loss_interctc_layer12=72.365, loss_interctc_layer15=66.817, loss_interctc_layer21=76.637, loss=75.323, backward_time=0.263, grad_norm=63.153, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=8.242e-05, train_time=1.628 +[gpua006:0/64] 2024-02-09 12:15:12,719 (trainer:756) INFO: 24epoch:train:8401-8500batch: iter_time=8.761e-05, forward_time=0.142, loss_ctc=71.956, loss_interctc_layer6=87.878, loss_interctc_layer12=72.916, loss_interctc_layer15=67.075, loss_interctc_layer21=74.322, loss=74.829, backward_time=0.210, grad_norm=61.200, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.241e-05, train_time=1.195 +[gpua006:0/64] 2024-02-09 12:17:35,244 (trainer:756) INFO: 24epoch:train:8501-8600batch: iter_time=8.346e-05, forward_time=0.143, loss_ctc=84.149, loss_interctc_layer6=92.920, loss_interctc_layer12=77.057, loss_interctc_layer15=70.905, loss_interctc_layer21=86.591, loss=82.325, backward_time=0.211, grad_norm=64.669, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.240e-05, train_time=1.424 +[gpua006:0/64] 2024-02-09 12:19:26,208 (trainer:756) INFO: 24epoch:train:8601-8700batch: iter_time=8.296e-05, forward_time=0.143, loss_ctc=70.761, loss_interctc_layer6=84.245, loss_interctc_layer12=70.643, loss_interctc_layer15=65.183, loss_interctc_layer21=73.064, loss=72.779, backward_time=0.211, grad_norm=66.350, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.239e-05, train_time=1.110 +[gpua006:0/64] 2024-02-09 12:20:37,573 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-09 12:20:55,981 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 12:20:59,619 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 12:20:59,619 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-09 12:20:59,622 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 12:26:26,956 (trainer:756) INFO: 24epoch:train:8701-8800batch: iter_time=3.054, forward_time=0.189, loss_ctc=86.644, loss_interctc_layer6=89.233, loss_interctc_layer12=74.077, loss_interctc_layer15=68.086, loss_interctc_layer21=89.653, loss=81.539, backward_time=0.228, grad_norm=68.381, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.238e-05, train_time=4.207 +[gpua006:0/64] 2024-02-09 12:28:04,288 (trainer:756) INFO: 24epoch:train:8801-8900batch: iter_time=9.300e-05, forward_time=0.141, loss_ctc=92.375, loss_interctc_layer6=82.327, loss_interctc_layer12=68.109, loss_interctc_layer15=62.360, loss_interctc_layer21=95.676, loss=80.169, backward_time=0.210, grad_norm=61.116, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=8.236e-05, train_time=0.973 +[gpua006:0/64] 2024-02-09 12:29:49,772 (trainer:756) INFO: 24epoch:train:8901-9000batch: iter_time=9.079e-05, forward_time=0.142, loss_ctc=75.994, loss_interctc_layer6=80.254, loss_interctc_layer12=66.475, loss_interctc_layer15=61.109, loss_interctc_layer21=78.411, loss=72.449, backward_time=0.209, grad_norm=78.937, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.235e-05, train_time=1.055 +[gpua006:0/64] 2024-02-09 12:32:05,832 (trainer:756) INFO: 24epoch:train:9001-9100batch: iter_time=8.541e-05, forward_time=0.142, loss_ctc=75.707, loss_interctc_layer6=85.768, loss_interctc_layer12=71.363, loss_interctc_layer15=65.615, loss_interctc_layer21=78.213, loss=75.333, backward_time=0.209, grad_norm=67.399, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.234e-05, train_time=1.360 +[gpua006:0/64] 2024-02-09 12:34:17,458 (trainer:756) INFO: 24epoch:train:9101-9200batch: iter_time=9.066e-05, forward_time=0.141, loss_ctc=74.140, loss_interctc_layer6=77.515, loss_interctc_layer12=64.493, loss_interctc_layer15=59.254, loss_interctc_layer21=76.385, loss=70.357, backward_time=0.209, grad_norm=67.594, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.233e-05, train_time=1.316 +[gpua006:0/64] 2024-02-09 12:37:07,037 (trainer:756) INFO: 24epoch:train:9201-9300batch: iter_time=8.293e-05, forward_time=0.142, loss_ctc=83.864, loss_interctc_layer6=83.051, loss_interctc_layer12=68.796, loss_interctc_layer15=62.983, loss_interctc_layer21=86.767, loss=77.092, backward_time=0.209, grad_norm=90.043, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.232e-05, train_time=1.696 +[gpua006:0/64] 2024-02-09 12:39:31,057 (trainer:756) INFO: 24epoch:train:9301-9400batch: iter_time=8.189e-05, forward_time=0.142, loss_ctc=66.815, loss_interctc_layer6=77.047, loss_interctc_layer12=63.546, loss_interctc_layer15=58.201, loss_interctc_layer21=69.128, loss=66.947, backward_time=0.209, grad_norm=74.782, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.231e-05, train_time=1.439 +[gpua006:0/64] 2024-02-09 12:41:19,456 (trainer:756) INFO: 24epoch:train:9401-9500batch: iter_time=8.972e-05, forward_time=0.142, loss_ctc=84.172, loss_interctc_layer6=84.589, loss_interctc_layer12=70.030, loss_interctc_layer15=64.256, loss_interctc_layer21=86.815, loss=77.972, backward_time=0.209, grad_norm=63.264, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=8.229e-05, train_time=1.085 +[gpua006:0/64] 2024-02-09 12:43:47,000 (trainer:756) INFO: 24epoch:train:9501-9600batch: iter_time=8.359e-05, forward_time=0.208, loss_ctc=82.790, loss_interctc_layer6=84.975, loss_interctc_layer12=70.852, loss_interctc_layer15=65.101, loss_interctc_layer21=85.465, loss=77.837, backward_time=0.221, grad_norm=64.571, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.228e-05, train_time=1.474 +[gpua006:0/64] 2024-02-09 12:46:12,822 (trainer:756) INFO: 24epoch:train:9601-9700batch: iter_time=9.190e-05, forward_time=0.148, loss_ctc=78.933, loss_interctc_layer6=92.198, loss_interctc_layer12=77.049, loss_interctc_layer15=71.062, loss_interctc_layer21=81.185, loss=80.086, backward_time=0.222, grad_norm=69.257, clip=100.000, loss_scale=1.846e+31, optim_step_time=0.136, optim0_lr0=8.227e-05, train_time=1.459 +[gpua006:0/64] 2024-02-09 12:48:10,541 (trainer:756) INFO: 24epoch:train:9701-9800batch: iter_time=8.571e-05, forward_time=0.142, loss_ctc=85.657, loss_interctc_layer6=87.758, loss_interctc_layer12=72.594, loss_interctc_layer15=66.485, loss_interctc_layer21=88.277, loss=80.154, backward_time=0.210, grad_norm=79.105, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.226e-05, train_time=1.177 +[gpua006:0/64] 2024-02-09 12:50:08,176 (trainer:756) INFO: 24epoch:train:9801-9900batch: iter_time=8.426e-05, forward_time=0.142, loss_ctc=80.460, loss_interctc_layer6=88.685, loss_interctc_layer12=74.379, loss_interctc_layer15=68.670, loss_interctc_layer21=82.855, loss=79.010, backward_time=0.210, grad_norm=73.700, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=8.225e-05, train_time=1.176 +[gpua006:0/64] 2024-02-09 12:52:25,778 (trainer:756) INFO: 24epoch:train:9901-10000batch: iter_time=8.725e-05, forward_time=0.143, loss_ctc=78.589, loss_interctc_layer6=87.330, loss_interctc_layer12=72.921, loss_interctc_layer15=67.003, loss_interctc_layer21=80.961, loss=77.361, backward_time=0.208, grad_norm=71.137, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.224e-05, train_time=1.376 +[gpua006:0/64] 2024-02-09 12:52:45,806 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-09 12:53:04,386 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 12:53:07,795 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 12:53:07,795 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-09 12:53:07,841 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 12:59:43,605 (trainer:756) INFO: 24epoch:train:10001-10100batch: iter_time=3.111, forward_time=0.143, loss_ctc=85.986, loss_interctc_layer6=83.938, loss_interctc_layer12=69.511, loss_interctc_layer15=63.754, loss_interctc_layer21=89.016, loss=78.441, backward_time=0.209, grad_norm=163.263, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.222e-05, train_time=4.378 +[gpua006:0/64] 2024-02-09 13:01:30,966 (trainer:756) INFO: 24epoch:train:10101-10200batch: iter_time=8.234e-05, forward_time=0.143, loss_ctc=94.441, loss_interctc_layer6=85.337, loss_interctc_layer12=70.620, loss_interctc_layer15=64.763, loss_interctc_layer21=97.871, loss=82.606, backward_time=0.210, grad_norm=62.573, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.221e-05, train_time=1.073 +[gpua006:0/64] 2024-02-09 13:03:27,561 (trainer:756) INFO: 24epoch:train:10201-10300batch: iter_time=8.192e-05, forward_time=0.142, loss_ctc=65.747, loss_interctc_layer6=73.709, loss_interctc_layer12=60.809, loss_interctc_layer15=55.729, loss_interctc_layer21=67.914, loss=64.782, backward_time=0.210, grad_norm=71.086, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.220e-05, train_time=1.164 +[gpua006:0/64] 2024-02-09 13:05:44,086 (trainer:756) INFO: 24epoch:train:10301-10400batch: iter_time=8.390e-05, forward_time=0.143, loss_ctc=74.495, loss_interctc_layer6=87.345, loss_interctc_layer12=72.658, loss_interctc_layer15=66.724, loss_interctc_layer21=76.711, loss=75.587, backward_time=0.208, grad_norm=62.592, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.219e-05, train_time=1.366 +[gpua006:0/64] 2024-02-09 13:07:57,655 (trainer:756) INFO: 24epoch:train:10401-10500batch: iter_time=8.813e-05, forward_time=0.185, loss_ctc=87.911, loss_interctc_layer6=85.153, loss_interctc_layer12=70.589, loss_interctc_layer15=64.618, loss_interctc_layer21=90.842, loss=79.823, backward_time=0.214, grad_norm=70.005, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.218e-05, train_time=1.335 +[gpua006:0/64] 2024-02-09 13:10:06,045 (trainer:756) INFO: 24epoch:train:10501-10600batch: iter_time=8.653e-05, forward_time=0.181, loss_ctc=74.170, loss_interctc_layer6=76.834, loss_interctc_layer12=63.506, loss_interctc_layer15=58.081, loss_interctc_layer21=76.852, loss=69.889, backward_time=0.244, grad_norm=106.223, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.217e-05, train_time=1.282 +[gpua006:0/64] 2024-02-09 13:13:02,036 (trainer:756) INFO: 24epoch:train:10601-10700batch: iter_time=8.351e-05, forward_time=0.142, loss_ctc=72.803, loss_interctc_layer6=81.441, loss_interctc_layer12=67.559, loss_interctc_layer15=61.983, loss_interctc_layer21=75.018, loss=71.761, backward_time=0.209, grad_norm=75.123, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.215e-05, train_time=1.761 +[gpua006:0/64] 2024-02-09 13:15:11,125 (trainer:756) INFO: 24epoch:train:10701-10800batch: iter_time=8.501e-05, forward_time=0.142, loss_ctc=87.005, loss_interctc_layer6=85.275, loss_interctc_layer12=70.866, loss_interctc_layer15=65.094, loss_interctc_layer21=90.023, loss=79.652, backward_time=0.207, grad_norm=54.427, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=8.214e-05, train_time=1.291 +[gpua006:0/64] 2024-02-09 13:17:32,085 (trainer:756) INFO: 24epoch:train:10801-10900batch: iter_time=8.385e-05, forward_time=0.143, loss_ctc=74.682, loss_interctc_layer6=85.878, loss_interctc_layer12=71.987, loss_interctc_layer15=66.513, loss_interctc_layer21=76.860, loss=75.184, backward_time=0.208, grad_norm=72.878, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.213e-05, train_time=1.409 +[gpua006:0/64] 2024-02-09 13:19:20,040 (trainer:756) INFO: 24epoch:train:10901-11000batch: iter_time=8.613e-05, forward_time=0.142, loss_ctc=74.913, loss_interctc_layer6=87.239, loss_interctc_layer12=72.269, loss_interctc_layer15=66.305, loss_interctc_layer21=77.364, loss=75.618, backward_time=0.210, grad_norm=64.758, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.212e-05, train_time=1.079 +[gpua006:0/64] 2024-02-09 13:21:49,368 (trainer:756) INFO: 24epoch:train:11001-11100batch: iter_time=8.115e-05, forward_time=0.143, loss_ctc=91.066, loss_interctc_layer6=92.540, loss_interctc_layer12=76.878, loss_interctc_layer15=70.661, loss_interctc_layer21=93.849, loss=84.999, backward_time=0.210, grad_norm=95.983, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.211e-05, train_time=1.493 +[gpua006:0/64] 2024-02-09 13:24:16,427 (trainer:756) INFO: 24epoch:train:11101-11200batch: iter_time=8.434e-05, forward_time=0.142, loss_ctc=73.496, loss_interctc_layer6=84.122, loss_interctc_layer12=70.683, loss_interctc_layer15=65.119, loss_interctc_layer21=75.710, loss=73.826, backward_time=0.209, grad_norm=81.702, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.210e-05, train_time=1.470 +[gpua006:0/64] 2024-02-09 13:25:29,382 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-09 13:25:48,005 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 13:25:51,359 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 13:25:51,359 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-09 13:25:51,391 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 13:31:11,134 (trainer:756) INFO: 24epoch:train:11201-11300batch: iter_time=3.017, forward_time=0.184, loss_ctc=85.252, loss_interctc_layer6=89.337, loss_interctc_layer12=74.245, loss_interctc_layer15=68.116, loss_interctc_layer21=88.302, loss=81.051, backward_time=0.219, grad_norm=68.180, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.209e-05, train_time=4.146 +[gpua006:0/64] 2024-02-09 13:32:46,297 (trainer:756) INFO: 24epoch:train:11301-11400batch: iter_time=8.579e-05, forward_time=0.144, loss_ctc=85.204, loss_interctc_layer6=82.012, loss_interctc_layer12=67.709, loss_interctc_layer15=61.968, loss_interctc_layer21=88.506, loss=77.080, backward_time=0.211, grad_norm=64.632, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.207e-05, train_time=0.952 +[gpua006:0/64] 2024-02-09 13:34:58,221 (trainer:756) INFO: 24epoch:train:11401-11500batch: iter_time=8.871e-05, forward_time=0.142, loss_ctc=70.895, loss_interctc_layer6=80.631, loss_interctc_layer12=66.731, loss_interctc_layer15=61.381, loss_interctc_layer21=73.240, loss=70.576, backward_time=0.209, grad_norm=56.756, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.206e-05, train_time=1.319 +[gpua006:0/64] 2024-02-09 13:37:12,106 (trainer:756) INFO: 24epoch:train:11501-11600batch: iter_time=9.007e-05, forward_time=0.143, loss_ctc=73.426, loss_interctc_layer6=84.867, loss_interctc_layer12=70.432, loss_interctc_layer15=64.654, loss_interctc_layer21=75.784, loss=73.832, backward_time=0.208, grad_norm=69.593, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.205e-05, train_time=1.339 +[gpua006:0/64] 2024-02-09 13:39:53,720 (trainer:756) INFO: 24epoch:train:11601-11700batch: iter_time=8.956e-05, forward_time=0.141, loss_ctc=69.422, loss_interctc_layer6=77.362, loss_interctc_layer12=64.311, loss_interctc_layer15=58.981, loss_interctc_layer21=71.651, loss=68.345, backward_time=0.207, grad_norm=63.193, clip=100.000, loss_scale=3.691e+31, optim_step_time=0.136, optim0_lr0=8.204e-05, train_time=1.612 +[gpua006:0/64] 2024-02-09 13:42:13,172 (trainer:756) INFO: 24epoch:train:11701-11800batch: iter_time=3.008e-04, forward_time=0.260, loss_ctc=78.635, loss_interctc_layer6=83.234, loss_interctc_layer12=69.084, loss_interctc_layer15=63.222, loss_interctc_layer21=81.255, loss=75.086, backward_time=0.263, grad_norm=60.592, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=8.203e-05, train_time=1.397 +[gpua006:0/64] 2024-02-09 13:44:01,621 (trainer:756) INFO: 24epoch:train:11801-11900batch: iter_time=7.995e-05, forward_time=0.143, loss_ctc=64.712, loss_interctc_layer6=77.148, loss_interctc_layer12=63.694, loss_interctc_layer15=58.391, loss_interctc_layer21=66.938, loss=66.177, backward_time=0.209, grad_norm=88.811, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.202e-05, train_time=1.085 +[gpua006:0/64] 2024-02-09 13:46:32,674 (trainer:756) INFO: 24epoch:train:11901-12000batch: iter_time=7.889e-05, forward_time=0.142, loss_ctc=82.653, loss_interctc_layer6=84.865, loss_interctc_layer12=70.374, loss_interctc_layer15=64.565, loss_interctc_layer21=85.435, loss=77.579, backward_time=0.209, grad_norm=61.238, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.201e-05, train_time=1.510 +[gpua006:0/64] 2024-02-09 13:48:57,687 (trainer:756) INFO: 24epoch:train:12001-12100batch: iter_time=9.125e-05, forward_time=0.142, loss_ctc=78.237, loss_interctc_layer6=84.303, loss_interctc_layer12=70.303, loss_interctc_layer15=64.718, loss_interctc_layer21=80.872, loss=75.686, backward_time=0.208, grad_norm=69.762, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.135, optim0_lr0=8.199e-05, train_time=1.450 +[gpua006:0/64] 2024-02-09 13:51:08,286 (trainer:756) INFO: 24epoch:train:12101-12200batch: iter_time=8.726e-05, forward_time=0.142, loss_ctc=77.261, loss_interctc_layer6=92.204, loss_interctc_layer12=77.064, loss_interctc_layer15=71.212, loss_interctc_layer21=79.504, loss=79.449, backward_time=0.208, grad_norm=75.854, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.135, optim0_lr0=8.198e-05, train_time=1.305 +[gpua006:0/64] 2024-02-09 13:53:26,502 (trainer:756) INFO: 24epoch:train:12201-12300batch: iter_time=5.734e-04, forward_time=0.222, loss_ctc=81.565, loss_interctc_layer6=87.021, loss_interctc_layer12=71.936, loss_interctc_layer15=65.911, loss_interctc_layer21=84.172, loss=78.121, backward_time=0.269, grad_norm=72.084, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.197e-05, train_time=1.381 +[gpua006:0/64] 2024-02-09 13:55:27,283 (trainer:756) INFO: 24epoch:train:12301-12400batch: iter_time=9.154e-05, forward_time=0.143, loss_ctc=75.212, loss_interctc_layer6=88.199, loss_interctc_layer12=73.745, loss_interctc_layer15=68.068, loss_interctc_layer21=77.621, loss=76.569, backward_time=0.210, grad_norm=65.312, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.196e-05, train_time=1.209 +[gpua006:0/64] 2024-02-09 13:56:30,884 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 13:58:14,465 (trainer:756) INFO: 24epoch:train:12401-12500batch: iter_time=8.034e-05, forward_time=0.144, loss_ctc=73.328, loss_interctc_layer6=86.166, loss_interctc_layer12=72.006, loss_interctc_layer15=66.188, loss_interctc_layer21=75.561, loss=74.650, backward_time=0.208, grad_norm=70.329, clip=100.000, loss_scale=2.766e+31, optim_step_time=0.136, optim0_lr0=8.195e-05, train_time=1.672 +[gpua006:0/64] 2024-02-09 13:58:34,495 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-09 13:58:53,173 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 13:58:56,562 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 13:58:56,562 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-09 13:58:56,565 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 14:05:22,787 (trainer:756) INFO: 24epoch:train:12501-12600batch: iter_time=3.246, forward_time=0.189, loss_ctc=82.408, loss_interctc_layer6=85.047, loss_interctc_layer12=70.680, loss_interctc_layer15=64.808, loss_interctc_layer21=85.462, loss=77.681, backward_time=0.223, grad_norm=64.300, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.194e-05, train_time=4.283 +[gpua006:0/64] 2024-02-09 14:06:58,026 (trainer:756) INFO: 24epoch:train:12601-12700batch: iter_time=8.025e-05, forward_time=0.143, loss_ctc=85.627, loss_interctc_layer6=85.125, loss_interctc_layer12=70.449, loss_interctc_layer15=64.545, loss_interctc_layer21=88.740, loss=78.897, backward_time=0.212, grad_norm=67.914, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.192e-05, train_time=0.952 +[gpua006:0/64] 2024-02-09 14:09:39,011 (trainer:756) INFO: 24epoch:train:12701-12800batch: iter_time=7.882e-05, forward_time=0.141, loss_ctc=63.701, loss_interctc_layer6=73.680, loss_interctc_layer12=60.649, loss_interctc_layer15=55.581, loss_interctc_layer21=65.759, loss=63.874, backward_time=0.209, grad_norm=70.840, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.191e-05, train_time=1.610 +[gpua006:0/64] 2024-02-09 14:12:13,770 (trainer:756) INFO: 24epoch:train:12801-12900batch: iter_time=8.102e-05, forward_time=0.142, loss_ctc=73.759, loss_interctc_layer6=86.898, loss_interctc_layer12=72.347, loss_interctc_layer15=66.495, loss_interctc_layer21=76.044, loss=75.108, backward_time=0.210, grad_norm=75.451, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.190e-05, train_time=1.547 +[gpua006:0/64] 2024-02-09 14:14:57,266 (trainer:756) INFO: 24epoch:train:12901-13000batch: iter_time=3.508e-04, forward_time=0.190, loss_ctc=78.725, loss_interctc_layer6=84.620, loss_interctc_layer12=70.186, loss_interctc_layer15=64.267, loss_interctc_layer21=81.208, loss=75.801, backward_time=0.300, grad_norm=62.294, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.189e-05, train_time=1.633 +[gpua006:0/64] 2024-02-09 14:17:12,209 (trainer:756) INFO: 24epoch:train:13001-13100batch: iter_time=7.928e-04, forward_time=0.147, loss_ctc=68.698, loss_interctc_layer6=75.638, loss_interctc_layer12=62.471, loss_interctc_layer15=57.048, loss_interctc_layer21=71.173, loss=67.006, backward_time=0.211, grad_norm=65.895, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.188e-05, train_time=1.351 +[gpua006:0/64] 2024-02-09 14:17:57,985 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 14:19:39,247 (trainer:756) INFO: 24epoch:train:13101-13200batch: iter_time=7.764e-05, forward_time=0.145, loss_ctc=71.624, loss_interctc_layer6=81.148, loss_interctc_layer12=67.096, loss_interctc_layer15=61.635, loss_interctc_layer21=73.960, loss=71.093, backward_time=0.210, grad_norm=54.512, clip=100.000, loss_scale=1.332e+31, optim_step_time=0.137, optim0_lr0=8.187e-05, train_time=1.470 +[gpua006:0/64] 2024-02-09 14:21:39,234 (trainer:756) INFO: 24epoch:train:13201-13300batch: iter_time=7.889e-05, forward_time=0.142, loss_ctc=84.503, loss_interctc_layer6=84.763, loss_interctc_layer12=70.397, loss_interctc_layer15=64.621, loss_interctc_layer21=87.483, loss=78.353, backward_time=0.209, grad_norm=60.762, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.186e-05, train_time=1.200 +[gpua006:0/64] 2024-02-09 14:24:24,124 (trainer:756) INFO: 24epoch:train:13301-13400batch: iter_time=8.035e-05, forward_time=0.143, loss_ctc=73.460, loss_interctc_layer6=86.009, loss_interctc_layer12=72.241, loss_interctc_layer15=66.732, loss_interctc_layer21=75.550, loss=74.798, backward_time=0.212, grad_norm=66.467, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.184e-05, train_time=1.649 +[gpua006:0/64] 2024-02-09 14:26:59,832 (trainer:756) INFO: 24epoch:train:13401-13500batch: iter_time=8.075e-05, forward_time=0.142, loss_ctc=71.382, loss_interctc_layer6=86.867, loss_interctc_layer12=72.023, loss_interctc_layer15=66.084, loss_interctc_layer21=73.591, loss=73.989, backward_time=0.210, grad_norm=66.825, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=8.183e-05, train_time=1.557 +[gpua006:0/64] 2024-02-09 14:29:02,911 (trainer:756) INFO: 24epoch:train:13501-13600batch: iter_time=8.236e-05, forward_time=0.142, loss_ctc=84.243, loss_interctc_layer6=91.919, loss_interctc_layer12=76.137, loss_interctc_layer15=69.862, loss_interctc_layer21=86.751, loss=81.782, backward_time=0.211, grad_norm=62.510, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.182e-05, train_time=1.231 +[gpua006:0/64] 2024-02-09 14:31:36,838 (trainer:756) INFO: 24epoch:train:13601-13700batch: iter_time=7.789e-05, forward_time=0.141, loss_ctc=71.471, loss_interctc_layer6=83.981, loss_interctc_layer12=70.486, loss_interctc_layer15=65.077, loss_interctc_layer21=73.571, loss=72.917, backward_time=0.210, grad_norm=75.147, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.181e-05, train_time=1.539 +[gpua006:0/64] 2024-02-09 14:33:11,733 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-09 14:33:30,485 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 14:33:34,192 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 14:33:34,192 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-09 14:33:34,196 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 14:39:48,741 (trainer:756) INFO: 24epoch:train:13701-13800batch: iter_time=3.362, forward_time=0.220, loss_ctc=82.671, loss_interctc_layer6=89.605, loss_interctc_layer12=74.489, loss_interctc_layer15=68.304, loss_interctc_layer21=85.547, loss=80.123, backward_time=0.224, grad_norm=66.050, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.180e-05, train_time=4.919 +[gpua006:0/64] 2024-02-09 14:41:23,563 (trainer:756) INFO: 24epoch:train:13801-13900batch: iter_time=8.216e-05, forward_time=0.143, loss_ctc=84.788, loss_interctc_layer6=81.875, loss_interctc_layer12=67.689, loss_interctc_layer15=61.856, loss_interctc_layer21=87.961, loss=76.834, backward_time=0.211, grad_norm=75.803, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.179e-05, train_time=0.948 +[gpua006:0/64] 2024-02-09 14:43:19,301 (trainer:756) INFO: 24epoch:train:13901-14000batch: iter_time=7.654e-05, forward_time=0.142, loss_ctc=70.167, loss_interctc_layer6=80.186, loss_interctc_layer12=66.328, loss_interctc_layer15=60.908, loss_interctc_layer21=72.504, loss=70.018, backward_time=0.212, grad_norm=77.740, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.178e-05, train_time=1.157 +[gpua006:0/64] 2024-02-09 14:45:07,507 (trainer:756) INFO: 24epoch:train:14001-14100batch: iter_time=7.767e-05, forward_time=0.142, loss_ctc=73.705, loss_interctc_layer6=85.602, loss_interctc_layer12=71.103, loss_interctc_layer15=65.169, loss_interctc_layer21=75.938, loss=74.303, backward_time=0.211, grad_norm=69.988, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.177e-05, train_time=1.082 +[gpua006:0/64] 2024-02-09 14:48:06,364 (trainer:756) INFO: 24epoch:train:14101-14200batch: iter_time=7.738e-05, forward_time=0.143, loss_ctc=68.760, loss_interctc_layer6=76.941, loss_interctc_layer12=63.976, loss_interctc_layer15=58.683, loss_interctc_layer21=70.991, loss=67.871, backward_time=0.208, grad_norm=77.721, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.175e-05, train_time=1.788 +[gpua006:0/64] 2024-02-09 14:49:58,954 (trainer:756) INFO: 24epoch:train:14201-14300batch: iter_time=7.659e-05, forward_time=0.142, loss_ctc=77.472, loss_interctc_layer6=82.500, loss_interctc_layer12=68.363, loss_interctc_layer15=62.494, loss_interctc_layer21=80.104, loss=74.187, backward_time=0.210, grad_norm=65.950, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.174e-05, train_time=1.126 +[gpua006:0/64] 2024-02-09 14:52:38,157 (trainer:756) INFO: 24epoch:train:14301-14400batch: iter_time=7.732e-05, forward_time=0.141, loss_ctc=63.759, loss_interctc_layer6=76.586, loss_interctc_layer12=63.299, loss_interctc_layer15=57.905, loss_interctc_layer21=65.920, loss=65.494, backward_time=0.210, grad_norm=53.212, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.173e-05, train_time=1.592 +[gpua006:0/64] 2024-02-09 14:55:38,703 (trainer:756) INFO: 24epoch:train:14401-14500batch: iter_time=8.201e-05, forward_time=0.142, loss_ctc=81.512, loss_interctc_layer6=84.619, loss_interctc_layer12=70.282, loss_interctc_layer15=64.409, loss_interctc_layer21=84.148, loss=76.994, backward_time=0.208, grad_norm=62.297, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.172e-05, train_time=1.805 +[gpua006:0/64] 2024-02-09 14:57:35,082 (trainer:756) INFO: 24epoch:train:14501-14600batch: iter_time=9.149e-05, forward_time=0.154, loss_ctc=78.896, loss_interctc_layer6=84.436, loss_interctc_layer12=70.333, loss_interctc_layer15=64.600, loss_interctc_layer21=81.609, loss=75.974, backward_time=0.210, grad_norm=80.625, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.171e-05, train_time=1.164 +[gpua006:0/64] 2024-02-09 15:00:23,971 (trainer:756) INFO: 24epoch:train:14601-14700batch: iter_time=7.992e-05, forward_time=0.202, loss_ctc=75.821, loss_interctc_layer6=91.132, loss_interctc_layer12=76.139, loss_interctc_layer15=70.295, loss_interctc_layer21=78.083, loss=78.294, backward_time=0.277, grad_norm=67.906, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.144, optim0_lr0=8.170e-05, train_time=1.688 +[gpua006:0/64] 2024-02-09 15:02:11,875 (trainer:756) INFO: 24epoch:train:14701-14800batch: iter_time=7.669e-05, forward_time=0.142, loss_ctc=81.898, loss_interctc_layer6=87.422, loss_interctc_layer12=72.290, loss_interctc_layer15=66.194, loss_interctc_layer21=84.418, loss=78.445, backward_time=0.211, grad_norm=66.770, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.169e-05, train_time=1.079 +[gpua006:0/64] 2024-02-09 15:04:52,156 (trainer:756) INFO: 24epoch:train:14801-14900batch: iter_time=7.756e-05, forward_time=0.142, loss_ctc=74.161, loss_interctc_layer6=87.928, loss_interctc_layer12=73.701, loss_interctc_layer15=68.043, loss_interctc_layer21=76.429, loss=76.052, backward_time=0.208, grad_norm=66.236, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.167e-05, train_time=1.603 +[gpua006:0/64] 2024-02-09 15:07:08,331 (trainer:756) INFO: 24epoch:train:14901-15000batch: iter_time=7.838e-05, forward_time=0.145, loss_ctc=73.703, loss_interctc_layer6=86.568, loss_interctc_layer12=72.162, loss_interctc_layer15=66.303, loss_interctc_layer21=76.054, loss=74.958, backward_time=0.211, grad_norm=166.864, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.166e-05, train_time=1.362 +[gpua006:0/64] 2024-02-09 15:38:29,506 (trainer:355) INFO: 24epoch results: [train] iter_time=0.239, forward_time=0.156, loss_ctc=78.659, loss_interctc_layer6=84.771, loss_interctc_layer12=70.535, loss_interctc_layer15=64.834, loss_interctc_layer21=81.206, loss=76.001, backward_time=0.216, grad_norm=71.798, clip=100.000, loss_scale=2.315e+31, optim_step_time=0.138, optim0_lr0=8.253e-05, train_time=1.570, time=6 hours, 32 minutes and 58.2 seconds, total_count=360000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=45.323, cer_ctc=0.210, loss_interctc_layer6=50.707, cer_interctc_layer6=0.226, loss_interctc_layer12=37.930, cer_interctc_layer12=0.160, loss_interctc_layer15=33.433, cer_interctc_layer15=0.134, loss_interctc_layer21=47.787, cer_interctc_layer21=0.223, loss=43.036, time=30 minutes and 57.19 seconds, total_count=112104, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-09 15:38:49,132 (trainer:410) INFO: The best model has been updated: valid.loss_ctc, valid.total_count +[gpua006:0/64] 2024-02-09 15:38:49,259 (trainer:289) INFO: 25/45epoch started. Estimated time to finish: 1 week, 8 hours and 12 minutes +[gpua006:0/64] 2024-02-09 15:38:49,585 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-09 15:39:07,521 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 15:39:10,856 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 15:39:10,857 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-09 15:39:10,860 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 15:45:26,193 (trainer:756) INFO: 25epoch:train:1-100batch: iter_time=2.548, forward_time=0.187, loss_ctc=80.339, loss_interctc_layer6=83.184, loss_interctc_layer12=68.941, loss_interctc_layer15=63.226, loss_interctc_layer21=82.787, loss=75.696, backward_time=0.220, grad_norm=78.751, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.165e-05, train_time=3.966 +[gpua006:0/64] 2024-02-09 15:47:04,283 (trainer:756) INFO: 25epoch:train:101-200batch: iter_time=9.513e-05, forward_time=0.141, loss_ctc=66.023, loss_interctc_layer6=77.326, loss_interctc_layer12=64.886, loss_interctc_layer15=59.845, loss_interctc_layer21=67.974, loss=67.211, backward_time=0.209, grad_norm=89.309, clip=100.000, loss_scale=1.704e+31, optim_step_time=0.136, optim0_lr0=8.164e-05, train_time=0.981 +[gpua006:0/64] 2024-02-09 15:49:34,597 (trainer:756) INFO: 25epoch:train:201-300batch: iter_time=9.361e-05, forward_time=0.144, loss_ctc=99.121, loss_interctc_layer6=110.748, loss_interctc_layer12=93.535, loss_interctc_layer15=86.399, loss_interctc_layer21=101.869, loss=98.334, backward_time=0.208, grad_norm=93.642, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.163e-05, train_time=1.503 +[gpua006:0/64] 2024-02-09 15:51:46,171 (trainer:756) INFO: 25epoch:train:301-400batch: iter_time=1.091e-04, forward_time=0.196, loss_ctc=75.082, loss_interctc_layer6=88.842, loss_interctc_layer12=75.173, loss_interctc_layer15=69.679, loss_interctc_layer21=77.218, loss=77.199, backward_time=0.217, grad_norm=64.578, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.162e-05, train_time=1.315 +[gpua006:0/64] 2024-02-09 15:53:53,249 (trainer:756) INFO: 25epoch:train:401-500batch: iter_time=1.032e-04, forward_time=0.156, loss_ctc=82.593, loss_interctc_layer6=87.442, loss_interctc_layer12=72.884, loss_interctc_layer15=67.212, loss_interctc_layer21=85.281, loss=79.083, backward_time=0.214, grad_norm=90.826, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.161e-05, train_time=1.271 +[gpua006:0/64] 2024-02-09 15:55:58,271 (trainer:756) INFO: 25epoch:train:501-600batch: iter_time=7.419e-05, forward_time=0.190, loss_ctc=80.183, loss_interctc_layer6=85.069, loss_interctc_layer12=70.955, loss_interctc_layer15=65.352, loss_interctc_layer21=82.911, loss=76.894, backward_time=0.222, grad_norm=102.833, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.159e-05, train_time=1.250 +[gpua006:0/64] 2024-02-09 15:58:40,392 (trainer:756) INFO: 25epoch:train:601-700batch: iter_time=8.019e-05, forward_time=0.177, loss_ctc=76.016, loss_interctc_layer6=83.414, loss_interctc_layer12=69.135, loss_interctc_layer15=63.174, loss_interctc_layer21=78.454, loss=74.039, backward_time=0.228, grad_norm=75.766, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.158e-05, train_time=1.621 +[gpua006:0/64] 2024-02-09 16:01:10,342 (trainer:756) INFO: 25epoch:train:701-800batch: iter_time=7.887e-05, forward_time=0.142, loss_ctc=76.683, loss_interctc_layer6=78.721, loss_interctc_layer12=65.430, loss_interctc_layer15=60.059, loss_interctc_layer21=79.089, loss=71.997, backward_time=0.205, grad_norm=60.847, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.157e-05, train_time=1.498 +[gpua006:0/64] 2024-02-09 16:04:07,127 (trainer:756) INFO: 25epoch:train:801-900batch: iter_time=8.424e-05, forward_time=0.147, loss_ctc=78.121, loss_interctc_layer6=84.723, loss_interctc_layer12=70.244, loss_interctc_layer15=64.390, loss_interctc_layer21=80.980, loss=75.692, backward_time=0.207, grad_norm=71.593, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.156e-05, train_time=1.768 +[gpua006:0/64] 2024-02-09 16:06:22,736 (trainer:756) INFO: 25epoch:train:901-1000batch: iter_time=8.066e-05, forward_time=0.154, loss_ctc=72.096, loss_interctc_layer6=83.944, loss_interctc_layer12=71.467, loss_interctc_layer15=66.563, loss_interctc_layer21=74.359, loss=73.686, backward_time=0.209, grad_norm=69.247, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=8.155e-05, train_time=1.356 +[gpua006:0/64] 2024-02-09 16:10:42,705 (trainer:756) INFO: 25epoch:train:1001-1100batch: iter_time=1.988e-04, forward_time=0.213, loss_ctc=78.637, loss_interctc_layer6=84.727, loss_interctc_layer12=70.981, loss_interctc_layer15=65.551, loss_interctc_layer21=81.107, loss=76.200, backward_time=1.065, grad_norm=89.516, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.145, optim0_lr0=8.154e-05, train_time=2.599 +[gpua006:0/64] 2024-02-09 16:14:09,664 (trainer:756) INFO: 25epoch:train:1101-1200batch: iter_time=8.945e-05, forward_time=0.162, loss_ctc=78.577, loss_interctc_layer6=87.584, loss_interctc_layer12=73.583, loss_interctc_layer15=67.731, loss_interctc_layer21=81.127, loss=77.720, backward_time=0.211, grad_norm=92.752, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.153e-05, train_time=2.064 +[gpua006:0/64] 2024-02-09 16:15:18,933 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 16:16:01,855 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-09 16:16:20,033 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 16:16:23,443 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 16:16:23,443 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-09 16:16:23,469 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 16:22:45,429 (trainer:756) INFO: 25epoch:train:1201-1300batch: iter_time=2.677, forward_time=0.161, loss_ctc=76.621, loss_interctc_layer6=78.976, loss_interctc_layer12=65.621, loss_interctc_layer15=60.134, loss_interctc_layer21=79.376, loss=72.146, backward_time=0.208, grad_norm=69.222, clip=100.000, loss_scale=1.280e+31, optim_step_time=0.136, optim0_lr0=8.152e-05, train_time=5.163 +[gpua006:0/64] 2024-02-09 16:24:21,804 (trainer:756) INFO: 25epoch:train:1301-1400batch: iter_time=8.244e-05, forward_time=0.143, loss_ctc=78.591, loss_interctc_layer6=81.974, loss_interctc_layer12=68.479, loss_interctc_layer15=63.237, loss_interctc_layer21=81.035, loss=74.663, backward_time=0.209, grad_norm=69.319, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.150e-05, train_time=0.963 +[gpua006:0/64] 2024-02-09 16:26:17,151 (trainer:756) INFO: 25epoch:train:1401-1500batch: iter_time=8.079e-05, forward_time=0.142, loss_ctc=94.753, loss_interctc_layer6=99.135, loss_interctc_layer12=83.669, loss_interctc_layer15=77.193, loss_interctc_layer21=97.847, loss=90.519, backward_time=0.210, grad_norm=78.528, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.149e-05, train_time=1.154 +[gpua006:0/64] 2024-02-09 16:27:45,288 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 16:28:49,578 (trainer:756) INFO: 25epoch:train:1501-1600batch: iter_time=4.397e-04, forward_time=0.215, loss_ctc=86.981, loss_interctc_layer6=94.767, loss_interctc_layer12=79.395, loss_interctc_layer15=73.315, loss_interctc_layer21=89.723, loss=84.836, backward_time=0.232, grad_norm=73.285, clip=100.000, loss_scale=8.092e+30, optim_step_time=0.145, optim0_lr0=8.148e-05, train_time=1.524 +[gpua006:0/64] 2024-02-09 16:31:13,418 (trainer:756) INFO: 25epoch:train:1601-1700batch: iter_time=8.440e-05, forward_time=0.142, loss_ctc=80.400, loss_interctc_layer6=80.487, loss_interctc_layer12=66.463, loss_interctc_layer15=60.987, loss_interctc_layer21=83.115, loss=74.290, backward_time=0.209, grad_norm=68.316, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=8.147e-05, train_time=1.438 +[gpua006:0/64] 2024-02-09 16:34:12,156 (trainer:756) INFO: 25epoch:train:1701-1800batch: iter_time=1.788e-04, forward_time=0.174, loss_ctc=89.052, loss_interctc_layer6=90.420, loss_interctc_layer12=75.626, loss_interctc_layer15=69.766, loss_interctc_layer21=91.984, loss=83.370, backward_time=0.220, grad_norm=76.942, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.142, optim0_lr0=8.146e-05, train_time=1.787 +[gpua006:0/64] 2024-02-09 16:36:18,226 (trainer:756) INFO: 25epoch:train:1801-1900batch: iter_time=7.916e-05, forward_time=0.144, loss_ctc=83.830, loss_interctc_layer6=85.752, loss_interctc_layer12=71.239, loss_interctc_layer15=65.361, loss_interctc_layer21=86.661, loss=78.569, backward_time=0.207, grad_norm=179.218, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.145e-05, train_time=1.261 +[gpua006:0/64] 2024-02-09 16:38:15,412 (trainer:756) INFO: 25epoch:train:1901-2000batch: iter_time=7.514e-05, forward_time=0.141, loss_ctc=83.666, loss_interctc_layer6=76.716, loss_interctc_layer12=63.386, loss_interctc_layer15=57.805, loss_interctc_layer21=86.797, loss=73.674, backward_time=0.209, grad_norm=58.842, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.144e-05, train_time=1.172 +[gpua006:0/64] 2024-02-09 16:40:39,990 (trainer:756) INFO: 25epoch:train:2001-2100batch: iter_time=7.791e-05, forward_time=0.142, loss_ctc=79.438, loss_interctc_layer6=81.456, loss_interctc_layer12=67.300, loss_interctc_layer15=61.638, loss_interctc_layer21=82.117, loss=74.390, backward_time=0.209, grad_norm=72.298, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.143e-05, train_time=1.446 +[gpua006:0/64] 2024-02-09 16:43:00,295 (trainer:756) INFO: 25epoch:train:2101-2200batch: iter_time=8.434e-05, forward_time=0.142, loss_ctc=79.986, loss_interctc_layer6=84.830, loss_interctc_layer12=70.765, loss_interctc_layer15=65.177, loss_interctc_layer21=82.767, loss=76.705, backward_time=0.207, grad_norm=98.474, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.141e-05, train_time=1.402 +[gpua006:0/64] 2024-02-09 16:44:59,099 (trainer:756) INFO: 25epoch:train:2201-2300batch: iter_time=7.945e-05, forward_time=0.142, loss_ctc=80.530, loss_interctc_layer6=84.308, loss_interctc_layer12=71.105, loss_interctc_layer15=65.887, loss_interctc_layer21=83.110, loss=76.988, backward_time=0.209, grad_norm=92.422, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.140e-05, train_time=1.188 +[gpua006:0/64] 2024-02-09 16:47:15,908 (trainer:756) INFO: 25epoch:train:2301-2400batch: iter_time=8.275e-05, forward_time=0.141, loss_ctc=84.173, loss_interctc_layer6=84.560, loss_interctc_layer12=70.778, loss_interctc_layer15=65.268, loss_interctc_layer21=87.073, loss=78.370, backward_time=0.210, grad_norm=67.800, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.139e-05, train_time=1.368 +[gpua006:0/64] 2024-02-09 16:49:16,298 (trainer:756) INFO: 25epoch:train:2401-2500batch: iter_time=7.874e-05, forward_time=0.178, loss_ctc=71.405, loss_interctc_layer6=80.939, loss_interctc_layer12=67.887, loss_interctc_layer15=62.573, loss_interctc_layer21=73.714, loss=71.303, backward_time=0.273, grad_norm=63.723, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.141, optim0_lr0=8.138e-05, train_time=1.204 +[gpua006:0/64] 2024-02-09 16:49:36,328 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-09 16:49:54,665 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 16:49:58,069 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 16:49:58,069 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-09 16:49:58,109 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 16:57:15,180 (trainer:756) INFO: 25epoch:train:2501-2600batch: iter_time=3.573, forward_time=0.174, loss_ctc=84.344, loss_interctc_layer6=82.545, loss_interctc_layer12=68.176, loss_interctc_layer15=62.397, loss_interctc_layer21=87.035, loss=76.899, backward_time=0.215, grad_norm=65.472, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=8.137e-05, train_time=4.789 +[gpua006:0/64] 2024-02-09 16:58:58,801 (trainer:756) INFO: 25epoch:train:2601-2700batch: iter_time=7.895e-05, forward_time=0.141, loss_ctc=67.737, loss_interctc_layer6=76.051, loss_interctc_layer12=63.471, loss_interctc_layer15=58.511, loss_interctc_layer21=69.727, loss=67.099, backward_time=0.210, grad_norm=90.402, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.136e-05, train_time=1.036 +[gpua006:0/64] 2024-02-09 17:01:14,184 (trainer:756) INFO: 25epoch:train:2701-2800batch: iter_time=8.258e-05, forward_time=0.142, loss_ctc=109.946, loss_interctc_layer6=110.026, loss_interctc_layer12=93.088, loss_interctc_layer15=85.720, loss_interctc_layer21=113.096, loss=102.375, backward_time=0.208, grad_norm=90.366, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.135e-05, train_time=1.354 +[gpua006:0/64] 2024-02-09 17:03:55,823 (trainer:756) INFO: 25epoch:train:2801-2900batch: iter_time=7.828e-05, forward_time=0.143, loss_ctc=76.769, loss_interctc_layer6=87.226, loss_interctc_layer12=73.453, loss_interctc_layer15=67.935, loss_interctc_layer21=79.033, loss=76.883, backward_time=0.207, grad_norm=69.480, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.134e-05, train_time=1.616 +[gpua006:0/64] 2024-02-09 17:06:05,380 (trainer:756) INFO: 25epoch:train:2901-3000batch: iter_time=7.927e-05, forward_time=0.142, loss_ctc=88.731, loss_interctc_layer6=87.556, loss_interctc_layer12=72.875, loss_interctc_layer15=67.004, loss_interctc_layer21=91.791, loss=81.591, backward_time=0.209, grad_norm=82.991, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.132e-05, train_time=1.295 +[gpua006:0/64] 2024-02-09 17:08:24,132 (trainer:756) INFO: 25epoch:train:3001-3100batch: iter_time=7.738e-05, forward_time=0.142, loss_ctc=85.326, loss_interctc_layer6=83.995, loss_interctc_layer12=69.787, loss_interctc_layer15=64.096, loss_interctc_layer21=88.096, loss=78.260, backward_time=0.209, grad_norm=68.783, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.131e-05, train_time=1.385 +[gpua006:0/64] 2024-02-09 17:10:34,891 (trainer:756) INFO: 25epoch:train:3101-3200batch: iter_time=9.073e-05, forward_time=0.141, loss_ctc=79.035, loss_interctc_layer6=82.410, loss_interctc_layer12=67.865, loss_interctc_layer15=62.028, loss_interctc_layer21=81.778, loss=74.623, backward_time=0.208, grad_norm=65.627, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.130e-05, train_time=1.309 +[gpua006:0/64] 2024-02-09 17:12:41,526 (trainer:756) INFO: 25epoch:train:3201-3300batch: iter_time=8.279e-05, forward_time=0.142, loss_ctc=79.942, loss_interctc_layer6=78.964, loss_interctc_layer12=65.359, loss_interctc_layer15=59.947, loss_interctc_layer21=82.773, loss=73.397, backward_time=0.206, grad_norm=68.289, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.135, optim0_lr0=8.129e-05, train_time=1.266 +[gpua006:0/64] 2024-02-09 17:15:08,855 (trainer:756) INFO: 25epoch:train:3301-3400batch: iter_time=8.172e-05, forward_time=0.213, loss_ctc=86.325, loss_interctc_layer6=83.973, loss_interctc_layer12=69.397, loss_interctc_layer15=63.578, loss_interctc_layer21=89.222, loss=78.499, backward_time=0.329, grad_norm=64.657, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.142, optim0_lr0=8.128e-05, train_time=1.473 +[gpua006:0/64] 2024-02-09 17:17:48,555 (trainer:756) INFO: 25epoch:train:3401-3500batch: iter_time=8.058e-05, forward_time=0.142, loss_ctc=74.796, loss_interctc_layer6=82.667, loss_interctc_layer12=69.891, loss_interctc_layer15=64.907, loss_interctc_layer21=77.358, loss=73.924, backward_time=0.207, grad_norm=87.219, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=8.127e-05, train_time=1.597 +[gpua006:0/64] 2024-02-09 17:19:44,599 (trainer:756) INFO: 25epoch:train:3501-3600batch: iter_time=8.046e-05, forward_time=0.141, loss_ctc=79.772, loss_interctc_layer6=82.969, loss_interctc_layer12=69.145, loss_interctc_layer15=63.561, loss_interctc_layer21=82.506, loss=75.591, backward_time=0.209, grad_norm=92.592, clip=100.000, loss_scale=7.099e+30, optim_step_time=0.136, optim0_lr0=8.126e-05, train_time=1.160 +[gpua006:0/64] 2024-02-09 17:21:56,265 (trainer:756) INFO: 25epoch:train:3601-3700batch: iter_time=8.079e-05, forward_time=0.143, loss_ctc=82.769, loss_interctc_layer6=86.986, loss_interctc_layer12=72.997, loss_interctc_layer15=67.362, loss_interctc_layer21=85.436, loss=79.110, backward_time=0.210, grad_norm=122.096, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.125e-05, train_time=1.316 +[gpua006:0/64] 2024-02-09 17:23:12,608 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-09 17:23:31,161 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 17:23:34,580 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 17:23:34,580 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-09 17:23:34,583 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 17:29:42,798 (trainer:756) INFO: 25epoch:train:3701-3800batch: iter_time=3.201, forward_time=0.141, loss_ctc=76.647, loss_interctc_layer6=78.434, loss_interctc_layer12=65.199, loss_interctc_layer15=59.699, loss_interctc_layer21=79.369, loss=71.870, backward_time=0.209, grad_norm=70.345, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.124e-05, train_time=4.665 +[gpua006:0/64] 2024-02-09 17:31:21,813 (trainer:756) INFO: 25epoch:train:3801-3900batch: iter_time=7.886e-05, forward_time=0.141, loss_ctc=68.951, loss_interctc_layer6=80.265, loss_interctc_layer12=66.962, loss_interctc_layer15=61.826, loss_interctc_layer21=71.026, loss=69.806, backward_time=0.209, grad_norm=73.364, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.122e-05, train_time=0.990 +[gpua006:0/64] 2024-02-09 17:33:19,052 (trainer:756) INFO: 25epoch:train:3901-4000batch: iter_time=7.893e-05, forward_time=0.142, loss_ctc=88.478, loss_interctc_layer6=98.521, loss_interctc_layer12=82.957, loss_interctc_layer15=76.590, loss_interctc_layer21=90.636, loss=87.437, backward_time=0.209, grad_norm=91.221, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.121e-05, train_time=1.171 +[gpua006:0/64] 2024-02-09 17:35:31,049 (trainer:756) INFO: 25epoch:train:4001-4100batch: iter_time=9.010e-05, forward_time=0.143, loss_ctc=79.556, loss_interctc_layer6=94.591, loss_interctc_layer12=79.445, loss_interctc_layer15=73.186, loss_interctc_layer21=81.943, loss=81.744, backward_time=0.210, grad_norm=213.918, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.120e-05, train_time=1.320 +[gpua006:0/64] 2024-02-09 17:38:07,769 (trainer:756) INFO: 25epoch:train:4101-4200batch: iter_time=8.304e-05, forward_time=0.266, loss_ctc=74.973, loss_interctc_layer6=80.445, loss_interctc_layer12=66.478, loss_interctc_layer15=60.982, loss_interctc_layer21=77.465, loss=72.068, backward_time=0.240, grad_norm=60.036, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.146, optim0_lr0=8.119e-05, train_time=1.567 +[gpua006:0/64] 2024-02-09 17:40:40,322 (trainer:756) INFO: 25epoch:train:4201-4300batch: iter_time=8.628e-05, forward_time=0.143, loss_ctc=81.060, loss_interctc_layer6=88.810, loss_interctc_layer12=74.045, loss_interctc_layer15=68.237, loss_interctc_layer21=83.842, loss=79.199, backward_time=0.209, grad_norm=69.983, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=8.118e-05, train_time=1.525 +[gpua006:0/64] 2024-02-09 17:43:23,161 (trainer:756) INFO: 25epoch:train:4301-4400batch: iter_time=7.909e-05, forward_time=0.165, loss_ctc=78.766, loss_interctc_layer6=84.797, loss_interctc_layer12=70.387, loss_interctc_layer15=64.482, loss_interctc_layer21=81.542, loss=75.995, backward_time=0.208, grad_norm=212.778, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=8.117e-05, train_time=1.628 +[gpua006:0/64] 2024-02-09 17:45:00,077 (trainer:756) INFO: 25epoch:train:4401-4500batch: iter_time=8.253e-05, forward_time=0.141, loss_ctc=77.089, loss_interctc_layer6=75.770, loss_interctc_layer12=62.447, loss_interctc_layer15=56.939, loss_interctc_layer21=79.930, loss=70.435, backward_time=0.210, grad_norm=64.946, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=8.116e-05, train_time=0.969 +[gpua006:0/64] 2024-02-09 17:46:54,279 (trainer:756) INFO: 25epoch:train:4501-4600batch: iter_time=8.316e-05, forward_time=0.141, loss_ctc=72.385, loss_interctc_layer6=80.586, loss_interctc_layer12=66.463, loss_interctc_layer15=60.778, loss_interctc_layer21=74.974, loss=71.037, backward_time=0.209, grad_norm=70.755, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=8.115e-05, train_time=1.142 +[gpua006:0/64] 2024-02-09 17:49:47,879 (trainer:756) INFO: 25epoch:train:4601-4700batch: iter_time=8.299e-05, forward_time=0.141, loss_ctc=75.334, loss_interctc_layer6=84.596, loss_interctc_layer12=70.519, loss_interctc_layer15=64.957, loss_interctc_layer21=77.842, loss=74.650, backward_time=0.207, grad_norm=75.211, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=8.113e-05, train_time=1.736 +[gpua006:0/64] 2024-02-09 17:51:48,719 (trainer:756) INFO: 25epoch:train:4701-4800batch: iter_time=8.369e-05, forward_time=0.142, loss_ctc=76.454, loss_interctc_layer6=83.325, loss_interctc_layer12=70.221, loss_interctc_layer15=65.009, loss_interctc_layer21=78.842, loss=74.770, backward_time=0.210, grad_norm=74.945, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.112e-05, train_time=1.208 +[gpua006:0/64] 2024-02-09 17:54:21,946 (trainer:756) INFO: 25epoch:train:4801-4900batch: iter_time=8.172e-05, forward_time=0.142, loss_ctc=76.741, loss_interctc_layer6=83.178, loss_interctc_layer12=69.337, loss_interctc_layer15=63.968, loss_interctc_layer21=79.375, loss=74.520, backward_time=0.209, grad_norm=62.953, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.111e-05, train_time=1.531 +[gpua006:0/64] 2024-02-09 17:56:42,888 (trainer:756) INFO: 25epoch:train:4901-5000batch: iter_time=7.698e-05, forward_time=0.143, loss_ctc=69.065, loss_interctc_layer6=81.276, loss_interctc_layer12=68.044, loss_interctc_layer15=62.664, loss_interctc_layer21=71.350, loss=70.480, backward_time=0.211, grad_norm=87.500, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=8.110e-05, train_time=1.410 +[gpua006:0/64] 2024-02-09 17:57:02,917 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-09 17:57:21,268 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 17:57:24,659 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 17:57:24,659 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-09 17:57:24,668 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 18:04:41,458 (trainer:756) INFO: 25epoch:train:5001-5100batch: iter_time=3.409, forward_time=0.194, loss_ctc=83.254, loss_interctc_layer6=81.298, loss_interctc_layer12=67.132, loss_interctc_layer15=61.358, loss_interctc_layer21=86.113, loss=75.831, backward_time=0.218, grad_norm=72.625, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=8.109e-05, train_time=4.785 +[gpua006:0/64] 2024-02-09 18:06:43,650 (trainer:756) INFO: 25epoch:train:5101-5200batch: iter_time=8.106e-05, forward_time=0.141, loss_ctc=66.509, loss_interctc_layer6=74.464, loss_interctc_layer12=62.110, loss_interctc_layer15=57.201, loss_interctc_layer21=68.566, loss=65.770, backward_time=0.209, grad_norm=71.127, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.108e-05, train_time=1.222 +[gpua006:0/64] 2024-02-09 18:09:20,553 (trainer:756) INFO: 25epoch:train:5201-5300batch: iter_time=7.933e-05, forward_time=0.143, loss_ctc=109.848, loss_interctc_layer6=110.157, loss_interctc_layer12=92.155, loss_interctc_layer15=85.126, loss_interctc_layer21=113.305, loss=102.118, backward_time=0.209, grad_norm=102.376, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.107e-05, train_time=1.569 +[gpua006:0/64] 2024-02-09 18:11:21,601 (trainer:756) INFO: 25epoch:train:5301-5400batch: iter_time=7.692e-05, forward_time=0.143, loss_ctc=77.204, loss_interctc_layer6=86.907, loss_interctc_layer12=73.073, loss_interctc_layer15=67.621, loss_interctc_layer21=79.497, loss=76.861, backward_time=0.209, grad_norm=119.439, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.106e-05, train_time=1.210 +[gpua006:0/64] 2024-02-09 18:13:17,714 (trainer:756) INFO: 25epoch:train:5401-5500batch: iter_time=7.875e-05, forward_time=0.142, loss_ctc=87.264, loss_interctc_layer6=87.021, loss_interctc_layer12=72.190, loss_interctc_layer15=66.267, loss_interctc_layer21=90.150, loss=80.578, backward_time=0.210, grad_norm=67.498, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=8.105e-05, train_time=1.161 +[gpua006:0/64] 2024-02-09 18:16:27,247 (trainer:756) INFO: 25epoch:train:5501-5600batch: iter_time=8.214e-05, forward_time=0.142, loss_ctc=85.236, loss_interctc_layer6=83.931, loss_interctc_layer12=69.790, loss_interctc_layer15=64.070, loss_interctc_layer21=88.084, loss=78.222, backward_time=0.209, grad_norm=63.910, clip=100.000, loss_scale=1.420e+31, optim_step_time=0.136, optim0_lr0=8.103e-05, train_time=1.895 +[gpua006:0/64] 2024-02-09 18:18:36,013 (trainer:756) INFO: 25epoch:train:5601-5700batch: iter_time=8.335e-05, forward_time=0.141, loss_ctc=79.066, loss_interctc_layer6=82.008, loss_interctc_layer12=67.771, loss_interctc_layer15=61.930, loss_interctc_layer21=81.862, loss=74.528, backward_time=0.210, grad_norm=70.066, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.102e-05, train_time=1.287 +[gpua006:0/64] 2024-02-09 18:21:25,491 (trainer:756) INFO: 25epoch:train:5701-5800batch: iter_time=8.421e-05, forward_time=0.141, loss_ctc=80.267, loss_interctc_layer6=78.334, loss_interctc_layer12=64.877, loss_interctc_layer15=59.380, loss_interctc_layer21=83.076, loss=73.187, backward_time=0.207, grad_norm=78.549, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.101e-05, train_time=1.693 +[gpua006:0/64] 2024-02-09 18:24:25,673 (trainer:756) INFO: 25epoch:train:5801-5900batch: iter_time=8.047e-05, forward_time=0.272, loss_ctc=84.310, loss_interctc_layer6=83.174, loss_interctc_layer12=68.720, loss_interctc_layer15=62.788, loss_interctc_layer21=87.258, loss=77.250, backward_time=0.229, grad_norm=86.657, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.166, optim0_lr0=8.100e-05, train_time=1.802 +[gpua006:0/64] 2024-02-09 18:26:59,859 (trainer:756) INFO: 25epoch:train:5901-6000batch: iter_time=7.948e-05, forward_time=0.142, loss_ctc=74.470, loss_interctc_layer6=82.457, loss_interctc_layer12=69.759, loss_interctc_layer15=64.641, loss_interctc_layer21=76.874, loss=73.640, backward_time=0.207, grad_norm=71.080, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.099e-05, train_time=1.542 +[gpua006:0/64] 2024-02-09 18:29:03,503 (trainer:756) INFO: 25epoch:train:6001-6100batch: iter_time=7.658e-05, forward_time=0.152, loss_ctc=80.334, loss_interctc_layer6=82.647, loss_interctc_layer12=68.623, loss_interctc_layer15=63.081, loss_interctc_layer21=83.183, loss=75.573, backward_time=0.227, grad_norm=72.171, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.098e-05, train_time=1.236 +[gpua006:0/64] 2024-02-09 18:30:46,100 (trainer:756) INFO: 25epoch:train:6101-6200batch: iter_time=7.992e-05, forward_time=0.142, loss_ctc=81.327, loss_interctc_layer6=86.640, loss_interctc_layer12=72.464, loss_interctc_layer15=66.756, loss_interctc_layer21=84.039, loss=78.245, backward_time=0.209, grad_norm=105.937, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=8.097e-05, train_time=1.026 +[gpua006:0/64] 2024-02-09 18:32:01,173 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-09 18:32:19,866 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 18:32:23,272 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 18:32:23,272 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-09 18:32:23,284 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 18:38:09,129 (trainer:756) INFO: 25epoch:train:6201-6300batch: iter_time=2.928, forward_time=0.142, loss_ctc=76.502, loss_interctc_layer6=77.984, loss_interctc_layer12=64.635, loss_interctc_layer15=59.078, loss_interctc_layer21=79.144, loss=71.469, backward_time=0.209, grad_norm=58.555, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.096e-05, train_time=4.430 +[gpua006:0/64] 2024-02-09 18:40:03,260 (trainer:756) INFO: 25epoch:train:6301-6400batch: iter_time=8.216e-05, forward_time=0.143, loss_ctc=68.833, loss_interctc_layer6=80.255, loss_interctc_layer12=67.015, loss_interctc_layer15=61.754, loss_interctc_layer21=70.906, loss=69.753, backward_time=0.208, grad_norm=67.153, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.095e-05, train_time=1.141 +[gpua006:0/64] 2024-02-09 18:41:55,219 (trainer:756) INFO: 25epoch:train:6401-6500batch: iter_time=7.990e-05, forward_time=0.141, loss_ctc=87.699, loss_interctc_layer6=98.604, loss_interctc_layer12=82.597, loss_interctc_layer15=76.545, loss_interctc_layer21=90.365, loss=87.162, backward_time=0.208, grad_norm=93.963, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.094e-05, train_time=1.119 +[gpua006:0/64] 2024-02-09 18:45:11,458 (trainer:756) INFO: 25epoch:train:6501-6600batch: iter_time=8.539e-05, forward_time=0.142, loss_ctc=79.390, loss_interctc_layer6=93.775, loss_interctc_layer12=78.464, loss_interctc_layer15=72.373, loss_interctc_layer21=81.823, loss=81.165, backward_time=0.207, grad_norm=86.543, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.092e-05, train_time=1.962 +[gpua006:0/64] 2024-02-09 18:47:34,597 (trainer:756) INFO: 25epoch:train:6601-6700batch: iter_time=8.132e-05, forward_time=0.202, loss_ctc=74.807, loss_interctc_layer6=80.604, loss_interctc_layer12=66.477, loss_interctc_layer15=60.942, loss_interctc_layer21=77.542, loss=72.074, backward_time=0.293, grad_norm=60.449, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=8.091e-05, train_time=1.430 +[gpua006:0/64] 2024-02-09 18:50:22,637 (trainer:756) INFO: 25epoch:train:6701-6800batch: iter_time=8.047e-05, forward_time=0.143, loss_ctc=81.446, loss_interctc_layer6=89.050, loss_interctc_layer12=74.236, loss_interctc_layer15=68.319, loss_interctc_layer21=84.156, loss=79.441, backward_time=0.208, grad_norm=87.277, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=8.090e-05, train_time=1.682 +[gpua006:0/64] 2024-02-09 18:52:34,809 (trainer:756) INFO: 25epoch:train:6801-6900batch: iter_time=8.152e-05, forward_time=0.141, loss_ctc=78.099, loss_interctc_layer6=85.015, loss_interctc_layer12=70.458, loss_interctc_layer15=64.545, loss_interctc_layer21=80.789, loss=75.781, backward_time=0.208, grad_norm=67.225, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=8.089e-05, train_time=1.322 +[gpua006:0/64] 2024-02-09 18:54:38,529 (trainer:756) INFO: 25epoch:train:6901-7000batch: iter_time=7.956e-05, forward_time=0.141, loss_ctc=77.099, loss_interctc_layer6=75.523, loss_interctc_layer12=62.068, loss_interctc_layer15=56.519, loss_interctc_layer21=80.185, loss=70.279, backward_time=0.209, grad_norm=97.662, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.088e-05, train_time=1.237 +[gpua006:0/64] 2024-02-09 18:58:43,822 (trainer:756) INFO: 25epoch:train:7001-7100batch: iter_time=8.830e-05, forward_time=0.142, loss_ctc=72.631, loss_interctc_layer6=81.554, loss_interctc_layer12=67.159, loss_interctc_layer15=61.509, loss_interctc_layer21=74.962, loss=71.563, backward_time=0.206, grad_norm=64.216, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.087e-05, train_time=2.453 +[gpua006:0/64] 2024-02-09 19:00:40,074 (trainer:756) INFO: 25epoch:train:7101-7200batch: iter_time=8.438e-05, forward_time=0.142, loss_ctc=74.226, loss_interctc_layer6=84.365, loss_interctc_layer12=70.099, loss_interctc_layer15=64.559, loss_interctc_layer21=76.632, loss=73.976, backward_time=0.211, grad_norm=94.557, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.086e-05, train_time=1.162 +[gpua006:0/64] 2024-02-09 19:02:41,674 (trainer:756) INFO: 25epoch:train:7201-7300batch: iter_time=8.530e-05, forward_time=0.141, loss_ctc=76.879, loss_interctc_layer6=83.659, loss_interctc_layer12=70.633, loss_interctc_layer15=65.327, loss_interctc_layer21=79.262, loss=75.152, backward_time=0.209, grad_norm=72.305, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.085e-05, train_time=1.216 +[gpua006:0/64] 2024-02-09 19:05:10,400 (trainer:756) INFO: 25epoch:train:7301-7400batch: iter_time=8.179e-05, forward_time=0.141, loss_ctc=76.722, loss_interctc_layer6=83.134, loss_interctc_layer12=68.971, loss_interctc_layer15=63.487, loss_interctc_layer21=79.623, loss=74.387, backward_time=0.208, grad_norm=85.566, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=8.084e-05, train_time=1.487 +[gpua006:0/64] 2024-02-09 19:07:15,113 (trainer:756) INFO: 25epoch:train:7401-7500batch: iter_time=8.396e-05, forward_time=0.142, loss_ctc=68.926, loss_interctc_layer6=80.631, loss_interctc_layer12=67.314, loss_interctc_layer15=61.845, loss_interctc_layer21=71.123, loss=69.968, backward_time=0.209, grad_norm=61.175, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.083e-05, train_time=1.247 +[gpua006:0/64] 2024-02-09 19:07:35,141 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-09 19:07:53,773 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 19:07:57,178 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 19:07:57,178 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-09 19:07:57,264 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 19:15:17,433 (trainer:756) INFO: 25epoch:train:7501-7600batch: iter_time=3.604, forward_time=0.186, loss_ctc=78.495, loss_interctc_layer6=81.258, loss_interctc_layer12=67.149, loss_interctc_layer15=61.437, loss_interctc_layer21=81.021, loss=73.872, backward_time=0.217, grad_norm=171.639, clip=100.000, loss_scale=2.840e+31, optim_step_time=0.138, optim0_lr0=8.081e-05, train_time=4.822 +[gpua006:0/64] 2024-02-09 19:17:02,360 (trainer:756) INFO: 25epoch:train:7601-7700batch: iter_time=7.882e-05, forward_time=0.141, loss_ctc=63.154, loss_interctc_layer6=74.322, loss_interctc_layer12=61.903, loss_interctc_layer15=56.994, loss_interctc_layer21=65.182, loss=64.311, backward_time=0.209, grad_norm=77.674, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.080e-05, train_time=1.050 +[gpua006:0/64] 2024-02-09 19:19:53,558 (trainer:756) INFO: 25epoch:train:7701-7800batch: iter_time=8.246e-05, forward_time=0.143, loss_ctc=97.089, loss_interctc_layer6=109.771, loss_interctc_layer12=91.798, loss_interctc_layer15=84.748, loss_interctc_layer21=99.814, loss=96.644, backward_time=0.210, grad_norm=109.765, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.079e-05, train_time=1.712 +[gpua006:0/64] 2024-02-09 19:22:07,370 (trainer:756) INFO: 25epoch:train:7801-7900batch: iter_time=9.340e-05, forward_time=0.141, loss_ctc=71.961, loss_interctc_layer6=85.106, loss_interctc_layer12=71.146, loss_interctc_layer15=65.653, loss_interctc_layer21=74.053, loss=73.584, backward_time=0.209, grad_norm=71.001, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.078e-05, train_time=1.338 +[gpua006:0/64] 2024-02-09 19:24:29,519 (trainer:756) INFO: 25epoch:train:7901-8000batch: iter_time=8.513e-05, forward_time=0.142, loss_ctc=80.649, loss_interctc_layer6=86.153, loss_interctc_layer12=71.289, loss_interctc_layer15=65.413, loss_interctc_layer21=83.446, loss=77.390, backward_time=0.210, grad_norm=65.512, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.077e-05, train_time=1.421 +[gpua006:0/64] 2024-02-09 19:26:57,869 (trainer:756) INFO: 25epoch:train:8001-8100batch: iter_time=8.609e-05, forward_time=0.141, loss_ctc=79.602, loss_interctc_layer6=83.616, loss_interctc_layer12=69.356, loss_interctc_layer15=63.636, loss_interctc_layer21=82.465, loss=75.735, backward_time=0.209, grad_norm=72.143, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.076e-05, train_time=1.483 +[gpua006:0/64] 2024-02-09 19:30:06,739 (trainer:756) INFO: 25epoch:train:8101-8200batch: iter_time=8.510e-05, forward_time=0.142, loss_ctc=73.992, loss_interctc_layer6=81.736, loss_interctc_layer12=67.183, loss_interctc_layer15=61.153, loss_interctc_layer21=76.617, loss=72.136, backward_time=0.208, grad_norm=81.996, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.075e-05, train_time=1.888 +[gpua006:0/64] 2024-02-09 19:32:14,424 (trainer:756) INFO: 25epoch:train:8201-8300batch: iter_time=8.275e-05, forward_time=0.141, loss_ctc=75.478, loss_interctc_layer6=77.868, loss_interctc_layer12=64.332, loss_interctc_layer15=58.819, loss_interctc_layer21=78.114, loss=70.922, backward_time=0.208, grad_norm=57.436, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.074e-05, train_time=1.277 +[gpua006:0/64] 2024-02-09 19:34:19,971 (trainer:756) INFO: 25epoch:train:8301-8400batch: iter_time=8.365e-05, forward_time=0.180, loss_ctc=76.730, loss_interctc_layer6=83.071, loss_interctc_layer12=68.463, loss_interctc_layer15=62.587, loss_interctc_layer21=79.420, loss=74.054, backward_time=0.245, grad_norm=63.658, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=8.073e-05, train_time=1.255 +[gpua006:0/64] 2024-02-09 19:36:49,370 (trainer:756) INFO: 25epoch:train:8401-8500batch: iter_time=8.724e-05, forward_time=0.172, loss_ctc=69.908, loss_interctc_layer6=82.183, loss_interctc_layer12=69.361, loss_interctc_layer15=64.358, loss_interctc_layer21=72.391, loss=71.640, backward_time=0.233, grad_norm=76.646, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.144, optim0_lr0=8.072e-05, train_time=1.493 +[gpua006:0/64] 2024-02-09 19:39:21,431 (trainer:756) INFO: 25epoch:train:8501-8600batch: iter_time=9.596e-05, forward_time=0.142, loss_ctc=76.086, loss_interctc_layer6=81.589, loss_interctc_layer12=67.709, loss_interctc_layer15=62.295, loss_interctc_layer21=78.698, loss=73.275, backward_time=0.207, grad_norm=70.588, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.070e-05, train_time=1.520 +[gpua006:0/64] 2024-02-09 19:41:39,202 (trainer:756) INFO: 25epoch:train:8601-8700batch: iter_time=8.792e-05, forward_time=0.142, loss_ctc=77.027, loss_interctc_layer6=86.601, loss_interctc_layer12=72.372, loss_interctc_layer15=66.701, loss_interctc_layer21=79.683, loss=76.477, backward_time=0.209, grad_norm=80.173, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.069e-05, train_time=1.379 +[gpua006:0/64] 2024-02-09 19:43:11,537 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-09 19:43:30,104 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 19:43:33,766 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 19:43:33,766 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-09 19:43:33,769 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 19:49:17,441 (trainer:756) INFO: 25epoch:train:8701-8800batch: iter_time=3.231, forward_time=0.142, loss_ctc=75.325, loss_interctc_layer6=77.778, loss_interctc_layer12=64.345, loss_interctc_layer15=58.864, loss_interctc_layer21=77.996, loss=70.862, backward_time=0.209, grad_norm=62.958, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.068e-05, train_time=4.582 +[gpua006:0/64] 2024-02-09 19:50:54,172 (trainer:756) INFO: 25epoch:train:8801-8900batch: iter_time=8.112e-05, forward_time=0.141, loss_ctc=67.753, loss_interctc_layer6=79.643, loss_interctc_layer12=66.458, loss_interctc_layer15=61.150, loss_interctc_layer21=69.723, loss=68.945, backward_time=0.210, grad_norm=71.615, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.067e-05, train_time=0.967 +[gpua006:0/64] 2024-02-09 19:53:18,065 (trainer:756) INFO: 25epoch:train:8901-9000batch: iter_time=7.867e-05, forward_time=0.142, loss_ctc=85.559, loss_interctc_layer6=97.529, loss_interctc_layer12=81.511, loss_interctc_layer15=75.285, loss_interctc_layer21=88.137, loss=85.604, backward_time=0.208, grad_norm=108.648, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.066e-05, train_time=1.439 +[gpua006:0/64] 2024-02-09 19:55:25,833 (trainer:756) INFO: 25epoch:train:9001-9100batch: iter_time=8.377e-05, forward_time=0.142, loss_ctc=78.761, loss_interctc_layer6=93.366, loss_interctc_layer12=77.947, loss_interctc_layer15=71.829, loss_interctc_layer21=81.144, loss=80.609, backward_time=0.207, grad_norm=102.355, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.065e-05, train_time=1.277 +[gpua006:0/64] 2024-02-09 19:57:59,211 (trainer:756) INFO: 25epoch:train:9101-9200batch: iter_time=8.720e-05, forward_time=0.180, loss_ctc=74.942, loss_interctc_layer6=80.560, loss_interctc_layer12=66.653, loss_interctc_layer15=60.992, loss_interctc_layer21=77.584, loss=72.146, backward_time=0.231, grad_norm=67.304, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.064e-05, train_time=1.534 +[gpua006:0/64] 2024-02-09 20:00:33,090 (trainer:756) INFO: 25epoch:train:9201-9300batch: iter_time=8.145e-05, forward_time=0.164, loss_ctc=80.862, loss_interctc_layer6=89.021, loss_interctc_layer12=74.077, loss_interctc_layer15=68.216, loss_interctc_layer21=83.658, loss=79.167, backward_time=0.224, grad_norm=63.964, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.063e-05, train_time=1.538 +[gpua006:0/64] 2024-02-09 20:03:04,156 (trainer:756) INFO: 25epoch:train:9301-9400batch: iter_time=7.890e-05, forward_time=0.163, loss_ctc=77.158, loss_interctc_layer6=84.117, loss_interctc_layer12=69.680, loss_interctc_layer15=63.874, loss_interctc_layer21=79.852, loss=74.936, backward_time=0.207, grad_norm=73.487, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.062e-05, train_time=1.512 +[gpua006:0/64] 2024-02-09 20:03:19,451 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 20:05:09,876 (trainer:756) INFO: 25epoch:train:9401-9500batch: iter_time=8.328e-05, forward_time=0.140, loss_ctc=76.479, loss_interctc_layer6=75.380, loss_interctc_layer12=61.909, loss_interctc_layer15=56.365, loss_interctc_layer21=79.317, loss=69.890, backward_time=0.207, grad_norm=56.245, clip=100.000, loss_scale=2.274e+31, optim_step_time=0.136, optim0_lr0=8.061e-05, train_time=1.257 +[gpua006:0/64] 2024-02-09 20:06:58,860 (trainer:756) INFO: 25epoch:train:9501-9600batch: iter_time=8.331e-05, forward_time=0.141, loss_ctc=71.546, loss_interctc_layer6=79.575, loss_interctc_layer12=65.668, loss_interctc_layer15=60.107, loss_interctc_layer21=74.005, loss=70.180, backward_time=0.208, grad_norm=60.096, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.060e-05, train_time=1.090 +[gpua006:0/64] 2024-02-09 20:09:30,982 (trainer:756) INFO: 25epoch:train:9601-9700batch: iter_time=8.444e-05, forward_time=0.141, loss_ctc=74.241, loss_interctc_layer6=83.767, loss_interctc_layer12=69.580, loss_interctc_layer15=63.861, loss_interctc_layer21=76.701, loss=73.630, backward_time=0.207, grad_norm=61.434, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.058e-05, train_time=1.521 +[gpua006:0/64] 2024-02-09 20:12:15,078 (trainer:756) INFO: 25epoch:train:9701-9800batch: iter_time=8.325e-05, forward_time=0.141, loss_ctc=76.428, loss_interctc_layer6=83.493, loss_interctc_layer12=70.348, loss_interctc_layer15=64.980, loss_interctc_layer21=78.911, loss=74.832, backward_time=0.207, grad_norm=68.112, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.057e-05, train_time=1.641 +[gpua006:0/64] 2024-02-09 20:14:23,308 (trainer:756) INFO: 25epoch:train:9801-9900batch: iter_time=8.107e-05, forward_time=0.226, loss_ctc=76.004, loss_interctc_layer6=82.585, loss_interctc_layer12=68.537, loss_interctc_layer15=63.119, loss_interctc_layer21=78.763, loss=73.802, backward_time=0.223, grad_norm=64.339, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.056e-05, train_time=1.282 +[gpua006:0/64] 2024-02-09 20:16:20,780 (trainer:756) INFO: 25epoch:train:9901-10000batch: iter_time=7.740e-05, forward_time=0.141, loss_ctc=68.015, loss_interctc_layer6=79.788, loss_interctc_layer12=66.595, loss_interctc_layer15=61.220, loss_interctc_layer21=70.311, loss=69.186, backward_time=0.209, grad_norm=57.462, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.055e-05, train_time=1.175 +[gpua006:0/64] 2024-02-09 20:16:40,810 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-09 20:16:59,338 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 20:17:02,962 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 20:17:02,962 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-09 20:17:02,965 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 20:23:24,677 (trainer:756) INFO: 25epoch:train:10001-10100batch: iter_time=3.139, forward_time=0.179, loss_ctc=84.498, loss_interctc_layer6=81.863, loss_interctc_layer12=67.500, loss_interctc_layer15=61.761, loss_interctc_layer21=87.309, loss=76.586, backward_time=0.216, grad_norm=67.255, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.054e-05, train_time=4.238 +[gpua006:0/64] 2024-02-09 20:25:44,540 (trainer:756) INFO: 25epoch:train:10101-10200batch: iter_time=8.032e-05, forward_time=0.141, loss_ctc=66.299, loss_interctc_layer6=74.288, loss_interctc_layer12=61.737, loss_interctc_layer15=56.830, loss_interctc_layer21=68.380, loss=65.507, backward_time=0.209, grad_norm=59.426, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.053e-05, train_time=1.399 +[gpua006:0/64] 2024-02-09 20:27:38,691 (trainer:756) INFO: 25epoch:train:10201-10300batch: iter_time=8.089e-05, forward_time=0.166, loss_ctc=108.981, loss_interctc_layer6=109.118, loss_interctc_layer12=90.807, loss_interctc_layer15=83.700, loss_interctc_layer21=112.563, loss=101.034, backward_time=0.234, grad_norm=121.826, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=8.052e-05, train_time=1.141 +[gpua006:0/64] 2024-02-09 20:30:06,408 (trainer:756) INFO: 25epoch:train:10301-10400batch: iter_time=8.485e-05, forward_time=0.142, loss_ctc=75.387, loss_interctc_layer6=85.121, loss_interctc_layer12=70.955, loss_interctc_layer15=65.629, loss_interctc_layer21=77.781, loss=74.974, backward_time=0.209, grad_norm=97.792, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.051e-05, train_time=1.477 +[gpua006:0/64] 2024-02-09 20:32:14,398 (trainer:756) INFO: 25epoch:train:10401-10500batch: iter_time=8.170e-05, forward_time=0.142, loss_ctc=86.123, loss_interctc_layer6=86.052, loss_interctc_layer12=71.221, loss_interctc_layer15=65.431, loss_interctc_layer21=89.172, loss=79.600, backward_time=0.210, grad_norm=71.292, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.050e-05, train_time=1.280 +[gpua006:0/64] 2024-02-09 20:34:31,035 (trainer:756) INFO: 25epoch:train:10501-10600batch: iter_time=7.969e-05, forward_time=0.147, loss_ctc=84.704, loss_interctc_layer6=83.054, loss_interctc_layer12=68.957, loss_interctc_layer15=63.244, loss_interctc_layer21=87.607, loss=77.513, backward_time=0.210, grad_norm=70.762, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.049e-05, train_time=1.366 +[gpua006:0/64] 2024-02-09 20:36:40,153 (trainer:756) INFO: 25epoch:train:10601-10700batch: iter_time=7.804e-05, forward_time=0.202, loss_ctc=78.037, loss_interctc_layer6=81.444, loss_interctc_layer12=66.992, loss_interctc_layer15=61.067, loss_interctc_layer21=80.850, loss=73.678, backward_time=0.230, grad_norm=67.599, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=8.048e-05, train_time=1.291 +[gpua006:0/64] 2024-02-09 20:39:22,899 (trainer:756) INFO: 25epoch:train:10701-10800batch: iter_time=8.205e-05, forward_time=0.142, loss_ctc=79.905, loss_interctc_layer6=78.116, loss_interctc_layer12=64.444, loss_interctc_layer15=58.974, loss_interctc_layer21=82.686, loss=72.825, backward_time=0.208, grad_norm=62.328, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.046e-05, train_time=1.627 +[gpua006:0/64] 2024-02-09 20:41:44,696 (trainer:756) INFO: 25epoch:train:10801-10900batch: iter_time=8.217e-05, forward_time=0.142, loss_ctc=85.141, loss_interctc_layer6=82.936, loss_interctc_layer12=68.295, loss_interctc_layer15=62.452, loss_interctc_layer21=88.255, loss=77.416, backward_time=0.209, grad_norm=117.132, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.045e-05, train_time=1.417 +[gpua006:0/64] 2024-02-09 20:43:47,612 (trainer:756) INFO: 25epoch:train:10901-11000batch: iter_time=7.644e-05, forward_time=0.141, loss_ctc=73.600, loss_interctc_layer6=81.325, loss_interctc_layer12=68.813, loss_interctc_layer15=63.731, loss_interctc_layer21=76.087, loss=72.711, backward_time=0.209, grad_norm=68.052, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.044e-05, train_time=1.230 +[gpua006:0/64] 2024-02-09 20:46:04,883 (trainer:756) INFO: 25epoch:train:11001-11100batch: iter_time=7.911e-05, forward_time=0.167, loss_ctc=79.110, loss_interctc_layer6=81.502, loss_interctc_layer12=67.250, loss_interctc_layer15=61.769, loss_interctc_layer21=81.953, loss=74.317, backward_time=0.240, grad_norm=68.175, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.043e-05, train_time=1.373 +[gpua006:0/64] 2024-02-09 20:48:56,419 (trainer:756) INFO: 25epoch:train:11101-11200batch: iter_time=8.236e-05, forward_time=0.142, loss_ctc=82.162, loss_interctc_layer6=86.819, loss_interctc_layer12=72.596, loss_interctc_layer15=66.760, loss_interctc_layer21=84.816, loss=78.630, backward_time=0.209, grad_norm=67.023, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=8.042e-05, train_time=1.715 +[gpua006:0/64] 2024-02-09 20:50:12,920 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-09 20:50:31,785 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 20:50:35,262 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 20:50:35,262 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-09 20:50:35,265 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 20:56:40,343 (trainer:756) INFO: 25epoch:train:11201-11300batch: iter_time=3.075, forward_time=0.180, loss_ctc=75.770, loss_interctc_layer6=77.833, loss_interctc_layer12=64.476, loss_interctc_layer15=58.855, loss_interctc_layer21=78.354, loss=71.058, backward_time=0.215, grad_norm=80.394, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=8.041e-05, train_time=4.639 +[gpua006:0/64] 2024-02-09 20:58:14,881 (trainer:756) INFO: 25epoch:train:11301-11400batch: iter_time=7.908e-05, forward_time=0.142, loss_ctc=68.709, loss_interctc_layer6=80.477, loss_interctc_layer12=67.022, loss_interctc_layer15=61.733, loss_interctc_layer21=70.853, loss=69.759, backward_time=0.210, grad_norm=75.759, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=8.040e-05, train_time=0.946 +[gpua006:0/64] 2024-02-09 21:00:50,534 (trainer:756) INFO: 25epoch:train:11401-11500batch: iter_time=8.596e-05, forward_time=0.143, loss_ctc=86.693, loss_interctc_layer6=97.423, loss_interctc_layer12=81.408, loss_interctc_layer15=75.519, loss_interctc_layer21=89.270, loss=86.063, backward_time=0.209, grad_norm=77.234, clip=100.000, loss_scale=3.793e+31, optim_step_time=0.137, optim0_lr0=8.039e-05, train_time=1.556 +[gpua006:0/64] 2024-02-09 21:03:16,911 (trainer:756) INFO: 25epoch:train:11501-11600batch: iter_time=8.808e-05, forward_time=0.207, loss_ctc=78.551, loss_interctc_layer6=93.651, loss_interctc_layer12=78.398, loss_interctc_layer15=72.425, loss_interctc_layer21=80.903, loss=80.785, backward_time=0.265, grad_norm=78.037, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.143, optim0_lr0=8.038e-05, train_time=1.463 +[gpua006:0/64] 2024-02-09 21:05:41,714 (trainer:756) INFO: 25epoch:train:11601-11700batch: iter_time=8.192e-05, forward_time=0.143, loss_ctc=73.843, loss_interctc_layer6=79.869, loss_interctc_layer12=65.631, loss_interctc_layer15=60.080, loss_interctc_layer21=76.347, loss=71.154, backward_time=0.210, grad_norm=86.178, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.037e-05, train_time=1.448 +[gpua006:0/64] 2024-02-09 21:07:58,952 (trainer:756) INFO: 25epoch:train:11701-11800batch: iter_time=8.051e-05, forward_time=0.142, loss_ctc=79.508, loss_interctc_layer6=87.881, loss_interctc_layer12=73.093, loss_interctc_layer15=67.092, loss_interctc_layer21=82.016, loss=77.918, backward_time=0.208, grad_norm=78.922, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.036e-05, train_time=1.372 +[gpua006:0/64] 2024-02-09 21:10:31,836 (trainer:756) INFO: 25epoch:train:11801-11900batch: iter_time=8.853e-05, forward_time=0.142, loss_ctc=77.325, loss_interctc_layer6=83.415, loss_interctc_layer12=68.971, loss_interctc_layer15=63.050, loss_interctc_layer21=80.069, loss=74.566, backward_time=0.209, grad_norm=71.291, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.035e-05, train_time=1.529 +[gpua006:0/64] 2024-02-09 21:12:46,338 (trainer:756) INFO: 25epoch:train:11901-12000batch: iter_time=8.344e-05, forward_time=0.140, loss_ctc=75.462, loss_interctc_layer6=75.003, loss_interctc_layer12=61.509, loss_interctc_layer15=55.857, loss_interctc_layer21=78.397, loss=69.246, backward_time=0.207, grad_norm=60.002, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.033e-05, train_time=1.345 +[gpua006:0/64] 2024-02-09 21:15:34,244 (trainer:756) INFO: 25epoch:train:12001-12100batch: iter_time=8.626e-05, forward_time=0.142, loss_ctc=72.181, loss_interctc_layer6=80.429, loss_interctc_layer12=66.334, loss_interctc_layer15=60.636, loss_interctc_layer21=74.482, loss=70.812, backward_time=0.207, grad_norm=91.542, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.032e-05, train_time=1.679 +[gpua006:0/64] 2024-02-09 21:17:49,026 (trainer:756) INFO: 25epoch:train:12101-12200batch: iter_time=8.887e-05, forward_time=0.253, loss_ctc=73.865, loss_interctc_layer6=83.813, loss_interctc_layer12=69.583, loss_interctc_layer15=63.834, loss_interctc_layer21=76.135, loss=73.446, backward_time=0.246, grad_norm=79.527, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.146, optim0_lr0=8.031e-05, train_time=1.347 +[gpua006:0/64] 2024-02-09 21:19:55,846 (trainer:756) INFO: 25epoch:train:12201-12300batch: iter_time=8.622e-05, forward_time=0.142, loss_ctc=75.454, loss_interctc_layer6=82.194, loss_interctc_layer12=69.122, loss_interctc_layer15=63.695, loss_interctc_layer21=77.809, loss=73.655, backward_time=0.209, grad_norm=80.411, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.030e-05, train_time=1.269 +[gpua006:0/64] 2024-02-09 21:22:32,426 (trainer:756) INFO: 25epoch:train:12301-12400batch: iter_time=8.966e-05, forward_time=0.142, loss_ctc=75.308, loss_interctc_layer6=82.141, loss_interctc_layer12=68.077, loss_interctc_layer15=62.687, loss_interctc_layer21=78.075, loss=73.258, backward_time=0.209, grad_norm=79.268, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.029e-05, train_time=1.566 +[gpua006:0/64] 2024-02-09 21:24:45,670 (trainer:756) INFO: 25epoch:train:12401-12500batch: iter_time=8.131e-05, forward_time=0.141, loss_ctc=68.256, loss_interctc_layer6=80.456, loss_interctc_layer12=67.261, loss_interctc_layer15=61.735, loss_interctc_layer21=70.561, loss=69.654, backward_time=0.209, grad_norm=59.824, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.028e-05, train_time=1.332 +[gpua006:0/64] 2024-02-09 21:25:05,700 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-09 21:25:24,559 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 21:25:27,950 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 21:25:27,951 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-09 21:25:27,988 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 21:31:12,784 (trainer:756) INFO: 25epoch:train:12501-12600batch: iter_time=2.808, forward_time=0.142, loss_ctc=83.230, loss_interctc_layer6=81.451, loss_interctc_layer12=67.338, loss_interctc_layer15=61.554, loss_interctc_layer21=85.942, loss=75.903, backward_time=0.209, grad_norm=66.818, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.027e-05, train_time=3.871 +[gpua006:0/64] 2024-02-09 21:33:04,280 (trainer:756) INFO: 25epoch:train:12601-12700batch: iter_time=7.942e-05, forward_time=0.179, loss_ctc=65.802, loss_interctc_layer6=74.001, loss_interctc_layer12=61.427, loss_interctc_layer15=56.424, loss_interctc_layer21=67.917, loss=65.114, backward_time=0.276, grad_norm=136.002, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=8.026e-05, train_time=1.114 +[gpua006:0/64] 2024-02-09 21:35:57,202 (trainer:756) INFO: 25epoch:train:12701-12800batch: iter_time=9.321e-05, forward_time=0.144, loss_ctc=109.105, loss_interctc_layer6=109.937, loss_interctc_layer12=91.231, loss_interctc_layer15=84.950, loss_interctc_layer21=111.796, loss=101.404, backward_time=0.207, grad_norm=122.937, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.025e-05, train_time=1.730 +[gpua006:0/64] 2024-02-09 21:38:50,623 (trainer:756) INFO: 25epoch:train:12801-12900batch: iter_time=8.407e-05, forward_time=0.142, loss_ctc=75.618, loss_interctc_layer6=85.287, loss_interctc_layer12=71.487, loss_interctc_layer15=65.933, loss_interctc_layer21=78.052, loss=75.275, backward_time=0.209, grad_norm=92.134, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.024e-05, train_time=1.734 +[gpua006:0/64] 2024-02-09 21:41:00,452 (trainer:756) INFO: 25epoch:train:12901-13000batch: iter_time=8.345e-05, forward_time=0.142, loss_ctc=85.702, loss_interctc_layer6=85.750, loss_interctc_layer12=70.990, loss_interctc_layer15=65.229, loss_interctc_layer21=88.719, loss=79.278, backward_time=0.210, grad_norm=66.574, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.023e-05, train_time=1.298 +[gpua006:0/64] 2024-02-09 21:42:41,572 (trainer:756) INFO: 25epoch:train:13001-13100batch: iter_time=8.262e-05, forward_time=0.142, loss_ctc=84.359, loss_interctc_layer6=83.662, loss_interctc_layer12=69.363, loss_interctc_layer15=63.559, loss_interctc_layer21=87.484, loss=77.685, backward_time=0.210, grad_norm=67.262, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.135, optim0_lr0=8.022e-05, train_time=1.011 +[gpua006:0/64] 2024-02-09 21:45:23,854 (trainer:756) INFO: 25epoch:train:13101-13200batch: iter_time=8.279e-05, forward_time=0.142, loss_ctc=77.777, loss_interctc_layer6=80.740, loss_interctc_layer12=66.480, loss_interctc_layer15=60.473, loss_interctc_layer21=80.635, loss=73.221, backward_time=0.208, grad_norm=64.654, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.021e-05, train_time=1.623 +[gpua006:0/64] 2024-02-09 21:48:35,815 (trainer:756) INFO: 25epoch:train:13201-13300batch: iter_time=8.549e-05, forward_time=0.259, loss_ctc=79.100, loss_interctc_layer6=77.344, loss_interctc_layer12=63.799, loss_interctc_layer15=58.318, loss_interctc_layer21=81.892, loss=72.091, backward_time=0.236, grad_norm=67.797, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.150, optim0_lr0=8.019e-05, train_time=1.919 +[gpua006:0/64] 2024-02-09 21:51:03,311 (trainer:756) INFO: 25epoch:train:13301-13400batch: iter_time=8.409e-05, forward_time=0.143, loss_ctc=83.073, loss_interctc_layer6=82.581, loss_interctc_layer12=68.093, loss_interctc_layer15=62.233, loss_interctc_layer21=86.175, loss=76.431, backward_time=0.210, grad_norm=70.264, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.018e-05, train_time=1.476 +[gpua006:0/64] 2024-02-09 21:53:43,426 (trainer:756) INFO: 25epoch:train:13401-13500batch: iter_time=8.836e-05, forward_time=0.141, loss_ctc=73.137, loss_interctc_layer6=81.140, loss_interctc_layer12=68.333, loss_interctc_layer15=63.343, loss_interctc_layer21=75.449, loss=72.280, backward_time=0.210, grad_norm=70.834, clip=100.000, loss_scale=7.586e+31, optim_step_time=0.136, optim0_lr0=8.017e-05, train_time=1.601 +[gpua006:0/64] 2024-02-09 21:55:57,445 (trainer:756) INFO: 25epoch:train:13501-13600batch: iter_time=8.171e-05, forward_time=0.142, loss_ctc=78.636, loss_interctc_layer6=81.101, loss_interctc_layer12=67.128, loss_interctc_layer15=61.605, loss_interctc_layer21=81.536, loss=74.001, backward_time=0.210, grad_norm=67.554, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.136, optim0_lr0=8.016e-05, train_time=1.340 +[gpua006:0/64] 2024-02-09 21:57:51,005 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 21:58:34,119 (trainer:756) INFO: 25epoch:train:13601-13700batch: iter_time=8.390e-05, forward_time=0.143, loss_ctc=81.697, loss_interctc_layer6=86.172, loss_interctc_layer12=72.023, loss_interctc_layer15=66.300, loss_interctc_layer21=84.213, loss=78.081, backward_time=0.210, grad_norm=106.515, clip=100.000, loss_scale=6.843e+31, optim_step_time=0.136, optim0_lr0=8.015e-05, train_time=1.567 +[gpua006:0/64] 2024-02-09 21:59:55,848 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-09 22:00:14,164 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 22:00:17,583 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 22:00:17,583 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-09 22:00:17,587 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 22:06:01,101 (trainer:756) INFO: 25epoch:train:13701-13800batch: iter_time=3.242, forward_time=0.204, loss_ctc=76.162, loss_interctc_layer6=77.309, loss_interctc_layer12=63.938, loss_interctc_layer15=58.345, loss_interctc_layer21=78.737, loss=70.898, backward_time=0.226, grad_norm=65.090, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=8.014e-05, train_time=4.469 +[gpua006:0/64] 2024-02-09 22:07:57,716 (trainer:756) INFO: 25epoch:train:13801-13900batch: iter_time=8.170e-05, forward_time=0.144, loss_ctc=75.650, loss_interctc_layer6=79.480, loss_interctc_layer12=66.170, loss_interctc_layer15=60.989, loss_interctc_layer21=77.923, loss=72.042, backward_time=0.209, grad_norm=93.540, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.013e-05, train_time=1.166 +[gpua006:0/64] 2024-02-09 22:10:10,814 (trainer:756) INFO: 25epoch:train:13901-14000batch: iter_time=7.837e-05, forward_time=0.142, loss_ctc=91.322, loss_interctc_layer6=96.804, loss_interctc_layer12=80.732, loss_interctc_layer15=74.441, loss_interctc_layer21=93.917, loss=87.443, backward_time=0.209, grad_norm=91.989, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.012e-05, train_time=1.331 +[gpua006:0/64] 2024-02-09 22:12:15,574 (trainer:756) INFO: 25epoch:train:14001-14100batch: iter_time=8.427e-05, forward_time=0.142, loss_ctc=85.854, loss_interctc_layer6=94.414, loss_interctc_layer12=78.852, loss_interctc_layer15=72.657, loss_interctc_layer21=88.431, loss=84.042, backward_time=0.209, grad_norm=70.202, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.135, optim0_lr0=8.011e-05, train_time=1.247 +[gpua006:0/64] 2024-02-09 22:14:38,872 (trainer:756) INFO: 25epoch:train:14101-14200batch: iter_time=8.645e-05, forward_time=0.143, loss_ctc=79.383, loss_interctc_layer6=80.254, loss_interctc_layer12=66.057, loss_interctc_layer15=60.455, loss_interctc_layer21=82.159, loss=73.662, backward_time=0.207, grad_norm=79.049, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.135, optim0_lr0=8.010e-05, train_time=1.433 +[gpua006:0/64] 2024-02-09 22:17:16,651 (trainer:756) INFO: 25epoch:train:14201-14300batch: iter_time=2.276e-04, forward_time=0.155, loss_ctc=86.925, loss_interctc_layer6=89.101, loss_interctc_layer12=74.154, loss_interctc_layer15=68.123, loss_interctc_layer21=89.914, loss=81.643, backward_time=0.208, grad_norm=77.789, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.009e-05, train_time=1.577 +[gpua006:0/64] 2024-02-09 22:19:15,210 (trainer:756) INFO: 25epoch:train:14301-14400batch: iter_time=8.289e-05, forward_time=0.222, loss_ctc=81.409, loss_interctc_layer6=84.153, loss_interctc_layer12=69.443, loss_interctc_layer15=63.505, loss_interctc_layer21=84.298, loss=76.562, backward_time=0.243, grad_norm=67.929, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=8.008e-05, train_time=1.185 +[gpua006:0/64] 2024-02-09 22:21:54,869 (trainer:756) INFO: 25epoch:train:14401-14500batch: iter_time=8.208e-05, forward_time=0.141, loss_ctc=81.652, loss_interctc_layer6=74.887, loss_interctc_layer12=61.277, loss_interctc_layer15=55.782, loss_interctc_layer21=84.817, loss=71.683, backward_time=0.208, grad_norm=97.325, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.007e-05, train_time=1.597 +[gpua006:0/64] 2024-02-09 22:24:48,574 (trainer:756) INFO: 25epoch:train:14501-14600batch: iter_time=8.277e-05, forward_time=0.158, loss_ctc=78.045, loss_interctc_layer6=80.074, loss_interctc_layer12=66.023, loss_interctc_layer15=60.380, loss_interctc_layer21=80.700, loss=73.044, backward_time=0.209, grad_norm=77.465, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.006e-05, train_time=1.737 +[gpua006:0/64] 2024-02-09 22:27:10,083 (trainer:756) INFO: 25epoch:train:14601-14700batch: iter_time=6.895e-04, forward_time=0.256, loss_ctc=77.988, loss_interctc_layer6=83.498, loss_interctc_layer12=69.202, loss_interctc_layer15=63.540, loss_interctc_layer21=80.584, loss=74.962, backward_time=0.231, grad_norm=73.418, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.145, optim0_lr0=8.004e-05, train_time=1.413 +[gpua006:0/64] 2024-02-09 22:29:33,398 (trainer:756) INFO: 25epoch:train:14701-14800batch: iter_time=8.600e-05, forward_time=0.142, loss_ctc=78.248, loss_interctc_layer6=82.314, loss_interctc_layer12=69.029, loss_interctc_layer15=63.721, loss_interctc_layer21=80.707, loss=74.804, backward_time=0.209, grad_norm=72.797, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.003e-05, train_time=1.435 +[gpua006:0/64] 2024-02-09 22:32:01,069 (trainer:756) INFO: 25epoch:train:14801-14900batch: iter_time=8.244e-05, forward_time=0.141, loss_ctc=80.973, loss_interctc_layer6=81.644, loss_interctc_layer12=67.767, loss_interctc_layer15=62.322, loss_interctc_layer21=83.749, loss=75.291, backward_time=0.209, grad_norm=65.574, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.002e-05, train_time=1.476 +[gpua006:0/64] 2024-02-09 22:33:58,867 (trainer:756) INFO: 25epoch:train:14901-15000batch: iter_time=8.317e-05, forward_time=0.142, loss_ctc=70.345, loss_interctc_layer6=80.314, loss_interctc_layer12=67.102, loss_interctc_layer15=61.585, loss_interctc_layer21=72.676, loss=70.404, backward_time=0.211, grad_norm=69.020, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=8.001e-05, train_time=1.178 +[gpua006:0/64] 2024-02-09 23:05:16,032 (trainer:355) INFO: 25epoch results: [train] iter_time=0.250, forward_time=0.156, loss_ctc=79.170, loss_interctc_layer6=84.418, loss_interctc_layer12=70.260, loss_interctc_layer15=64.593, loss_interctc_layer21=81.794, loss=76.047, backward_time=0.221, grad_norm=80.507, clip=100.000, loss_scale=2.463e+31, optim_step_time=0.137, optim0_lr0=8.082e-05, train_time=1.660, time=6 hours, 55 minutes and 33.5 seconds, total_count=375000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=45.864, cer_ctc=0.211, loss_interctc_layer6=51.545, cer_interctc_layer6=0.228, loss_interctc_layer12=38.814, cer_interctc_layer12=0.161, loss_interctc_layer15=34.302, cer_interctc_layer15=0.135, loss_interctc_layer21=48.586, cer_interctc_layer21=0.222, loss=43.822, time=30 minutes and 53.17 seconds, total_count=116775, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-09 23:05:35,284 (trainer:410) INFO: The best model has been updated: valid.total_count +[gpua006:0/64] 2024-02-09 23:05:35,310 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/20epoch.pth +[gpua006:0/64] 2024-02-09 23:05:35,310 (trainer:289) INFO: 26/45epoch started. Estimated time to finish: 6 days, 19 hours and 5 minutes +[gpua006:0/64] 2024-02-09 23:05:35,326 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-09 23:05:53,038 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 23:05:56,358 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 23:05:56,358 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-09 23:05:56,362 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 23:12:44,071 (trainer:756) INFO: 26epoch:train:1-100batch: iter_time=2.876, forward_time=0.181, loss_ctc=73.938, loss_interctc_layer6=78.854, loss_interctc_layer12=66.012, loss_interctc_layer15=60.837, loss_interctc_layer21=76.371, loss=71.202, backward_time=0.217, grad_norm=63.459, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=8.000e-05, train_time=4.287 +[gpua006:0/64] 2024-02-09 23:14:19,847 (trainer:756) INFO: 26epoch:train:101-200batch: iter_time=1.002e-04, forward_time=0.144, loss_ctc=67.274, loss_interctc_layer6=83.712, loss_interctc_layer12=69.962, loss_interctc_layer15=64.517, loss_interctc_layer21=69.248, loss=70.943, backward_time=0.211, grad_norm=64.102, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.999e-05, train_time=0.958 +[gpua006:0/64] 2024-02-09 23:16:01,618 (trainer:756) INFO: 26epoch:train:201-300batch: iter_time=7.702e-05, forward_time=0.141, loss_ctc=75.698, loss_interctc_layer6=83.172, loss_interctc_layer12=69.656, loss_interctc_layer15=64.262, loss_interctc_layer21=78.187, loss=74.195, backward_time=0.210, grad_norm=65.386, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.998e-05, train_time=1.017 +[gpua006:0/64] 2024-02-09 23:18:04,032 (trainer:756) INFO: 26epoch:train:301-400batch: iter_time=9.702e-05, forward_time=0.144, loss_ctc=84.720, loss_interctc_layer6=86.527, loss_interctc_layer12=71.915, loss_interctc_layer15=66.021, loss_interctc_layer21=87.380, loss=79.312, backward_time=0.215, grad_norm=113.317, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.997e-05, train_time=1.224 +[gpua006:0/64] 2024-02-09 23:20:01,796 (trainer:756) INFO: 26epoch:train:401-500batch: iter_time=8.097e-05, forward_time=0.144, loss_ctc=68.567, loss_interctc_layer6=81.622, loss_interctc_layer12=67.933, loss_interctc_layer15=62.409, loss_interctc_layer21=70.640, loss=70.234, backward_time=0.211, grad_norm=60.293, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.996e-05, train_time=1.174 +[gpua006:0/64] 2024-02-09 23:22:05,162 (trainer:756) INFO: 26epoch:train:501-600batch: iter_time=1.542e-04, forward_time=0.171, loss_ctc=83.328, loss_interctc_layer6=89.231, loss_interctc_layer12=73.976, loss_interctc_layer15=67.918, loss_interctc_layer21=86.224, loss=80.135, backward_time=0.213, grad_norm=77.915, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=7.995e-05, train_time=1.236 +[gpua006:0/64] 2024-02-09 23:24:19,594 (trainer:756) INFO: 26epoch:train:601-700batch: iter_time=8.837e-05, forward_time=0.201, loss_ctc=75.908, loss_interctc_layer6=83.319, loss_interctc_layer12=68.935, loss_interctc_layer15=63.147, loss_interctc_layer21=78.503, loss=73.962, backward_time=0.239, grad_norm=67.854, clip=100.000, loss_scale=5.314e+31, optim_step_time=0.139, optim0_lr0=7.994e-05, train_time=1.344 +[gpua006:0/64] 2024-02-09 23:26:34,335 (trainer:756) INFO: 26epoch:train:701-800batch: iter_time=8.121e-05, forward_time=0.142, loss_ctc=76.262, loss_interctc_layer6=83.352, loss_interctc_layer12=69.448, loss_interctc_layer15=63.911, loss_interctc_layer21=78.973, loss=74.389, backward_time=0.208, grad_norm=68.010, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.137, optim0_lr0=7.993e-05, train_time=1.347 +[gpua006:0/64] 2024-02-09 23:28:33,957 (trainer:756) INFO: 26epoch:train:801-900batch: iter_time=7.980e-05, forward_time=0.143, loss_ctc=73.396, loss_interctc_layer6=85.356, loss_interctc_layer12=71.150, loss_interctc_layer15=65.382, loss_interctc_layer21=75.740, loss=74.205, backward_time=0.213, grad_norm=71.961, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.137, optim0_lr0=7.992e-05, train_time=1.196 +[gpua006:0/64] 2024-02-09 23:29:14,997 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-09 23:30:39,303 (trainer:756) INFO: 26epoch:train:901-1000batch: iter_time=8.592e-05, forward_time=0.146, loss_ctc=85.568, loss_interctc_layer6=99.277, loss_interctc_layer12=82.476, loss_interctc_layer15=76.956, loss_interctc_layer21=88.133, loss=86.482, backward_time=0.209, grad_norm=91.649, clip=100.000, loss_scale=5.327e+31, optim_step_time=0.137, optim0_lr0=7.991e-05, train_time=1.252 +[gpua006:0/64] 2024-02-09 23:33:11,147 (trainer:756) INFO: 26epoch:train:1001-1100batch: iter_time=8.292e-05, forward_time=0.165, loss_ctc=74.456, loss_interctc_layer6=79.255, loss_interctc_layer12=65.844, loss_interctc_layer15=60.530, loss_interctc_layer21=77.131, loss=71.443, backward_time=0.215, grad_norm=64.112, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=7.990e-05, train_time=1.519 +[gpua006:0/64] 2024-02-09 23:35:30,843 (trainer:756) INFO: 26epoch:train:1101-1200batch: iter_time=8.221e-05, forward_time=0.204, loss_ctc=78.709, loss_interctc_layer6=89.908, loss_interctc_layer12=75.402, loss_interctc_layer15=69.735, loss_interctc_layer21=81.280, loss=79.007, backward_time=0.225, grad_norm=67.912, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=7.989e-05, train_time=1.397 +[gpua006:0/64] 2024-02-09 23:36:55,965 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-09 23:37:14,201 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-09 23:37:17,793 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-09 23:37:17,793 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-09 23:37:17,796 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-09 23:42:54,332 (trainer:756) INFO: 26epoch:train:1201-1300batch: iter_time=3.104, forward_time=0.149, loss_ctc=75.769, loss_interctc_layer6=79.009, loss_interctc_layer12=65.734, loss_interctc_layer15=60.462, loss_interctc_layer21=78.360, loss=71.867, backward_time=0.210, grad_norm=62.423, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.987e-05, train_time=4.435 +[gpua006:0/64] 2024-02-09 23:44:35,055 (trainer:756) INFO: 26epoch:train:1301-1400batch: iter_time=4.277e-04, forward_time=0.150, loss_ctc=63.736, loss_interctc_layer6=76.305, loss_interctc_layer12=64.025, loss_interctc_layer15=59.131, loss_interctc_layer21=65.774, loss=65.794, backward_time=0.224, grad_norm=58.798, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=7.986e-05, train_time=1.004 +[gpua006:0/64] 2024-02-09 23:46:49,770 (trainer:756) INFO: 26epoch:train:1401-1500batch: iter_time=7.925e-05, forward_time=0.143, loss_ctc=71.803, loss_interctc_layer6=82.482, loss_interctc_layer12=68.275, loss_interctc_layer15=62.699, loss_interctc_layer21=73.830, loss=71.818, backward_time=0.210, grad_norm=74.546, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.985e-05, train_time=1.350 +[gpua006:0/64] 2024-02-09 23:48:31,816 (trainer:756) INFO: 26epoch:train:1501-1600batch: iter_time=8.024e-05, forward_time=0.142, loss_ctc=77.761, loss_interctc_layer6=85.275, loss_interctc_layer12=71.077, loss_interctc_layer15=65.375, loss_interctc_layer21=80.240, loss=75.946, backward_time=0.210, grad_norm=104.984, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.984e-05, train_time=1.020 +[gpua006:0/64] 2024-02-09 23:51:04,796 (trainer:756) INFO: 26epoch:train:1601-1700batch: iter_time=7.915e-05, forward_time=0.233, loss_ctc=84.623, loss_interctc_layer6=84.363, loss_interctc_layer12=69.749, loss_interctc_layer15=63.946, loss_interctc_layer21=87.469, loss=78.030, backward_time=0.225, grad_norm=59.009, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.154, optim0_lr0=7.983e-05, train_time=1.529 +[gpua006:0/64] 2024-02-09 23:53:35,243 (trainer:756) INFO: 26epoch:train:1701-1800batch: iter_time=8.148e-05, forward_time=0.142, loss_ctc=77.043, loss_interctc_layer6=81.611, loss_interctc_layer12=67.375, loss_interctc_layer15=61.700, loss_interctc_layer21=79.580, loss=73.462, backward_time=0.208, grad_norm=81.118, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.982e-05, train_time=1.505 +[gpua006:0/64] 2024-02-09 23:55:55,955 (trainer:756) INFO: 26epoch:train:1801-1900batch: iter_time=8.223e-05, forward_time=0.154, loss_ctc=84.064, loss_interctc_layer6=85.129, loss_interctc_layer12=70.491, loss_interctc_layer15=64.693, loss_interctc_layer21=86.686, loss=78.213, backward_time=0.215, grad_norm=69.140, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=7.981e-05, train_time=1.405 +[gpua006:0/64] 2024-02-09 23:58:13,485 (trainer:756) INFO: 26epoch:train:1901-2000batch: iter_time=8.081e-05, forward_time=0.164, loss_ctc=79.594, loss_interctc_layer6=87.068, loss_interctc_layer12=72.009, loss_interctc_layer15=65.914, loss_interctc_layer21=82.241, loss=77.365, backward_time=0.270, grad_norm=75.856, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=7.980e-05, train_time=1.376 +[gpua006:0/64] 2024-02-10 00:00:21,227 (trainer:756) INFO: 26epoch:train:2001-2100batch: iter_time=8.012e-05, forward_time=0.149, loss_ctc=65.534, loss_interctc_layer6=76.671, loss_interctc_layer12=63.907, loss_interctc_layer15=58.745, loss_interctc_layer21=67.448, loss=66.461, backward_time=0.208, grad_norm=58.780, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.979e-05, train_time=1.277 +[gpua006:0/64] 2024-02-10 00:01:52,644 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 00:02:15,640 (trainer:756) INFO: 26epoch:train:2101-2200batch: iter_time=7.668e-05, forward_time=0.167, loss_ctc=96.710, loss_interctc_layer6=103.293, loss_interctc_layer12=86.223, loss_interctc_layer15=79.568, loss_interctc_layer21=99.693, loss=93.097, backward_time=0.209, grad_norm=93.300, clip=100.000, loss_scale=3.626e+31, optim_step_time=0.136, optim0_lr0=7.978e-05, train_time=1.144 +[gpua006:0/64] 2024-02-10 00:04:07,078 (trainer:756) INFO: 26epoch:train:2201-2300batch: iter_time=7.545e-05, forward_time=0.141, loss_ctc=72.379, loss_interctc_layer6=79.262, loss_interctc_layer12=65.673, loss_interctc_layer15=60.545, loss_interctc_layer21=74.893, loss=70.550, backward_time=0.209, grad_norm=85.754, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.977e-05, train_time=1.114 +[gpua006:0/64] 2024-02-10 00:06:15,417 (trainer:756) INFO: 26epoch:train:2301-2400batch: iter_time=8.052e-05, forward_time=0.150, loss_ctc=83.994, loss_interctc_layer6=85.773, loss_interctc_layer12=71.399, loss_interctc_layer15=65.740, loss_interctc_layer21=86.834, loss=78.748, backward_time=0.227, grad_norm=86.471, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=7.976e-05, train_time=1.280 +[gpua006:0/64] 2024-02-10 00:08:43,057 (trainer:756) INFO: 26epoch:train:2401-2500batch: iter_time=8.286e-05, forward_time=0.224, loss_ctc=69.877, loss_interctc_layer6=78.509, loss_interctc_layer12=65.271, loss_interctc_layer15=60.124, loss_interctc_layer21=72.060, loss=69.168, backward_time=0.241, grad_norm=57.318, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=7.975e-05, train_time=1.479 +[gpua006:0/64] 2024-02-10 00:09:03,151 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-10 00:09:21,732 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 00:09:25,085 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 00:09:25,085 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-10 00:09:25,144 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 00:15:49,835 (trainer:756) INFO: 26epoch:train:2501-2600batch: iter_time=2.940, forward_time=0.175, loss_ctc=71.840, loss_interctc_layer6=77.947, loss_interctc_layer12=64.908, loss_interctc_layer15=59.806, loss_interctc_layer21=74.288, loss=69.758, backward_time=0.216, grad_norm=66.132, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.974e-05, train_time=4.267 +[gpua006:0/64] 2024-02-10 00:17:25,514 (trainer:756) INFO: 26epoch:train:2601-2700batch: iter_time=7.828e-05, forward_time=0.142, loss_ctc=65.540, loss_interctc_layer6=81.973, loss_interctc_layer12=68.150, loss_interctc_layer15=62.705, loss_interctc_layer21=67.559, loss=69.186, backward_time=0.209, grad_norm=73.838, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.973e-05, train_time=0.958 +[gpua006:0/64] 2024-02-10 00:19:32,401 (trainer:756) INFO: 26epoch:train:2701-2800batch: iter_time=7.860e-05, forward_time=0.142, loss_ctc=73.680, loss_interctc_layer6=81.625, loss_interctc_layer12=67.995, loss_interctc_layer15=62.570, loss_interctc_layer21=75.905, loss=72.355, backward_time=0.209, grad_norm=67.483, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.972e-05, train_time=1.269 +[gpua006:0/64] 2024-02-10 00:21:29,610 (trainer:756) INFO: 26epoch:train:2801-2900batch: iter_time=8.078e-05, forward_time=0.142, loss_ctc=82.936, loss_interctc_layer6=85.186, loss_interctc_layer12=70.453, loss_interctc_layer15=64.476, loss_interctc_layer21=85.638, loss=77.738, backward_time=0.208, grad_norm=64.267, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.971e-05, train_time=1.172 +[gpua006:0/64] 2024-02-10 00:23:30,292 (trainer:756) INFO: 26epoch:train:2901-3000batch: iter_time=8.542e-05, forward_time=0.214, loss_ctc=66.964, loss_interctc_layer6=80.333, loss_interctc_layer12=66.625, loss_interctc_layer15=61.109, loss_interctc_layer21=69.034, loss=68.813, backward_time=0.282, grad_norm=67.142, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.144, optim0_lr0=7.969e-05, train_time=1.206 +[gpua006:0/64] 2024-02-10 00:25:24,523 (trainer:756) INFO: 26epoch:train:3001-3100batch: iter_time=8.262e-05, forward_time=0.143, loss_ctc=83.804, loss_interctc_layer6=88.940, loss_interctc_layer12=73.675, loss_interctc_layer15=67.592, loss_interctc_layer21=86.828, loss=80.168, backward_time=0.210, grad_norm=65.925, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.968e-05, train_time=1.142 +[gpua006:0/64] 2024-02-10 00:27:35,102 (trainer:756) INFO: 26epoch:train:3101-3200batch: iter_time=8.526e-05, forward_time=0.142, loss_ctc=75.473, loss_interctc_layer6=82.636, loss_interctc_layer12=68.024, loss_interctc_layer15=62.246, loss_interctc_layer21=78.091, loss=73.294, backward_time=0.209, grad_norm=76.649, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.967e-05, train_time=1.306 +[gpua006:0/64] 2024-02-10 00:30:35,788 (trainer:756) INFO: 26epoch:train:3201-3300batch: iter_time=8.318e-05, forward_time=0.249, loss_ctc=75.948, loss_interctc_layer6=82.769, loss_interctc_layer12=68.826, loss_interctc_layer15=63.246, loss_interctc_layer21=78.583, loss=73.874, backward_time=0.244, grad_norm=78.619, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.966e-05, train_time=1.806 +[gpua006:0/64] 2024-02-10 00:32:39,965 (trainer:756) INFO: 26epoch:train:3301-3400batch: iter_time=8.287e-05, forward_time=0.144, loss_ctc=73.047, loss_interctc_layer6=84.523, loss_interctc_layer12=70.220, loss_interctc_layer15=64.492, loss_interctc_layer21=75.340, loss=73.524, backward_time=0.210, grad_norm=70.090, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.965e-05, train_time=1.242 +[gpua006:0/64] 2024-02-10 00:34:43,953 (trainer:756) INFO: 26epoch:train:3401-3500batch: iter_time=7.892e-05, forward_time=0.145, loss_ctc=85.878, loss_interctc_layer6=97.422, loss_interctc_layer12=81.905, loss_interctc_layer15=75.977, loss_interctc_layer21=88.546, loss=85.946, backward_time=0.210, grad_norm=108.925, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.964e-05, train_time=1.240 +[gpua006:0/64] 2024-02-10 00:36:47,091 (trainer:756) INFO: 26epoch:train:3501-3600batch: iter_time=7.940e-05, forward_time=0.142, loss_ctc=74.206, loss_interctc_layer6=79.039, loss_interctc_layer12=65.427, loss_interctc_layer15=59.976, loss_interctc_layer21=76.920, loss=71.113, backward_time=0.209, grad_norm=56.909, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.963e-05, train_time=1.231 +[gpua006:0/64] 2024-02-10 00:38:37,445 (trainer:756) INFO: 26epoch:train:3601-3700batch: iter_time=7.744e-05, forward_time=0.145, loss_ctc=78.482, loss_interctc_layer6=88.975, loss_interctc_layer12=74.290, loss_interctc_layer15=68.583, loss_interctc_layer21=80.799, loss=78.226, backward_time=0.211, grad_norm=76.474, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.962e-05, train_time=1.104 +[gpua006:0/64] 2024-02-10 00:39:54,837 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-10 00:40:13,273 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 00:40:16,649 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 00:40:16,649 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-10 00:40:16,757 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 00:46:18,142 (trainer:756) INFO: 26epoch:train:3701-3800batch: iter_time=3.136, forward_time=0.255, loss_ctc=72.646, loss_interctc_layer6=78.976, loss_interctc_layer12=65.569, loss_interctc_layer15=60.105, loss_interctc_layer21=75.013, loss=70.462, backward_time=0.240, grad_norm=68.304, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.156, optim0_lr0=7.961e-05, train_time=4.607 +[gpua006:0/64] 2024-02-10 00:48:04,653 (trainer:756) INFO: 26epoch:train:3801-3900batch: iter_time=8.921e-05, forward_time=0.141, loss_ctc=61.769, loss_interctc_layer6=75.573, loss_interctc_layer12=63.234, loss_interctc_layer15=58.478, loss_interctc_layer21=63.749, loss=64.560, backward_time=0.211, grad_norm=68.515, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.960e-05, train_time=1.065 +[gpua006:0/64] 2024-02-10 00:49:41,734 (trainer:756) INFO: 26epoch:train:3901-4000batch: iter_time=9.010e-05, forward_time=0.142, loss_ctc=70.064, loss_interctc_layer6=82.081, loss_interctc_layer12=68.011, loss_interctc_layer15=62.491, loss_interctc_layer21=72.102, loss=70.950, backward_time=0.211, grad_norm=73.337, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.959e-05, train_time=0.970 +[gpua006:0/64] 2024-02-10 00:52:07,212 (trainer:756) INFO: 26epoch:train:4001-4100batch: iter_time=8.834e-05, forward_time=0.142, loss_ctc=74.331, loss_interctc_layer6=84.563, loss_interctc_layer12=70.446, loss_interctc_layer15=64.682, loss_interctc_layer21=76.699, loss=74.144, backward_time=0.209, grad_norm=83.809, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.958e-05, train_time=1.454 +[gpua006:0/64] 2024-02-10 00:54:27,992 (trainer:756) INFO: 26epoch:train:4101-4200batch: iter_time=8.244e-05, forward_time=0.142, loss_ctc=83.353, loss_interctc_layer6=84.072, loss_interctc_layer12=69.386, loss_interctc_layer15=63.501, loss_interctc_layer21=86.286, loss=77.320, backward_time=0.207, grad_norm=67.457, clip=100.000, loss_scale=2.454e+31, optim_step_time=0.137, optim0_lr0=7.957e-05, train_time=1.408 +[gpua006:0/64] 2024-02-10 00:57:02,614 (trainer:756) INFO: 26epoch:train:4201-4300batch: iter_time=8.596e-05, forward_time=0.238, loss_ctc=73.178, loss_interctc_layer6=80.863, loss_interctc_layer12=66.733, loss_interctc_layer15=60.992, loss_interctc_layer21=75.727, loss=71.499, backward_time=0.228, grad_norm=69.966, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.154, optim0_lr0=7.956e-05, train_time=1.545 +[gpua006:0/64] 2024-02-10 00:59:31,161 (trainer:756) INFO: 26epoch:train:4301-4400batch: iter_time=8.734e-05, forward_time=0.150, loss_ctc=80.353, loss_interctc_layer6=84.520, loss_interctc_layer12=69.766, loss_interctc_layer15=63.876, loss_interctc_layer21=83.247, loss=76.352, backward_time=0.222, grad_norm=73.275, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.955e-05, train_time=1.486 +[gpua006:0/64] 2024-02-10 01:01:33,891 (trainer:756) INFO: 26epoch:train:4401-4500batch: iter_time=8.402e-05, forward_time=0.142, loss_ctc=75.516, loss_interctc_layer6=86.976, loss_interctc_layer12=71.902, loss_interctc_layer15=65.748, loss_interctc_layer21=78.101, loss=75.648, backward_time=0.209, grad_norm=63.855, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.954e-05, train_time=1.227 +[gpua006:0/64] 2024-02-10 01:03:46,710 (trainer:756) INFO: 26epoch:train:4501-4600batch: iter_time=9.509e-05, forward_time=0.142, loss_ctc=62.685, loss_interctc_layer6=75.670, loss_interctc_layer12=62.970, loss_interctc_layer15=57.851, loss_interctc_layer21=64.609, loss=64.757, backward_time=0.210, grad_norm=78.876, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.953e-05, train_time=1.328 +[gpua006:0/64] 2024-02-10 01:05:58,531 (trainer:756) INFO: 26epoch:train:4601-4700batch: iter_time=8.180e-05, forward_time=0.144, loss_ctc=92.403, loss_interctc_layer6=102.216, loss_interctc_layer12=85.816, loss_interctc_layer15=79.442, loss_interctc_layer21=95.317, loss=91.039, backward_time=0.209, grad_norm=82.811, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.952e-05, train_time=1.318 +[gpua006:0/64] 2024-02-10 01:07:51,862 (trainer:756) INFO: 26epoch:train:4701-4800batch: iter_time=8.110e-05, forward_time=0.141, loss_ctc=67.959, loss_interctc_layer6=78.316, loss_interctc_layer12=64.981, loss_interctc_layer15=59.836, loss_interctc_layer21=70.244, loss=68.267, backward_time=0.209, grad_norm=59.769, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.951e-05, train_time=1.133 +[gpua006:0/64] 2024-02-10 01:10:00,385 (trainer:756) INFO: 26epoch:train:4801-4900batch: iter_time=1.981e-04, forward_time=0.228, loss_ctc=79.365, loss_interctc_layer6=85.130, loss_interctc_layer12=70.678, loss_interctc_layer15=65.024, loss_interctc_layer21=82.125, loss=76.464, backward_time=0.233, grad_norm=87.595, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.156, optim0_lr0=7.950e-05, train_time=1.285 +[gpua006:0/64] 2024-02-10 01:12:16,454 (trainer:756) INFO: 26epoch:train:4901-5000batch: iter_time=7.821e-05, forward_time=0.159, loss_ctc=67.964, loss_interctc_layer6=78.138, loss_interctc_layer12=64.969, loss_interctc_layer15=59.718, loss_interctc_layer21=69.933, loss=68.144, backward_time=0.213, grad_norm=58.220, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.135, optim0_lr0=7.948e-05, train_time=1.360 +[gpua006:0/64] 2024-02-10 01:12:36,484 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-10 01:12:54,599 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 01:12:57,976 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 01:12:57,977 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-10 01:12:58,024 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 01:18:56,138 (trainer:756) INFO: 26epoch:train:5001-5100batch: iter_time=2.883, forward_time=0.144, loss_ctc=76.172, loss_interctc_layer6=77.317, loss_interctc_layer12=64.396, loss_interctc_layer15=59.114, loss_interctc_layer21=79.252, loss=71.250, backward_time=0.210, grad_norm=65.770, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.947e-05, train_time=3.996 +[gpua006:0/64] 2024-02-10 01:19:38,490 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 01:21:00,622 (trainer:756) INFO: 26epoch:train:5101-5200batch: iter_time=8.361e-05, forward_time=0.142, loss_ctc=66.233, loss_interctc_layer6=82.094, loss_interctc_layer12=68.235, loss_interctc_layer15=62.795, loss_interctc_layer21=68.099, loss=69.491, backward_time=0.209, grad_norm=69.489, clip=100.000, loss_scale=2.458e+31, optim_step_time=0.137, optim0_lr0=7.946e-05, train_time=1.245 +[gpua006:0/64] 2024-02-10 01:22:04,232 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 01:22:41,573 (trainer:756) INFO: 26epoch:train:5201-5300batch: iter_time=8.251e-05, forward_time=0.143, loss_ctc=77.539, loss_interctc_layer6=81.076, loss_interctc_layer12=67.456, loss_interctc_layer15=61.987, loss_interctc_layer21=80.094, loss=73.630, backward_time=0.211, grad_norm=106.010, clip=100.000, loss_scale=1.680e+31, optim_step_time=0.137, optim0_lr0=7.945e-05, train_time=1.009 +[gpua006:0/64] 2024-02-10 01:24:45,837 (trainer:756) INFO: 26epoch:train:5301-5400batch: iter_time=8.246e-05, forward_time=0.142, loss_ctc=84.155, loss_interctc_layer6=84.740, loss_interctc_layer12=70.108, loss_interctc_layer15=64.251, loss_interctc_layer21=86.839, loss=78.019, backward_time=0.210, grad_norm=61.303, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.944e-05, train_time=1.242 +[gpua006:0/64] 2024-02-10 01:27:10,468 (trainer:756) INFO: 26epoch:train:5401-5500batch: iter_time=8.242e-05, forward_time=0.258, loss_ctc=67.894, loss_interctc_layer6=79.707, loss_interctc_layer12=65.692, loss_interctc_layer15=60.120, loss_interctc_layer21=69.991, loss=68.681, backward_time=0.242, grad_norm=59.157, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.159, optim0_lr0=7.943e-05, train_time=1.445 +[gpua006:0/64] 2024-02-10 01:29:18,494 (trainer:756) INFO: 26epoch:train:5501-5600batch: iter_time=8.133e-05, forward_time=0.143, loss_ctc=85.941, loss_interctc_layer6=87.940, loss_interctc_layer12=72.561, loss_interctc_layer15=66.367, loss_interctc_layer21=88.740, loss=80.310, backward_time=0.208, grad_norm=113.789, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.942e-05, train_time=1.281 +[gpua006:0/64] 2024-02-10 01:31:37,110 (trainer:756) INFO: 26epoch:train:5601-5700batch: iter_time=7.853e-05, forward_time=0.166, loss_ctc=78.627, loss_interctc_layer6=81.547, loss_interctc_layer12=67.160, loss_interctc_layer15=61.402, loss_interctc_layer21=81.257, loss=73.999, backward_time=0.210, grad_norm=115.256, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.941e-05, train_time=1.386 +[gpua006:0/64] 2024-02-10 01:33:49,855 (trainer:756) INFO: 26epoch:train:5701-5800batch: iter_time=8.308e-05, forward_time=0.143, loss_ctc=78.221, loss_interctc_layer6=81.957, loss_interctc_layer12=68.185, loss_interctc_layer15=62.607, loss_interctc_layer21=80.943, loss=74.383, backward_time=0.210, grad_norm=65.084, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.940e-05, train_time=1.327 +[gpua006:0/64] 2024-02-10 01:35:31,209 (trainer:756) INFO: 26epoch:train:5801-5900batch: iter_time=7.961e-05, forward_time=0.143, loss_ctc=74.743, loss_interctc_layer6=84.146, loss_interctc_layer12=69.939, loss_interctc_layer15=64.122, loss_interctc_layer21=77.107, loss=74.011, backward_time=0.210, grad_norm=61.443, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.939e-05, train_time=1.013 +[gpua006:0/64] 2024-02-10 01:37:45,837 (trainer:756) INFO: 26epoch:train:5901-6000batch: iter_time=8.451e-05, forward_time=0.143, loss_ctc=87.169, loss_interctc_layer6=96.254, loss_interctc_layer12=80.120, loss_interctc_layer15=73.944, loss_interctc_layer21=90.152, loss=85.528, backward_time=0.210, grad_norm=93.797, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.938e-05, train_time=1.346 +[gpua006:0/64] 2024-02-10 01:40:45,174 (trainer:756) INFO: 26epoch:train:6001-6100batch: iter_time=8.866e-05, forward_time=0.260, loss_ctc=78.104, loss_interctc_layer6=78.371, loss_interctc_layer12=64.809, loss_interctc_layer15=59.283, loss_interctc_layer21=80.972, loss=72.308, backward_time=0.259, grad_norm=56.323, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.144, optim0_lr0=7.937e-05, train_time=1.793 +[gpua006:0/64] 2024-02-10 01:42:48,012 (trainer:756) INFO: 26epoch:train:6101-6200batch: iter_time=8.181e-05, forward_time=0.144, loss_ctc=79.255, loss_interctc_layer6=87.341, loss_interctc_layer12=72.922, loss_interctc_layer15=67.126, loss_interctc_layer21=81.869, loss=77.703, backward_time=0.212, grad_norm=69.853, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.936e-05, train_time=1.226 +[gpua006:0/64] 2024-02-10 01:43:57,205 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-10 01:44:15,445 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 01:44:19,048 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 01:44:19,049 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-10 01:44:19,052 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 01:49:50,461 (trainer:756) INFO: 26epoch:train:6201-6300batch: iter_time=3.140, forward_time=0.171, loss_ctc=74.525, loss_interctc_layer6=78.679, loss_interctc_layer12=65.264, loss_interctc_layer15=59.824, loss_interctc_layer21=76.978, loss=71.054, backward_time=0.216, grad_norm=68.184, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.935e-05, train_time=4.225 +[gpua006:0/64] 2024-02-10 01:51:36,646 (trainer:756) INFO: 26epoch:train:6301-6400batch: iter_time=8.397e-05, forward_time=0.141, loss_ctc=63.227, loss_interctc_layer6=75.395, loss_interctc_layer12=63.198, loss_interctc_layer15=58.106, loss_interctc_layer21=65.321, loss=65.049, backward_time=0.209, grad_norm=74.843, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.934e-05, train_time=1.063 +[gpua006:0/64] 2024-02-10 01:53:15,987 (trainer:756) INFO: 26epoch:train:6401-6500batch: iter_time=8.042e-05, forward_time=0.142, loss_ctc=71.001, loss_interctc_layer6=81.651, loss_interctc_layer12=67.495, loss_interctc_layer15=61.817, loss_interctc_layer21=73.214, loss=71.035, backward_time=0.211, grad_norm=83.449, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.933e-05, train_time=0.993 +[gpua006:0/64] 2024-02-10 01:55:44,836 (trainer:756) INFO: 26epoch:train:6501-6600batch: iter_time=7.893e-05, forward_time=0.142, loss_ctc=77.942, loss_interctc_layer6=84.025, loss_interctc_layer12=69.864, loss_interctc_layer15=64.132, loss_interctc_layer21=80.495, loss=75.292, backward_time=0.209, grad_norm=73.062, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.932e-05, train_time=1.488 +[gpua006:0/64] 2024-02-10 01:57:56,780 (trainer:756) INFO: 26epoch:train:6601-6700batch: iter_time=4.710e-04, forward_time=0.262, loss_ctc=85.798, loss_interctc_layer6=83.442, loss_interctc_layer12=68.634, loss_interctc_layer15=62.727, loss_interctc_layer21=88.899, loss=77.900, backward_time=0.242, grad_norm=79.881, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.146, optim0_lr0=7.931e-05, train_time=1.318 +[gpua006:0/64] 2024-02-10 02:00:46,709 (trainer:756) INFO: 26epoch:train:6701-6800batch: iter_time=8.269e-05, forward_time=0.143, loss_ctc=76.370, loss_interctc_layer6=80.740, loss_interctc_layer12=66.520, loss_interctc_layer15=60.936, loss_interctc_layer21=78.921, loss=72.697, backward_time=0.208, grad_norm=58.883, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.930e-05, train_time=1.700 +[gpua006:0/64] 2024-02-10 02:03:04,443 (trainer:756) INFO: 26epoch:train:6801-6900batch: iter_time=7.799e-05, forward_time=0.142, loss_ctc=81.774, loss_interctc_layer6=84.925, loss_interctc_layer12=70.056, loss_interctc_layer15=64.233, loss_interctc_layer21=84.437, loss=77.085, backward_time=0.208, grad_norm=114.250, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.929e-05, train_time=1.377 +[gpua006:0/64] 2024-02-10 02:05:03,373 (trainer:756) INFO: 26epoch:train:6901-7000batch: iter_time=7.827e-05, forward_time=0.142, loss_ctc=79.876, loss_interctc_layer6=86.833, loss_interctc_layer12=71.791, loss_interctc_layer15=65.661, loss_interctc_layer21=82.567, loss=77.346, backward_time=0.210, grad_norm=67.414, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.928e-05, train_time=1.189 +[gpua006:0/64] 2024-02-10 02:07:09,704 (trainer:756) INFO: 26epoch:train:7001-7100batch: iter_time=7.816e-05, forward_time=0.142, loss_ctc=65.156, loss_interctc_layer6=76.125, loss_interctc_layer12=63.181, loss_interctc_layer15=58.087, loss_interctc_layer21=67.312, loss=65.972, backward_time=0.210, grad_norm=62.381, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.927e-05, train_time=1.263 +[gpua006:0/64] 2024-02-10 02:09:30,402 (trainer:756) INFO: 26epoch:train:7101-7200batch: iter_time=3.822e-04, forward_time=0.242, loss_ctc=96.090, loss_interctc_layer6=102.585, loss_interctc_layer12=85.733, loss_interctc_layer15=79.257, loss_interctc_layer21=99.465, loss=92.626, backward_time=0.263, grad_norm=188.653, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.148, optim0_lr0=7.926e-05, train_time=1.406 +[gpua006:0/64] 2024-02-10 02:11:53,228 (trainer:756) INFO: 26epoch:train:7201-7300batch: iter_time=8.333e-05, forward_time=0.142, loss_ctc=70.035, loss_interctc_layer6=77.046, loss_interctc_layer12=63.767, loss_interctc_layer15=58.567, loss_interctc_layer21=72.523, loss=68.388, backward_time=0.209, grad_norm=65.886, clip=100.000, loss_scale=1.359e+31, optim_step_time=0.136, optim0_lr0=7.925e-05, train_time=1.429 +[gpua006:0/64] 2024-02-10 02:14:13,442 (trainer:756) INFO: 26epoch:train:7301-7400batch: iter_time=7.970e-05, forward_time=0.142, loss_ctc=82.646, loss_interctc_layer6=84.313, loss_interctc_layer12=69.985, loss_interctc_layer15=64.304, loss_interctc_layer21=85.562, loss=77.362, backward_time=0.208, grad_norm=68.133, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.923e-05, train_time=1.402 +[gpua006:0/64] 2024-02-10 02:16:34,811 (trainer:756) INFO: 26epoch:train:7401-7500batch: iter_time=8.333e-05, forward_time=0.142, loss_ctc=68.838, loss_interctc_layer6=77.077, loss_interctc_layer12=64.028, loss_interctc_layer15=58.776, loss_interctc_layer21=70.908, loss=67.925, backward_time=0.209, grad_norm=67.727, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.922e-05, train_time=1.413 +[gpua006:0/64] 2024-02-10 02:16:54,867 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-10 02:17:13,148 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 02:17:16,551 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 02:17:16,552 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-10 02:17:16,555 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 02:23:55,680 (trainer:756) INFO: 26epoch:train:7501-7600batch: iter_time=3.217, forward_time=0.142, loss_ctc=75.679, loss_interctc_layer6=77.244, loss_interctc_layer12=64.178, loss_interctc_layer15=58.975, loss_interctc_layer21=78.526, loss=70.920, backward_time=0.210, grad_norm=82.293, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.921e-05, train_time=4.408 +[gpua006:0/64] 2024-02-10 02:26:04,568 (trainer:756) INFO: 26epoch:train:7601-7700batch: iter_time=8.315e-05, forward_time=0.142, loss_ctc=65.261, loss_interctc_layer6=81.399, loss_interctc_layer12=67.547, loss_interctc_layer15=62.027, loss_interctc_layer21=67.210, loss=68.689, backward_time=0.210, grad_norm=63.070, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.920e-05, train_time=1.289 +[gpua006:0/64] 2024-02-10 02:27:46,658 (trainer:756) INFO: 26epoch:train:7701-7800batch: iter_time=8.208e-05, forward_time=0.141, loss_ctc=76.847, loss_interctc_layer6=80.971, loss_interctc_layer12=67.296, loss_interctc_layer15=61.896, loss_interctc_layer21=79.367, loss=73.275, backward_time=0.209, grad_norm=67.706, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.919e-05, train_time=1.021 +[gpua006:0/64] 2024-02-10 02:28:16,268 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 02:30:09,740 (trainer:756) INFO: 26epoch:train:7801-7900batch: iter_time=9.300e-05, forward_time=0.270, loss_ctc=84.310, loss_interctc_layer6=84.864, loss_interctc_layer12=70.054, loss_interctc_layer15=64.123, loss_interctc_layer21=87.203, loss=78.111, backward_time=0.237, grad_norm=203.256, clip=100.000, loss_scale=1.270e+31, optim_step_time=0.169, optim0_lr0=7.918e-05, train_time=1.431 +[gpua006:0/64] 2024-02-10 02:32:40,352 (trainer:756) INFO: 26epoch:train:7901-8000batch: iter_time=8.425e-05, forward_time=0.142, loss_ctc=66.819, loss_interctc_layer6=78.867, loss_interctc_layer12=64.987, loss_interctc_layer15=59.507, loss_interctc_layer21=68.843, loss=67.805, backward_time=0.209, grad_norm=71.292, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.917e-05, train_time=1.500 +[gpua006:0/64] 2024-02-10 02:34:59,862 (trainer:756) INFO: 26epoch:train:8001-8100batch: iter_time=8.246e-05, forward_time=0.143, loss_ctc=85.406, loss_interctc_layer6=87.724, loss_interctc_layer12=72.503, loss_interctc_layer15=66.393, loss_interctc_layer21=88.333, loss=80.072, backward_time=0.208, grad_norm=75.371, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.916e-05, train_time=1.400 +[gpua006:0/64] 2024-02-10 02:37:38,380 (trainer:756) INFO: 26epoch:train:8101-8200batch: iter_time=8.278e-05, forward_time=0.142, loss_ctc=77.652, loss_interctc_layer6=81.859, loss_interctc_layer12=67.394, loss_interctc_layer15=61.529, loss_interctc_layer21=80.307, loss=73.748, backward_time=0.207, grad_norm=55.518, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.915e-05, train_time=1.585 +[gpua006:0/64] 2024-02-10 02:40:04,451 (trainer:756) INFO: 26epoch:train:8201-8300batch: iter_time=8.400e-05, forward_time=0.142, loss_ctc=77.579, loss_interctc_layer6=81.899, loss_interctc_layer12=68.034, loss_interctc_layer15=62.447, loss_interctc_layer21=80.305, loss=74.053, backward_time=0.208, grad_norm=71.157, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.914e-05, train_time=1.460 +[gpua006:0/64] 2024-02-10 02:41:47,145 (trainer:756) INFO: 26epoch:train:8301-8400batch: iter_time=7.987e-05, forward_time=0.143, loss_ctc=74.646, loss_interctc_layer6=84.108, loss_interctc_layer12=69.813, loss_interctc_layer15=64.034, loss_interctc_layer21=76.951, loss=73.910, backward_time=0.211, grad_norm=72.813, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.913e-05, train_time=1.027 +[gpua006:0/64] 2024-02-10 02:44:18,625 (trainer:756) INFO: 26epoch:train:8401-8500batch: iter_time=8.633e-05, forward_time=0.142, loss_ctc=87.182, loss_interctc_layer6=96.332, loss_interctc_layer12=80.301, loss_interctc_layer15=74.200, loss_interctc_layer21=90.487, loss=85.700, backward_time=0.209, grad_norm=100.572, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.912e-05, train_time=1.515 +[gpua006:0/64] 2024-02-10 02:46:05,336 (trainer:756) INFO: 26epoch:train:8501-8600batch: iter_time=7.577e-05, forward_time=0.141, loss_ctc=77.586, loss_interctc_layer6=78.053, loss_interctc_layer12=64.524, loss_interctc_layer15=59.042, loss_interctc_layer21=80.445, loss=71.930, backward_time=0.209, grad_norm=66.475, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.911e-05, train_time=1.067 +[gpua006:0/64] 2024-02-10 02:48:12,742 (trainer:756) INFO: 26epoch:train:8601-8700batch: iter_time=8.041e-05, forward_time=0.141, loss_ctc=79.230, loss_interctc_layer6=87.988, loss_interctc_layer12=73.152, loss_interctc_layer15=67.405, loss_interctc_layer21=81.733, loss=77.902, backward_time=0.208, grad_norm=67.327, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.910e-05, train_time=1.274 +[gpua006:0/64] 2024-02-10 02:50:07,408 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-10 02:50:25,857 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 02:50:29,236 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 02:50:29,236 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-10 02:50:29,340 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 02:56:35,665 (trainer:756) INFO: 26epoch:train:8701-8800batch: iter_time=3.206, forward_time=0.259, loss_ctc=71.538, loss_interctc_layer6=77.958, loss_interctc_layer12=64.482, loss_interctc_layer15=59.056, loss_interctc_layer21=74.125, loss=69.432, backward_time=0.239, grad_norm=60.401, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.155, optim0_lr0=7.909e-05, train_time=5.027 +[gpua006:0/64] 2024-02-10 02:58:51,146 (trainer:756) INFO: 26epoch:train:8801-8900batch: iter_time=8.526e-05, forward_time=0.142, loss_ctc=61.307, loss_interctc_layer6=75.074, loss_interctc_layer12=62.755, loss_interctc_layer15=57.868, loss_interctc_layer21=63.082, loss=64.017, backward_time=0.209, grad_norm=56.817, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.908e-05, train_time=1.357 +[gpua006:0/64] 2024-02-10 03:00:26,222 (trainer:756) INFO: 26epoch:train:8901-9000batch: iter_time=7.923e-05, forward_time=0.141, loss_ctc=69.899, loss_interctc_layer6=81.707, loss_interctc_layer12=67.626, loss_interctc_layer15=61.937, loss_interctc_layer21=72.171, loss=70.668, backward_time=0.210, grad_norm=124.590, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.907e-05, train_time=0.951 +[gpua006:0/64] 2024-02-10 03:02:08,788 (trainer:756) INFO: 26epoch:train:9001-9100batch: iter_time=8.278e-05, forward_time=0.141, loss_ctc=72.697, loss_interctc_layer6=83.861, loss_interctc_layer12=69.659, loss_interctc_layer15=63.870, loss_interctc_layer21=75.181, loss=73.054, backward_time=0.209, grad_norm=69.125, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.906e-05, train_time=1.025 +[gpua006:0/64] 2024-02-10 03:04:09,657 (trainer:756) INFO: 26epoch:train:9101-9200batch: iter_time=8.207e-05, forward_time=0.141, loss_ctc=81.332, loss_interctc_layer6=83.140, loss_interctc_layer12=68.372, loss_interctc_layer15=62.448, loss_interctc_layer21=84.054, loss=75.869, backward_time=0.207, grad_norm=77.553, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.905e-05, train_time=1.208 +[gpua006:0/64] 2024-02-10 03:06:55,564 (trainer:756) INFO: 26epoch:train:9201-9300batch: iter_time=8.839e-05, forward_time=0.141, loss_ctc=74.024, loss_interctc_layer6=80.046, loss_interctc_layer12=65.726, loss_interctc_layer15=60.135, loss_interctc_layer21=76.638, loss=71.314, backward_time=0.207, grad_norm=94.016, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.904e-05, train_time=1.659 +[gpua006:0/64] 2024-02-10 03:09:36,752 (trainer:756) INFO: 26epoch:train:9301-9400batch: iter_time=9.204e-05, forward_time=0.142, loss_ctc=78.725, loss_interctc_layer6=83.632, loss_interctc_layer12=68.914, loss_interctc_layer15=63.053, loss_interctc_layer21=81.441, loss=75.153, backward_time=0.207, grad_norm=77.154, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.903e-05, train_time=1.612 +[gpua006:0/64] 2024-02-10 03:11:58,386 (trainer:756) INFO: 26epoch:train:9401-9500batch: iter_time=8.323e-05, forward_time=0.142, loss_ctc=75.951, loss_interctc_layer6=86.494, loss_interctc_layer12=71.431, loss_interctc_layer15=65.304, loss_interctc_layer21=78.664, loss=75.569, backward_time=0.208, grad_norm=77.112, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.902e-05, train_time=1.416 +[gpua006:0/64] 2024-02-10 03:14:08,718 (trainer:756) INFO: 26epoch:train:9501-9600batch: iter_time=8.213e-05, forward_time=0.252, loss_ctc=62.776, loss_interctc_layer6=75.353, loss_interctc_layer12=62.574, loss_interctc_layer15=57.412, loss_interctc_layer21=64.755, loss=64.574, backward_time=0.232, grad_norm=66.933, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=7.901e-05, train_time=1.303 +[gpua006:0/64] 2024-02-10 03:16:37,589 (trainer:756) INFO: 26epoch:train:9601-9700batch: iter_time=7.940e-05, forward_time=0.142, loss_ctc=92.985, loss_interctc_layer6=102.044, loss_interctc_layer12=85.675, loss_interctc_layer15=79.087, loss_interctc_layer21=96.178, loss=91.194, backward_time=0.207, grad_norm=88.556, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.900e-05, train_time=1.488 +[gpua006:0/64] 2024-02-10 03:17:22,047 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 03:18:32,451 (trainer:756) INFO: 26epoch:train:9701-9800batch: iter_time=7.558e-05, forward_time=0.142, loss_ctc=66.571, loss_interctc_layer6=77.374, loss_interctc_layer12=64.054, loss_interctc_layer15=58.908, loss_interctc_layer21=68.988, loss=67.179, backward_time=0.209, grad_norm=59.574, clip=100.000, loss_scale=6.863e+30, optim_step_time=0.137, optim0_lr0=7.899e-05, train_time=1.149 +[gpua006:0/64] 2024-02-10 03:20:50,608 (trainer:756) INFO: 26epoch:train:9801-9900batch: iter_time=7.893e-05, forward_time=0.141, loss_ctc=78.879, loss_interctc_layer6=84.220, loss_interctc_layer12=69.817, loss_interctc_layer15=64.068, loss_interctc_layer21=81.729, loss=75.742, backward_time=0.208, grad_norm=80.071, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.898e-05, train_time=1.381 +[gpua006:0/64] 2024-02-10 03:23:00,567 (trainer:756) INFO: 26epoch:train:9901-10000batch: iter_time=7.954e-05, forward_time=0.141, loss_ctc=66.991, loss_interctc_layer6=77.093, loss_interctc_layer12=63.887, loss_interctc_layer15=58.605, loss_interctc_layer21=69.116, loss=67.138, backward_time=0.207, grad_norm=149.679, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.897e-05, train_time=1.299 +[gpua006:0/64] 2024-02-10 03:23:20,596 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-10 03:23:39,247 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 03:23:42,624 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 03:23:42,624 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-10 03:23:42,629 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 03:30:39,594 (trainer:756) INFO: 26epoch:train:10001-10100batch: iter_time=3.134, forward_time=0.209, loss_ctc=70.904, loss_interctc_layer6=77.257, loss_interctc_layer12=64.154, loss_interctc_layer15=58.951, loss_interctc_layer21=73.668, loss=68.987, backward_time=0.225, grad_norm=83.653, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.896e-05, train_time=4.589 +[gpua006:0/64] 2024-02-10 03:32:44,439 (trainer:756) INFO: 26epoch:train:10101-10200batch: iter_time=7.822e-05, forward_time=0.142, loss_ctc=64.699, loss_interctc_layer6=80.994, loss_interctc_layer12=67.243, loss_interctc_layer15=61.827, loss_interctc_layer21=66.606, loss=68.274, backward_time=0.210, grad_norm=60.304, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.895e-05, train_time=1.249 +[gpua006:0/64] 2024-02-10 03:35:20,018 (trainer:756) INFO: 26epoch:train:10201-10300batch: iter_time=8.053e-05, forward_time=0.218, loss_ctc=72.642, loss_interctc_layer6=79.850, loss_interctc_layer12=66.337, loss_interctc_layer15=60.887, loss_interctc_layer21=75.134, loss=70.970, backward_time=0.249, grad_norm=73.801, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.153, optim0_lr0=7.894e-05, train_time=1.555 +[gpua006:0/64] 2024-02-10 03:37:53,410 (trainer:756) INFO: 26epoch:train:10301-10400batch: iter_time=8.432e-05, forward_time=0.142, loss_ctc=82.291, loss_interctc_layer6=84.664, loss_interctc_layer12=69.809, loss_interctc_layer15=63.827, loss_interctc_layer21=85.056, loss=77.129, backward_time=0.208, grad_norm=62.358, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.893e-05, train_time=1.534 +[gpua006:0/64] 2024-02-10 03:40:02,732 (trainer:756) INFO: 26epoch:train:10401-10500batch: iter_time=8.485e-05, forward_time=0.142, loss_ctc=65.706, loss_interctc_layer6=79.000, loss_interctc_layer12=65.099, loss_interctc_layer15=59.565, loss_interctc_layer21=67.800, loss=67.434, backward_time=0.208, grad_norm=62.594, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.892e-05, train_time=1.293 +[gpua006:0/64] 2024-02-10 03:43:29,515 (trainer:756) INFO: 26epoch:train:10501-10600batch: iter_time=1.011e-04, forward_time=0.237, loss_ctc=81.689, loss_interctc_layer6=87.271, loss_interctc_layer12=71.927, loss_interctc_layer15=65.728, loss_interctc_layer21=84.437, loss=78.210, backward_time=0.383, grad_norm=65.676, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.146, optim0_lr0=7.891e-05, train_time=2.067 +[gpua006:0/64] 2024-02-10 03:45:16,984 (trainer:756) INFO: 26epoch:train:10601-10700batch: iter_time=8.255e-05, forward_time=0.143, loss_ctc=73.689, loss_interctc_layer6=81.658, loss_interctc_layer12=67.168, loss_interctc_layer15=61.369, loss_interctc_layer21=76.510, loss=72.079, backward_time=0.210, grad_norm=72.761, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.890e-05, train_time=1.075 +[gpua006:0/64] 2024-02-10 03:46:58,746 (trainer:756) INFO: 26epoch:train:10701-10800batch: iter_time=8.875e-05, forward_time=0.142, loss_ctc=74.995, loss_interctc_layer6=82.011, loss_interctc_layer12=68.134, loss_interctc_layer15=62.631, loss_interctc_layer21=77.468, loss=73.048, backward_time=0.209, grad_norm=73.868, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.889e-05, train_time=1.017 +[gpua006:0/64] 2024-02-10 03:50:15,491 (trainer:756) INFO: 26epoch:train:10801-10900batch: iter_time=8.181e-05, forward_time=0.239, loss_ctc=72.266, loss_interctc_layer6=84.530, loss_interctc_layer12=70.260, loss_interctc_layer15=64.449, loss_interctc_layer21=74.552, loss=73.212, backward_time=0.228, grad_norm=133.966, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.141, optim0_lr0=7.887e-05, train_time=1.966 +[gpua006:0/64] 2024-02-10 03:52:14,340 (trainer:756) INFO: 26epoch:train:10901-11000batch: iter_time=8.626e-05, forward_time=0.143, loss_ctc=82.683, loss_interctc_layer6=96.033, loss_interctc_layer12=80.050, loss_interctc_layer15=74.188, loss_interctc_layer21=85.267, loss=83.644, backward_time=0.208, grad_norm=81.951, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.886e-05, train_time=1.189 +[gpua006:0/64] 2024-02-10 03:54:35,352 (trainer:756) INFO: 26epoch:train:11001-11100batch: iter_time=5.719e-04, forward_time=0.214, loss_ctc=73.418, loss_interctc_layer6=78.237, loss_interctc_layer12=64.498, loss_interctc_layer15=59.008, loss_interctc_layer21=76.258, loss=70.284, backward_time=0.253, grad_norm=62.979, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.142, optim0_lr0=7.885e-05, train_time=1.408 +[gpua006:0/64] 2024-02-10 03:57:02,359 (trainer:756) INFO: 26epoch:train:11101-11200batch: iter_time=7.977e-05, forward_time=0.142, loss_ctc=75.877, loss_interctc_layer6=87.229, loss_interctc_layer12=72.519, loss_interctc_layer15=66.787, loss_interctc_layer21=78.356, loss=76.154, backward_time=0.207, grad_norm=73.199, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.884e-05, train_time=1.472 +[gpua006:0/64] 2024-02-10 03:58:49,092 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-10 03:59:07,354 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 03:59:10,836 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 03:59:10,836 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-10 03:59:10,839 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 04:04:51,846 (trainer:756) INFO: 26epoch:train:11201-11300batch: iter_time=3.292, forward_time=0.172, loss_ctc=74.023, loss_interctc_layer6=78.258, loss_interctc_layer12=64.827, loss_interctc_layer15=59.390, loss_interctc_layer21=76.676, loss=70.635, backward_time=0.217, grad_norm=65.736, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.883e-05, train_time=4.695 +[gpua006:0/64] 2024-02-10 04:06:27,649 (trainer:756) INFO: 26epoch:train:11301-11400batch: iter_time=8.047e-05, forward_time=0.142, loss_ctc=62.581, loss_interctc_layer6=74.300, loss_interctc_layer12=62.088, loss_interctc_layer15=57.182, loss_interctc_layer21=64.497, loss=64.130, backward_time=0.209, grad_norm=60.540, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.882e-05, train_time=0.958 +[gpua006:0/64] 2024-02-10 04:08:15,673 (trainer:756) INFO: 26epoch:train:11401-11500batch: iter_time=8.050e-05, forward_time=0.142, loss_ctc=71.704, loss_interctc_layer6=81.547, loss_interctc_layer12=67.422, loss_interctc_layer15=61.769, loss_interctc_layer21=73.899, loss=71.268, backward_time=0.210, grad_norm=57.068, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.881e-05, train_time=1.080 +[gpua006:0/64] 2024-02-10 04:11:04,392 (trainer:756) INFO: 26epoch:train:11501-11600batch: iter_time=4.417e-04, forward_time=0.211, loss_ctc=77.291, loss_interctc_layer6=83.496, loss_interctc_layer12=69.221, loss_interctc_layer15=63.522, loss_interctc_layer21=79.854, loss=74.677, backward_time=0.285, grad_norm=65.385, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.144, optim0_lr0=7.880e-05, train_time=1.687 +[gpua006:0/64] 2024-02-10 04:13:33,176 (trainer:756) INFO: 26epoch:train:11601-11700batch: iter_time=8.003e-05, forward_time=0.143, loss_ctc=83.419, loss_interctc_layer6=82.078, loss_interctc_layer12=67.433, loss_interctc_layer15=61.612, loss_interctc_layer21=86.330, loss=76.174, backward_time=0.208, grad_norm=60.813, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.879e-05, train_time=1.488 +[gpua006:0/64] 2024-02-10 04:15:51,465 (trainer:756) INFO: 26epoch:train:11701-11800batch: iter_time=8.373e-05, forward_time=0.142, loss_ctc=75.861, loss_interctc_layer6=80.050, loss_interctc_layer12=65.937, loss_interctc_layer15=60.266, loss_interctc_layer21=78.677, loss=72.158, backward_time=0.209, grad_norm=77.146, clip=100.000, loss_scale=8.316e+30, optim_step_time=0.136, optim0_lr0=7.878e-05, train_time=1.383 +[gpua006:0/64] 2024-02-10 04:18:28,258 (trainer:756) INFO: 26epoch:train:11801-11900batch: iter_time=8.312e-05, forward_time=0.144, loss_ctc=81.409, loss_interctc_layer6=83.811, loss_interctc_layer12=69.164, loss_interctc_layer15=63.379, loss_interctc_layer21=84.439, loss=76.440, backward_time=0.207, grad_norm=72.221, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.877e-05, train_time=1.568 +[gpua006:0/64] 2024-02-10 04:20:54,079 (trainer:756) INFO: 26epoch:train:11901-12000batch: iter_time=9.784e-05, forward_time=0.204, loss_ctc=79.113, loss_interctc_layer6=86.096, loss_interctc_layer12=71.043, loss_interctc_layer15=64.917, loss_interctc_layer21=81.924, loss=76.618, backward_time=0.287, grad_norm=262.589, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.145, optim0_lr0=7.876e-05, train_time=1.456 +[gpua006:0/64] 2024-02-10 04:22:44,982 (trainer:756) INFO: 26epoch:train:12001-12100batch: iter_time=8.444e-05, forward_time=0.142, loss_ctc=64.707, loss_interctc_layer6=75.367, loss_interctc_layer12=62.585, loss_interctc_layer15=57.422, loss_interctc_layer21=66.836, loss=65.383, backward_time=0.210, grad_norm=68.110, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.875e-05, train_time=1.110 +[gpua006:0/64] 2024-02-10 04:25:13,857 (trainer:756) INFO: 26epoch:train:12101-12200batch: iter_time=7.979e-05, forward_time=0.165, loss_ctc=95.675, loss_interctc_layer6=101.984, loss_interctc_layer12=85.425, loss_interctc_layer15=78.918, loss_interctc_layer21=98.731, loss=92.146, backward_time=0.207, grad_norm=99.871, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.874e-05, train_time=1.489 +[gpua006:0/64] 2024-02-10 04:27:43,951 (trainer:756) INFO: 26epoch:train:12201-12300batch: iter_time=2.146e-04, forward_time=0.230, loss_ctc=70.751, loss_interctc_layer6=77.913, loss_interctc_layer12=64.638, loss_interctc_layer15=59.323, loss_interctc_layer21=73.096, loss=69.144, backward_time=0.230, grad_norm=69.316, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=7.873e-05, train_time=1.501 +[gpua006:0/64] 2024-02-10 04:30:19,045 (trainer:756) INFO: 26epoch:train:12301-12400batch: iter_time=8.272e-05, forward_time=0.160, loss_ctc=82.642, loss_interctc_layer6=83.879, loss_interctc_layer12=69.275, loss_interctc_layer15=63.547, loss_interctc_layer21=85.614, loss=76.991, backward_time=0.209, grad_norm=86.052, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.872e-05, train_time=1.549 +[gpua006:0/64] 2024-02-10 04:32:19,844 (trainer:756) INFO: 26epoch:train:12401-12500batch: iter_time=8.164e-05, forward_time=0.142, loss_ctc=68.730, loss_interctc_layer6=77.125, loss_interctc_layer12=63.970, loss_interctc_layer15=58.576, loss_interctc_layer21=70.845, loss=67.849, backward_time=0.209, grad_norm=79.045, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.871e-05, train_time=1.209 +[gpua006:0/64] 2024-02-10 04:32:39,878 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-10 04:32:58,520 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 04:33:01,886 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 04:33:01,887 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-10 04:33:01,890 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 04:38:51,282 (trainer:756) INFO: 26epoch:train:12501-12600batch: iter_time=2.874, forward_time=0.143, loss_ctc=74.698, loss_interctc_layer6=76.426, loss_interctc_layer12=63.498, loss_interctc_layer15=58.355, loss_interctc_layer21=77.425, loss=70.081, backward_time=0.211, grad_norm=110.079, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.870e-05, train_time=3.914 +[gpua006:0/64] 2024-02-10 04:40:46,796 (trainer:756) INFO: 26epoch:train:12601-12700batch: iter_time=7.976e-05, forward_time=0.142, loss_ctc=65.737, loss_interctc_layer6=81.103, loss_interctc_layer12=67.350, loss_interctc_layer15=61.871, loss_interctc_layer21=67.712, loss=68.755, backward_time=0.210, grad_norm=81.563, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.869e-05, train_time=1.155 +[gpua006:0/64] 2024-02-10 04:43:08,600 (trainer:756) INFO: 26epoch:train:12701-12800batch: iter_time=8.092e-05, forward_time=0.239, loss_ctc=77.347, loss_interctc_layer6=80.427, loss_interctc_layer12=66.828, loss_interctc_layer15=61.344, loss_interctc_layer21=79.922, loss=73.174, backward_time=0.267, grad_norm=76.087, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=7.868e-05, train_time=1.417 +[gpua006:0/64] 2024-02-10 04:45:45,447 (trainer:756) INFO: 26epoch:train:12801-12900batch: iter_time=8.378e-05, forward_time=0.142, loss_ctc=84.308, loss_interctc_layer6=84.767, loss_interctc_layer12=69.855, loss_interctc_layer15=63.859, loss_interctc_layer21=87.220, loss=78.002, backward_time=0.208, grad_norm=82.793, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.867e-05, train_time=1.567 +[gpua006:0/64] 2024-02-10 04:48:03,401 (trainer:756) INFO: 26epoch:train:12901-13000batch: iter_time=8.490e-05, forward_time=0.143, loss_ctc=66.877, loss_interctc_layer6=78.793, loss_interctc_layer12=64.756, loss_interctc_layer15=59.067, loss_interctc_layer21=69.025, loss=67.703, backward_time=0.208, grad_norm=58.272, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.866e-05, train_time=1.381 +[gpua006:0/64] 2024-02-10 04:50:18,853 (trainer:756) INFO: 26epoch:train:13001-13100batch: iter_time=8.082e-05, forward_time=0.142, loss_ctc=84.728, loss_interctc_layer6=87.284, loss_interctc_layer12=71.843, loss_interctc_layer15=65.598, loss_interctc_layer21=87.655, loss=79.422, backward_time=0.209, grad_norm=84.221, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.865e-05, train_time=1.354 +[gpua006:0/64] 2024-02-10 04:52:40,184 (trainer:756) INFO: 26epoch:train:13101-13200batch: iter_time=8.320e-05, forward_time=0.143, loss_ctc=76.718, loss_interctc_layer6=81.299, loss_interctc_layer12=66.933, loss_interctc_layer15=61.021, loss_interctc_layer21=79.367, loss=73.068, backward_time=0.209, grad_norm=66.997, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.864e-05, train_time=1.413 +[gpua006:0/64] 2024-02-10 04:54:56,254 (trainer:756) INFO: 26epoch:train:13201-13300batch: iter_time=8.238e-05, forward_time=0.219, loss_ctc=77.892, loss_interctc_layer6=81.406, loss_interctc_layer12=67.408, loss_interctc_layer15=61.841, loss_interctc_layer21=80.796, loss=73.869, backward_time=0.265, grad_norm=63.983, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.148, optim0_lr0=7.863e-05, train_time=1.358 +[gpua006:0/64] 2024-02-10 04:57:17,554 (trainer:756) INFO: 26epoch:train:13301-13400batch: iter_time=8.592e-05, forward_time=0.144, loss_ctc=75.090, loss_interctc_layer6=83.977, loss_interctc_layer12=69.690, loss_interctc_layer15=63.933, loss_interctc_layer21=77.496, loss=74.037, backward_time=0.209, grad_norm=65.598, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.862e-05, train_time=1.415 +[gpua006:0/64] 2024-02-10 05:00:00,625 (trainer:756) INFO: 26epoch:train:13401-13500batch: iter_time=8.129e-05, forward_time=0.146, loss_ctc=85.973, loss_interctc_layer6=95.323, loss_interctc_layer12=79.423, loss_interctc_layer15=73.579, loss_interctc_layer21=88.419, loss=84.544, backward_time=0.210, grad_norm=96.617, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.861e-05, train_time=1.630 +[gpua006:0/64] 2024-02-10 05:02:13,772 (trainer:756) INFO: 26epoch:train:13501-13600batch: iter_time=8.105e-05, forward_time=0.141, loss_ctc=78.495, loss_interctc_layer6=77.816, loss_interctc_layer12=64.194, loss_interctc_layer15=58.740, loss_interctc_layer21=81.436, loss=72.136, backward_time=0.208, grad_norm=68.763, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.860e-05, train_time=1.331 +[gpua006:0/64] 2024-02-10 05:05:08,640 (trainer:756) INFO: 26epoch:train:13601-13700batch: iter_time=8.121e-05, forward_time=0.237, loss_ctc=79.622, loss_interctc_layer6=87.256, loss_interctc_layer12=72.545, loss_interctc_layer15=66.755, loss_interctc_layer21=82.239, loss=77.683, backward_time=0.255, grad_norm=71.495, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.149, optim0_lr0=7.859e-05, train_time=1.748 +[gpua006:0/64] 2024-02-10 05:06:43,144 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-10 05:07:01,873 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 05:07:05,359 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 05:07:05,359 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-10 05:07:05,363 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 05:12:39,966 (trainer:756) INFO: 26epoch:train:13701-13800batch: iter_time=3.136, forward_time=0.147, loss_ctc=71.893, loss_interctc_layer6=77.932, loss_interctc_layer12=64.498, loss_interctc_layer15=58.976, loss_interctc_layer21=74.463, loss=69.553, backward_time=0.210, grad_norm=107.121, clip=100.000, loss_scale=1.663e+31, optim_step_time=0.139, optim0_lr0=7.858e-05, train_time=4.513 +[gpua006:0/64] 2024-02-10 05:14:50,221 (trainer:756) INFO: 26epoch:train:13801-13900batch: iter_time=7.980e-05, forward_time=0.142, loss_ctc=60.788, loss_interctc_layer6=74.786, loss_interctc_layer12=62.457, loss_interctc_layer15=57.504, loss_interctc_layer21=62.731, loss=63.653, backward_time=0.208, grad_norm=66.642, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.857e-05, train_time=1.302 +[gpua006:0/64] 2024-02-10 05:16:39,404 (trainer:756) INFO: 26epoch:train:13901-14000batch: iter_time=8.270e-05, forward_time=0.141, loss_ctc=69.352, loss_interctc_layer6=81.366, loss_interctc_layer12=67.212, loss_interctc_layer15=61.578, loss_interctc_layer21=71.586, loss=70.219, backward_time=0.220, grad_norm=70.878, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.856e-05, train_time=1.092 +[gpua006:0/64] 2024-02-10 05:19:38,656 (trainer:756) INFO: 26epoch:train:14001-14100batch: iter_time=1.140e-04, forward_time=0.253, loss_ctc=71.925, loss_interctc_layer6=83.122, loss_interctc_layer12=68.941, loss_interctc_layer15=63.197, loss_interctc_layer21=74.339, loss=72.305, backward_time=0.261, grad_norm=100.177, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.146, optim0_lr0=7.855e-05, train_time=1.792 +[gpua006:0/64] 2024-02-10 05:22:12,672 (trainer:756) INFO: 26epoch:train:14101-14200batch: iter_time=8.382e-05, forward_time=0.141, loss_ctc=81.014, loss_interctc_layer6=82.445, loss_interctc_layer12=67.719, loss_interctc_layer15=61.706, loss_interctc_layer21=83.892, loss=75.355, backward_time=0.209, grad_norm=85.933, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.854e-05, train_time=1.540 +[gpua006:0/64] 2024-02-10 05:23:43,898 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 05:24:40,411 (trainer:756) INFO: 26epoch:train:14201-14300batch: iter_time=8.288e-05, forward_time=0.143, loss_ctc=73.603, loss_interctc_layer6=80.200, loss_interctc_layer12=65.805, loss_interctc_layer15=60.053, loss_interctc_layer21=76.270, loss=71.186, backward_time=0.208, grad_norm=60.510, clip=100.000, loss_scale=1.608e+31, optim_step_time=0.136, optim0_lr0=7.853e-05, train_time=1.477 +[gpua006:0/64] 2024-02-10 05:26:49,242 (trainer:756) INFO: 26epoch:train:14301-14400batch: iter_time=8.136e-05, forward_time=0.141, loss_ctc=78.964, loss_interctc_layer6=83.391, loss_interctc_layer12=68.768, loss_interctc_layer15=62.886, loss_interctc_layer21=81.583, loss=75.119, backward_time=0.208, grad_norm=64.979, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.852e-05, train_time=1.288 +[gpua006:0/64] 2024-02-10 05:29:38,616 (trainer:756) INFO: 26epoch:train:14401-14500batch: iter_time=8.245e-05, forward_time=0.148, loss_ctc=74.948, loss_interctc_layer6=85.853, loss_interctc_layer12=70.784, loss_interctc_layer15=64.608, loss_interctc_layer21=77.668, loss=74.772, backward_time=0.211, grad_norm=70.059, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=7.851e-05, train_time=1.694 +[gpua006:0/64] 2024-02-10 05:31:52,755 (trainer:756) INFO: 26epoch:train:14501-14600batch: iter_time=8.701e-05, forward_time=0.247, loss_ctc=62.431, loss_interctc_layer6=75.177, loss_interctc_layer12=62.325, loss_interctc_layer15=57.241, loss_interctc_layer21=64.419, loss=64.319, backward_time=0.242, grad_norm=63.122, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.147, optim0_lr0=7.850e-05, train_time=1.339 +[gpua006:0/64] 2024-02-10 05:34:35,513 (trainer:756) INFO: 26epoch:train:14601-14700batch: iter_time=9.270e-05, forward_time=0.144, loss_ctc=92.191, loss_interctc_layer6=100.824, loss_interctc_layer12=84.650, loss_interctc_layer15=78.498, loss_interctc_layer21=94.980, loss=90.229, backward_time=0.208, grad_norm=96.986, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.849e-05, train_time=1.630 +[gpua006:0/64] 2024-02-10 05:36:36,869 (trainer:756) INFO: 26epoch:train:14701-14800batch: iter_time=8.281e-05, forward_time=0.141, loss_ctc=66.339, loss_interctc_layer6=77.302, loss_interctc_layer12=63.977, loss_interctc_layer15=58.767, loss_interctc_layer21=68.668, loss=67.011, backward_time=0.208, grad_norm=67.040, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.848e-05, train_time=1.213 +[gpua006:0/64] 2024-02-10 05:38:49,179 (trainer:756) INFO: 26epoch:train:14801-14900batch: iter_time=8.925e-04, forward_time=0.144, loss_ctc=78.258, loss_interctc_layer6=83.530, loss_interctc_layer12=69.100, loss_interctc_layer15=63.288, loss_interctc_layer21=81.018, loss=75.039, backward_time=0.212, grad_norm=67.306, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.156, optim0_lr0=7.847e-05, train_time=1.323 +[gpua006:0/64] 2024-02-10 05:41:35,834 (trainer:756) INFO: 26epoch:train:14901-15000batch: iter_time=2.457e-04, forward_time=0.168, loss_ctc=67.140, loss_interctc_layer6=77.098, loss_interctc_layer12=63.963, loss_interctc_layer15=58.598, loss_interctc_layer21=69.312, loss=67.222, backward_time=0.299, grad_norm=100.148, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.143, optim0_lr0=7.846e-05, train_time=1.666 +[gpua006:0/64] 2024-02-10 06:13:14,975 (trainer:355) INFO: 26epoch results: [train] iter_time=0.246, forward_time=0.163, loss_ctc=75.696, loss_interctc_layer6=83.100, loss_interctc_layer12=68.930, loss_interctc_layer15=63.290, loss_interctc_layer21=78.223, loss=73.848, backward_time=0.219, grad_norm=78.114, clip=100.000, loss_scale=1.868e+31, optim_step_time=0.139, optim0_lr0=7.922e-05, train_time=1.584, time=6 hours, 36 minutes and 24.55 seconds, total_count=390000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=43.850, cer_ctc=0.207, loss_interctc_layer6=49.925, cer_interctc_layer6=0.222, loss_interctc_layer12=36.851, cer_interctc_layer12=0.155, loss_interctc_layer15=32.382, cer_interctc_layer15=0.130, loss_interctc_layer21=46.377, cer_interctc_layer21=0.217, loss=41.877, time=31 minutes and 15.05 seconds, total_count=121446, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-10 06:13:33,821 (trainer:410) INFO: The best model has been updated: valid.cer_ctc, valid.loss_ctc, valid.total_count +[gpua006:0/64] 2024-02-10 06:13:33,864 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/18epoch.pth, exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/21epoch.pth +[gpua006:0/64] 2024-02-10 06:13:33,864 (trainer:289) INFO: 27/45epoch started. Estimated time to finish: 6 days, 7 hours and 3 minutes +[gpua006:0/64] 2024-02-10 06:13:33,878 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-10 06:13:51,267 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 06:13:54,574 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 06:13:54,574 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-10 06:13:54,577 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 06:20:01,437 (trainer:756) INFO: 27epoch:train:1-100batch: iter_time=2.842, forward_time=0.172, loss_ctc=80.472, loss_interctc_layer6=85.223, loss_interctc_layer12=71.867, loss_interctc_layer15=66.556, loss_interctc_layer21=83.241, loss=77.472, backward_time=0.219, grad_norm=69.139, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.845e-05, train_time=3.875 +[gpua006:0/64] 2024-02-10 06:21:38,426 (trainer:756) INFO: 27epoch:train:101-200batch: iter_time=9.478e-05, forward_time=0.142, loss_ctc=84.417, loss_interctc_layer6=82.841, loss_interctc_layer12=69.006, loss_interctc_layer15=63.377, loss_interctc_layer21=87.216, loss=77.372, backward_time=0.210, grad_norm=99.253, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.844e-05, train_time=0.970 +[gpua006:0/64] 2024-02-10 06:23:40,245 (trainer:756) INFO: 27epoch:train:201-300batch: iter_time=9.254e-05, forward_time=0.142, loss_ctc=89.326, loss_interctc_layer6=86.768, loss_interctc_layer12=72.146, loss_interctc_layer15=66.366, loss_interctc_layer21=92.393, loss=81.400, backward_time=0.211, grad_norm=73.304, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.843e-05, train_time=1.218 +[gpua006:0/64] 2024-02-10 06:25:51,820 (trainer:756) INFO: 27epoch:train:301-400batch: iter_time=8.530e-05, forward_time=0.206, loss_ctc=76.956, loss_interctc_layer6=90.585, loss_interctc_layer12=75.856, loss_interctc_layer15=70.063, loss_interctc_layer21=79.272, loss=78.546, backward_time=0.243, grad_norm=72.029, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.842e-05, train_time=1.313 +[gpua006:0/64] 2024-02-10 06:28:17,410 (trainer:756) INFO: 27epoch:train:401-500batch: iter_time=8.525e-05, forward_time=0.180, loss_ctc=85.951, loss_interctc_layer6=86.650, loss_interctc_layer12=71.948, loss_interctc_layer15=65.992, loss_interctc_layer21=88.796, loss=79.867, backward_time=0.221, grad_norm=62.116, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.841e-05, train_time=1.458 +[gpua006:0/64] 2024-02-10 06:30:58,989 (trainer:756) INFO: 27epoch:train:501-600batch: iter_time=8.956e-05, forward_time=0.154, loss_ctc=76.297, loss_interctc_layer6=83.925, loss_interctc_layer12=70.077, loss_interctc_layer15=64.389, loss_interctc_layer21=78.618, loss=74.661, backward_time=0.226, grad_norm=72.450, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.840e-05, train_time=1.616 +[gpua006:0/64] 2024-02-10 06:33:02,283 (trainer:756) INFO: 27epoch:train:601-700batch: iter_time=5.126e-04, forward_time=0.160, loss_ctc=80.052, loss_interctc_layer6=89.863, loss_interctc_layer12=74.799, loss_interctc_layer15=68.727, loss_interctc_layer21=82.647, loss=79.217, backward_time=0.228, grad_norm=89.655, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.839e-05, train_time=1.233 +[gpua006:0/64] 2024-02-10 06:35:46,283 (trainer:756) INFO: 27epoch:train:701-800batch: iter_time=8.699e-05, forward_time=0.201, loss_ctc=80.675, loss_interctc_layer6=83.273, loss_interctc_layer12=69.683, loss_interctc_layer15=64.332, loss_interctc_layer21=83.157, loss=76.224, backward_time=0.223, grad_norm=76.870, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.838e-05, train_time=1.639 +[gpua006:0/64] 2024-02-10 06:38:02,562 (trainer:756) INFO: 27epoch:train:801-900batch: iter_time=8.879e-05, forward_time=0.167, loss_ctc=83.515, loss_interctc_layer6=95.230, loss_interctc_layer12=80.273, loss_interctc_layer15=74.373, loss_interctc_layer21=86.106, loss=83.899, backward_time=0.219, grad_norm=83.816, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.837e-05, train_time=1.364 +[gpua006:0/64] 2024-02-10 06:40:10,940 (trainer:756) INFO: 27epoch:train:901-1000batch: iter_time=8.606e-05, forward_time=0.143, loss_ctc=93.850, loss_interctc_layer6=84.467, loss_interctc_layer12=70.860, loss_interctc_layer15=65.605, loss_interctc_layer21=96.965, loss=82.349, backward_time=0.207, grad_norm=91.033, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.836e-05, train_time=1.284 +[gpua006:0/64] 2024-02-10 06:42:25,089 (trainer:756) INFO: 27epoch:train:1001-1100batch: iter_time=8.382e-05, forward_time=0.161, loss_ctc=76.426, loss_interctc_layer6=84.494, loss_interctc_layer12=70.646, loss_interctc_layer15=64.977, loss_interctc_layer21=78.906, loss=75.090, backward_time=0.214, grad_norm=72.837, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.835e-05, train_time=1.341 +[gpua006:0/64] 2024-02-10 06:44:44,744 (trainer:756) INFO: 27epoch:train:1101-1200batch: iter_time=8.660e-05, forward_time=0.186, loss_ctc=79.891, loss_interctc_layer6=84.853, loss_interctc_layer12=71.159, loss_interctc_layer15=65.742, loss_interctc_layer21=82.305, loss=76.790, backward_time=0.251, grad_norm=71.865, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.834e-05, train_time=1.395 +[gpua006:0/64] 2024-02-10 06:46:00,844 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-10 06:46:19,295 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 06:46:22,695 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 06:46:22,695 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-10 06:46:22,748 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 06:51:42,440 (trainer:756) INFO: 27epoch:train:1201-1300batch: iter_time=2.947, forward_time=0.147, loss_ctc=77.721, loss_interctc_layer6=86.568, loss_interctc_layer12=71.892, loss_interctc_layer15=65.909, loss_interctc_layer21=80.260, loss=76.470, backward_time=0.211, grad_norm=71.540, clip=100.000, loss_scale=1.430e+31, optim_step_time=0.139, optim0_lr0=7.833e-05, train_time=4.178 +[gpua006:0/64] 2024-02-10 06:54:13,258 (trainer:756) INFO: 27epoch:train:1301-1400batch: iter_time=8.221e-05, forward_time=0.142, loss_ctc=81.534, loss_interctc_layer6=86.903, loss_interctc_layer12=72.999, loss_interctc_layer15=67.400, loss_interctc_layer21=84.383, loss=78.644, backward_time=0.209, grad_norm=77.006, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.832e-05, train_time=1.508 +[gpua006:0/64] 2024-02-10 06:56:45,358 (trainer:756) INFO: 27epoch:train:1401-1500batch: iter_time=8.793e-05, forward_time=0.143, loss_ctc=77.009, loss_interctc_layer6=83.031, loss_interctc_layer12=69.134, loss_interctc_layer15=63.470, loss_interctc_layer21=79.339, loss=74.397, backward_time=0.211, grad_norm=73.097, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=7.831e-05, train_time=1.521 +[gpua006:0/64] 2024-02-10 06:59:21,693 (trainer:756) INFO: 27epoch:train:1501-1600batch: iter_time=8.587e-05, forward_time=0.150, loss_ctc=82.966, loss_interctc_layer6=86.888, loss_interctc_layer12=72.053, loss_interctc_layer15=66.126, loss_interctc_layer21=85.731, loss=78.753, backward_time=0.211, grad_norm=190.089, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=7.830e-05, train_time=1.563 +[gpua006:0/64] 2024-02-10 07:02:34,076 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 07:03:03,631 (trainer:756) INFO: 27epoch:train:1601-1700batch: iter_time=9.009e-05, forward_time=0.173, loss_ctc=88.469, loss_interctc_layer6=94.035, loss_interctc_layer12=78.492, loss_interctc_layer15=72.231, loss_interctc_layer21=91.278, loss=84.901, backward_time=0.232, grad_norm=69.219, clip=100.000, loss_scale=1.864e+31, optim_step_time=0.140, optim0_lr0=7.829e-05, train_time=2.219 +[gpua006:0/64] 2024-02-10 07:05:35,226 (trainer:756) INFO: 27epoch:train:1701-1800batch: iter_time=2.009e-04, forward_time=0.198, loss_ctc=63.717, loss_interctc_layer6=79.448, loss_interctc_layer12=66.143, loss_interctc_layer15=60.866, loss_interctc_layer21=65.412, loss=67.117, backward_time=0.222, grad_norm=78.006, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=7.828e-05, train_time=1.514 +[gpua006:0/64] 2024-02-10 07:08:14,122 (trainer:756) INFO: 27epoch:train:1801-1900batch: iter_time=7.494e-05, forward_time=0.144, loss_ctc=87.724, loss_interctc_layer6=94.069, loss_interctc_layer12=78.249, loss_interctc_layer15=71.882, loss_interctc_layer21=90.635, loss=84.512, backward_time=0.208, grad_norm=86.689, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.827e-05, train_time=1.591 +[gpua006:0/64] 2024-02-10 07:10:46,474 (trainer:756) INFO: 27epoch:train:1901-2000batch: iter_time=8.424e-05, forward_time=0.141, loss_ctc=68.303, loss_interctc_layer6=83.124, loss_interctc_layer12=69.344, loss_interctc_layer15=63.881, loss_interctc_layer21=70.269, loss=70.984, backward_time=0.207, grad_norm=71.258, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.826e-05, train_time=1.523 +[gpua006:0/64] 2024-02-10 07:13:23,257 (trainer:756) INFO: 27epoch:train:2001-2100batch: iter_time=8.574e-05, forward_time=0.141, loss_ctc=81.229, loss_interctc_layer6=90.688, loss_interctc_layer12=76.117, loss_interctc_layer15=70.314, loss_interctc_layer21=83.983, loss=80.466, backward_time=0.209, grad_norm=83.169, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.825e-05, train_time=1.568 +[gpua006:0/64] 2024-02-10 07:15:43,615 (trainer:756) INFO: 27epoch:train:2101-2200batch: iter_time=8.763e-05, forward_time=0.142, loss_ctc=83.244, loss_interctc_layer6=84.893, loss_interctc_layer12=70.946, loss_interctc_layer15=65.496, loss_interctc_layer21=85.585, loss=78.033, backward_time=0.209, grad_norm=64.354, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.824e-05, train_time=1.403 +[gpua006:0/64] 2024-02-10 07:18:07,559 (trainer:756) INFO: 27epoch:train:2201-2300batch: iter_time=8.331e-05, forward_time=0.151, loss_ctc=75.178, loss_interctc_layer6=80.304, loss_interctc_layer12=66.476, loss_interctc_layer15=60.926, loss_interctc_layer21=77.962, loss=72.169, backward_time=0.211, grad_norm=72.039, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.823e-05, train_time=1.440 +[gpua006:0/64] 2024-02-10 07:20:47,130 (trainer:756) INFO: 27epoch:train:2301-2400batch: iter_time=0.045, forward_time=0.153, loss_ctc=77.452, loss_interctc_layer6=87.275, loss_interctc_layer12=73.077, loss_interctc_layer15=67.405, loss_interctc_layer21=79.926, loss=77.027, backward_time=0.213, grad_norm=71.462, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.822e-05, train_time=1.595 +[gpua006:0/64] 2024-02-10 07:21:27,625 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 07:23:10,575 (trainer:756) INFO: 27epoch:train:2401-2500batch: iter_time=7.657e-05, forward_time=0.167, loss_ctc=81.446, loss_interctc_layer6=86.917, loss_interctc_layer12=72.257, loss_interctc_layer15=66.319, loss_interctc_layer21=84.139, loss=78.216, backward_time=0.243, grad_norm=67.152, clip=100.000, loss_scale=6.300e+30, optim_step_time=0.140, optim0_lr0=7.821e-05, train_time=1.434 +[gpua006:0/64] 2024-02-10 07:23:30,601 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-10 07:23:48,822 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 07:23:52,181 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 07:23:52,181 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-10 07:23:52,186 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 07:30:27,407 (trainer:756) INFO: 27epoch:train:2501-2600batch: iter_time=3.099, forward_time=0.179, loss_ctc=79.091, loss_interctc_layer6=83.567, loss_interctc_layer12=70.097, loss_interctc_layer15=64.643, loss_interctc_layer21=81.704, loss=75.820, backward_time=0.216, grad_norm=80.056, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.140, optim0_lr0=7.820e-05, train_time=4.368 +[gpua006:0/64] 2024-02-10 07:32:25,483 (trainer:756) INFO: 27epoch:train:2601-2700batch: iter_time=7.837e-05, forward_time=0.142, loss_ctc=83.741, loss_interctc_layer6=82.641, loss_interctc_layer12=68.582, loss_interctc_layer15=62.789, loss_interctc_layer21=86.746, loss=76.900, backward_time=0.210, grad_norm=69.437, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.819e-05, train_time=1.181 +[gpua006:0/64] 2024-02-10 07:34:20,872 (trainer:756) INFO: 27epoch:train:2701-2800batch: iter_time=8.096e-05, forward_time=0.144, loss_ctc=88.324, loss_interctc_layer6=85.367, loss_interctc_layer12=70.888, loss_interctc_layer15=65.085, loss_interctc_layer21=91.237, loss=80.180, backward_time=0.210, grad_norm=78.710, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.818e-05, train_time=1.154 +[gpua006:0/64] 2024-02-10 07:36:14,521 (trainer:756) INFO: 27epoch:train:2801-2900batch: iter_time=7.531e-05, forward_time=0.142, loss_ctc=76.750, loss_interctc_layer6=90.095, loss_interctc_layer12=75.405, loss_interctc_layer15=69.711, loss_interctc_layer21=79.041, loss=78.201, backward_time=0.210, grad_norm=87.923, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.817e-05, train_time=1.136 +[gpua006:0/64] 2024-02-10 07:38:06,933 (trainer:756) INFO: 27epoch:train:2901-3000batch: iter_time=7.981e-05, forward_time=0.148, loss_ctc=85.534, loss_interctc_layer6=86.172, loss_interctc_layer12=71.379, loss_interctc_layer15=65.458, loss_interctc_layer21=88.257, loss=79.360, backward_time=0.214, grad_norm=147.156, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.816e-05, train_time=1.124 +[gpua006:0/64] 2024-02-10 07:40:13,439 (trainer:756) INFO: 27epoch:train:3001-3100batch: iter_time=8.251e-05, forward_time=0.150, loss_ctc=75.186, loss_interctc_layer6=83.013, loss_interctc_layer12=69.117, loss_interctc_layer15=63.481, loss_interctc_layer21=77.617, loss=73.683, backward_time=0.213, grad_norm=66.638, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.815e-05, train_time=1.265 +[gpua006:0/64] 2024-02-10 07:42:37,957 (trainer:756) INFO: 27epoch:train:3101-3200batch: iter_time=7.950e-05, forward_time=0.174, loss_ctc=79.107, loss_interctc_layer6=89.469, loss_interctc_layer12=74.356, loss_interctc_layer15=68.447, loss_interctc_layer21=81.759, loss=78.627, backward_time=0.217, grad_norm=64.631, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.814e-05, train_time=1.445 +[gpua006:0/64] 2024-02-10 07:44:47,328 (trainer:756) INFO: 27epoch:train:3201-3300batch: iter_time=7.431e-05, forward_time=0.145, loss_ctc=79.838, loss_interctc_layer6=82.161, loss_interctc_layer12=68.534, loss_interctc_layer15=63.172, loss_interctc_layer21=82.456, loss=75.232, backward_time=0.232, grad_norm=60.345, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.813e-05, train_time=1.293 +[gpua006:0/64] 2024-02-10 07:46:56,769 (trainer:756) INFO: 27epoch:train:3301-3400batch: iter_time=7.806e-05, forward_time=0.162, loss_ctc=82.773, loss_interctc_layer6=93.630, loss_interctc_layer12=78.405, loss_interctc_layer15=72.349, loss_interctc_layer21=85.453, loss=82.522, backward_time=0.223, grad_norm=72.040, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.812e-05, train_time=1.294 +[gpua006:0/64] 2024-02-10 07:48:59,398 (trainer:756) INFO: 27epoch:train:3401-3500batch: iter_time=8.017e-05, forward_time=0.157, loss_ctc=92.124, loss_interctc_layer6=82.223, loss_interctc_layer12=68.857, loss_interctc_layer15=63.552, loss_interctc_layer21=95.166, loss=80.384, backward_time=0.214, grad_norm=72.527, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.811e-05, train_time=1.226 +[gpua006:0/64] 2024-02-10 07:51:15,563 (trainer:756) INFO: 27epoch:train:3501-3600batch: iter_time=8.335e-05, forward_time=0.142, loss_ctc=75.213, loss_interctc_layer6=83.332, loss_interctc_layer12=69.543, loss_interctc_layer15=63.948, loss_interctc_layer21=77.599, loss=73.927, backward_time=0.209, grad_norm=72.509, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.810e-05, train_time=1.361 +[gpua006:0/64] 2024-02-10 07:53:45,021 (trainer:756) INFO: 27epoch:train:3601-3700batch: iter_time=7.637e-05, forward_time=0.142, loss_ctc=78.876, loss_interctc_layer6=83.392, loss_interctc_layer12=69.757, loss_interctc_layer15=64.372, loss_interctc_layer21=81.480, loss=75.575, backward_time=0.210, grad_norm=71.068, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.809e-05, train_time=1.495 +[gpua006:0/64] 2024-02-10 07:54:58,195 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-10 07:55:16,656 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 07:55:20,050 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 07:55:20,050 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-10 07:55:20,055 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 08:00:29,987 (trainer:756) INFO: 27epoch:train:3701-3800batch: iter_time=2.884, forward_time=0.177, loss_ctc=77.390, loss_interctc_layer6=86.003, loss_interctc_layer12=71.391, loss_interctc_layer15=65.425, loss_interctc_layer21=80.176, loss=76.077, backward_time=0.216, grad_norm=77.657, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.808e-05, train_time=4.049 +[gpua006:0/64] 2024-02-10 08:02:06,469 (trainer:756) INFO: 27epoch:train:3801-3900batch: iter_time=7.518e-05, forward_time=0.143, loss_ctc=81.428, loss_interctc_layer6=86.708, loss_interctc_layer12=72.605, loss_interctc_layer15=66.999, loss_interctc_layer21=84.032, loss=78.354, backward_time=0.212, grad_norm=64.711, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.807e-05, train_time=0.965 +[gpua006:0/64] 2024-02-10 08:03:47,024 (trainer:756) INFO: 27epoch:train:3901-4000batch: iter_time=7.851e-05, forward_time=0.168, loss_ctc=75.158, loss_interctc_layer6=81.603, loss_interctc_layer12=67.666, loss_interctc_layer15=62.057, loss_interctc_layer21=77.721, loss=72.841, backward_time=0.220, grad_norm=105.333, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.806e-05, train_time=1.005 +[gpua006:0/64] 2024-02-10 08:06:00,434 (trainer:756) INFO: 27epoch:train:4001-4100batch: iter_time=8.247e-05, forward_time=0.160, loss_ctc=82.647, loss_interctc_layer6=86.110, loss_interctc_layer12=71.476, loss_interctc_layer15=65.639, loss_interctc_layer21=85.541, loss=78.283, backward_time=0.214, grad_norm=60.403, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.805e-05, train_time=1.334 +[gpua006:0/64] 2024-02-10 08:07:59,507 (trainer:756) INFO: 27epoch:train:4101-4200batch: iter_time=8.380e-05, forward_time=0.143, loss_ctc=87.985, loss_interctc_layer6=94.048, loss_interctc_layer12=78.329, loss_interctc_layer15=72.072, loss_interctc_layer21=90.896, loss=84.666, backward_time=0.209, grad_norm=97.759, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.804e-05, train_time=1.191 +[gpua006:0/64] 2024-02-10 08:10:08,468 (trainer:756) INFO: 27epoch:train:4201-4300batch: iter_time=8.090e-05, forward_time=0.154, loss_ctc=62.535, loss_interctc_layer6=78.336, loss_interctc_layer12=65.143, loss_interctc_layer15=59.840, loss_interctc_layer21=64.195, loss=66.010, backward_time=0.214, grad_norm=67.298, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.803e-05, train_time=1.289 +[gpua006:0/64] 2024-02-10 08:12:40,285 (trainer:756) INFO: 27epoch:train:4301-4400batch: iter_time=8.276e-05, forward_time=0.143, loss_ctc=86.497, loss_interctc_layer6=93.244, loss_interctc_layer12=77.419, loss_interctc_layer15=71.102, loss_interctc_layer21=89.165, loss=83.485, backward_time=0.209, grad_norm=79.239, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.802e-05, train_time=1.518 +[gpua006:0/64] 2024-02-10 08:15:25,277 (trainer:756) INFO: 27epoch:train:4401-4500batch: iter_time=8.085e-05, forward_time=0.142, loss_ctc=67.615, loss_interctc_layer6=82.321, loss_interctc_layer12=68.653, loss_interctc_layer15=63.354, loss_interctc_layer21=69.818, loss=70.352, backward_time=0.209, grad_norm=63.788, clip=100.000, loss_scale=8.874e+30, optim_step_time=0.137, optim0_lr0=7.801e-05, train_time=1.650 +[gpua006:0/64] 2024-02-10 08:17:36,175 (trainer:756) INFO: 27epoch:train:4501-4600batch: iter_time=8.511e-04, forward_time=0.190, loss_ctc=79.382, loss_interctc_layer6=88.971, loss_interctc_layer12=74.668, loss_interctc_layer15=69.025, loss_interctc_layer21=82.275, loss=78.864, backward_time=0.223, grad_norm=78.096, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=7.800e-05, train_time=1.309 +[gpua006:0/64] 2024-02-10 08:19:47,882 (trainer:756) INFO: 27epoch:train:4601-4700batch: iter_time=7.919e-05, forward_time=0.142, loss_ctc=82.973, loss_interctc_layer6=83.589, loss_interctc_layer12=69.978, loss_interctc_layer15=64.485, loss_interctc_layer21=85.524, loss=77.310, backward_time=0.210, grad_norm=77.894, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.799e-05, train_time=1.317 +[gpua006:0/64] 2024-02-10 08:21:47,294 (trainer:756) INFO: 27epoch:train:4701-4800batch: iter_time=8.221e-05, forward_time=0.147, loss_ctc=73.688, loss_interctc_layer6=78.711, loss_interctc_layer12=64.957, loss_interctc_layer15=59.355, loss_interctc_layer21=76.352, loss=70.613, backward_time=0.224, grad_norm=55.714, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.798e-05, train_time=1.194 +[gpua006:0/64] 2024-02-10 08:23:58,020 (trainer:756) INFO: 27epoch:train:4801-4900batch: iter_time=8.344e-05, forward_time=0.160, loss_ctc=77.036, loss_interctc_layer6=87.422, loss_interctc_layer12=73.099, loss_interctc_layer15=67.345, loss_interctc_layer21=79.300, loss=76.840, backward_time=0.214, grad_norm=91.290, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.797e-05, train_time=1.307 +[gpua006:0/64] 2024-02-10 08:25:56,305 (trainer:756) INFO: 27epoch:train:4901-5000batch: iter_time=8.306e-05, forward_time=0.142, loss_ctc=81.069, loss_interctc_layer6=86.121, loss_interctc_layer12=71.385, loss_interctc_layer15=65.454, loss_interctc_layer21=83.789, loss=77.564, backward_time=0.210, grad_norm=66.427, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.796e-05, train_time=1.183 +[gpua006:0/64] 2024-02-10 08:26:16,334 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-10 08:26:34,571 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 08:26:37,942 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 08:26:37,942 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-10 08:26:37,971 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 08:32:37,875 (trainer:756) INFO: 27epoch:train:5001-5100batch: iter_time=2.959, forward_time=0.172, loss_ctc=77.689, loss_interctc_layer6=83.087, loss_interctc_layer12=69.495, loss_interctc_layer15=64.131, loss_interctc_layer21=80.234, loss=74.927, backward_time=0.215, grad_norm=77.612, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.795e-05, train_time=4.014 +[gpua006:0/64] 2024-02-10 08:34:13,019 (trainer:756) INFO: 27epoch:train:5101-5200batch: iter_time=8.132e-05, forward_time=0.143, loss_ctc=81.603, loss_interctc_layer6=82.454, loss_interctc_layer12=68.396, loss_interctc_layer15=62.713, loss_interctc_layer21=84.445, loss=75.922, backward_time=0.211, grad_norm=68.308, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.794e-05, train_time=0.952 +[gpua006:0/64] 2024-02-10 08:36:35,059 (trainer:756) INFO: 27epoch:train:5201-5300batch: iter_time=8.911e-05, forward_time=0.143, loss_ctc=86.825, loss_interctc_layer6=85.206, loss_interctc_layer12=70.637, loss_interctc_layer15=64.708, loss_interctc_layer21=89.765, loss=79.428, backward_time=0.210, grad_norm=73.684, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.793e-05, train_time=1.420 +[gpua006:0/64] 2024-02-10 08:38:52,437 (trainer:756) INFO: 27epoch:train:5301-5400batch: iter_time=8.820e-05, forward_time=0.143, loss_ctc=77.049, loss_interctc_layer6=89.310, loss_interctc_layer12=74.546, loss_interctc_layer15=68.659, loss_interctc_layer21=79.391, loss=77.791, backward_time=0.210, grad_norm=74.436, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.792e-05, train_time=1.374 +[gpua006:0/64] 2024-02-10 08:40:49,958 (trainer:756) INFO: 27epoch:train:5401-5500batch: iter_time=8.821e-05, forward_time=0.144, loss_ctc=85.119, loss_interctc_layer6=85.559, loss_interctc_layer12=70.753, loss_interctc_layer15=64.855, loss_interctc_layer21=87.919, loss=78.841, backward_time=0.212, grad_norm=81.137, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.791e-05, train_time=1.175 +[gpua006:0/64] 2024-02-10 08:43:06,304 (trainer:756) INFO: 27epoch:train:5501-5600batch: iter_time=8.324e-05, forward_time=0.162, loss_ctc=75.117, loss_interctc_layer6=82.607, loss_interctc_layer12=68.709, loss_interctc_layer15=63.117, loss_interctc_layer21=77.456, loss=73.401, backward_time=0.215, grad_norm=64.460, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.790e-05, train_time=1.363 +[gpua006:0/64] 2024-02-10 08:45:25,621 (trainer:756) INFO: 27epoch:train:5601-5700batch: iter_time=9.419e-05, forward_time=0.152, loss_ctc=78.355, loss_interctc_layer6=88.520, loss_interctc_layer12=73.505, loss_interctc_layer15=67.456, loss_interctc_layer21=80.849, loss=77.737, backward_time=0.214, grad_norm=70.685, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.789e-05, train_time=1.393 +[gpua006:0/64] 2024-02-10 08:47:34,398 (trainer:756) INFO: 27epoch:train:5701-5800batch: iter_time=8.898e-05, forward_time=0.145, loss_ctc=78.860, loss_interctc_layer6=81.666, loss_interctc_layer12=67.991, loss_interctc_layer15=62.611, loss_interctc_layer21=81.428, loss=74.511, backward_time=0.210, grad_norm=65.404, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.788e-05, train_time=1.288 +[gpua006:0/64] 2024-02-10 08:49:33,145 (trainer:756) INFO: 27epoch:train:5801-5900batch: iter_time=8.826e-05, forward_time=0.212, loss_ctc=81.518, loss_interctc_layer6=92.340, loss_interctc_layer12=77.723, loss_interctc_layer15=71.938, loss_interctc_layer21=84.245, loss=81.553, backward_time=0.263, grad_norm=76.501, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=7.787e-05, train_time=1.187 +[gpua006:0/64] 2024-02-10 08:51:37,187 (trainer:756) INFO: 27epoch:train:5901-6000batch: iter_time=7.555e-05, forward_time=0.142, loss_ctc=91.291, loss_interctc_layer6=82.204, loss_interctc_layer12=68.864, loss_interctc_layer15=63.157, loss_interctc_layer21=94.065, loss=79.916, backward_time=0.208, grad_norm=67.070, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.786e-05, train_time=1.240 +[gpua006:0/64] 2024-02-10 08:54:58,335 (trainer:756) INFO: 27epoch:train:6001-6100batch: iter_time=8.899e-05, forward_time=0.143, loss_ctc=75.102, loss_interctc_layer6=82.886, loss_interctc_layer12=68.737, loss_interctc_layer15=63.006, loss_interctc_layer21=77.522, loss=73.451, backward_time=0.209, grad_norm=71.152, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.785e-05, train_time=2.012 +[gpua006:0/64] 2024-02-10 08:57:10,054 (trainer:756) INFO: 27epoch:train:6101-6200batch: iter_time=8.430e-05, forward_time=0.142, loss_ctc=77.056, loss_interctc_layer6=82.206, loss_interctc_layer12=68.624, loss_interctc_layer15=63.204, loss_interctc_layer21=79.543, loss=74.127, backward_time=0.209, grad_norm=95.035, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.784e-05, train_time=1.317 +[gpua006:0/64] 2024-02-10 08:58:33,832 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-10 08:58:52,005 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 08:58:55,399 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 08:58:55,399 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-10 08:58:55,403 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 09:04:36,620 (trainer:756) INFO: 27epoch:train:6201-6300batch: iter_time=3.300, forward_time=0.170, loss_ctc=78.909, loss_interctc_layer6=85.683, loss_interctc_layer12=71.008, loss_interctc_layer15=65.053, loss_interctc_layer21=81.605, loss=76.452, backward_time=0.215, grad_norm=75.151, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.783e-05, train_time=4.465 +[gpua006:0/64] 2024-02-10 09:06:32,124 (trainer:756) INFO: 27epoch:train:6301-6400batch: iter_time=3.386e-04, forward_time=0.146, loss_ctc=86.534, loss_interctc_layer6=86.264, loss_interctc_layer12=72.140, loss_interctc_layer15=66.483, loss_interctc_layer21=89.367, loss=80.158, backward_time=0.210, grad_norm=108.472, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=7.782e-05, train_time=1.155 +[gpua006:0/64] 2024-02-10 09:08:12,947 (trainer:756) INFO: 27epoch:train:6401-6500batch: iter_time=3.776e-04, forward_time=0.144, loss_ctc=78.557, loss_interctc_layer6=81.310, loss_interctc_layer12=67.392, loss_interctc_layer15=61.744, loss_interctc_layer21=81.380, loss=74.076, backward_time=0.213, grad_norm=60.432, clip=100.000, loss_scale=1.775e+31, optim_step_time=0.138, optim0_lr0=7.781e-05, train_time=1.008 +[gpua006:0/64] 2024-02-10 09:10:03,484 (trainer:756) INFO: 27epoch:train:6501-6600batch: iter_time=8.044e-05, forward_time=0.143, loss_ctc=84.511, loss_interctc_layer6=85.543, loss_interctc_layer12=70.736, loss_interctc_layer15=64.875, loss_interctc_layer21=87.415, loss=78.616, backward_time=0.211, grad_norm=103.268, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.780e-05, train_time=1.105 +[gpua006:0/64] 2024-02-10 09:11:59,512 (trainer:756) INFO: 27epoch:train:6601-6700batch: iter_time=8.456e-05, forward_time=0.144, loss_ctc=91.122, loss_interctc_layer6=94.145, loss_interctc_layer12=78.305, loss_interctc_layer15=72.071, loss_interctc_layer21=94.062, loss=85.941, backward_time=0.211, grad_norm=63.407, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.779e-05, train_time=1.160 +[gpua006:0/64] 2024-02-10 09:14:37,148 (trainer:756) INFO: 27epoch:train:6701-6800batch: iter_time=8.422e-05, forward_time=0.143, loss_ctc=64.712, loss_interctc_layer6=79.353, loss_interctc_layer12=65.856, loss_interctc_layer15=60.445, loss_interctc_layer21=66.549, loss=67.383, backward_time=0.209, grad_norm=58.605, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.778e-05, train_time=1.575 +[gpua006:0/64] 2024-02-10 09:16:53,863 (trainer:756) INFO: 27epoch:train:6801-6900batch: iter_time=8.535e-05, forward_time=0.147, loss_ctc=89.206, loss_interctc_layer6=92.989, loss_interctc_layer12=77.191, loss_interctc_layer15=70.819, loss_interctc_layer21=92.204, loss=84.482, backward_time=0.219, grad_norm=64.402, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.777e-05, train_time=1.368 +[gpua006:0/64] 2024-02-10 09:19:16,777 (trainer:756) INFO: 27epoch:train:6901-7000batch: iter_time=8.110e-05, forward_time=0.257, loss_ctc=71.246, loss_interctc_layer6=82.481, loss_interctc_layer12=68.894, loss_interctc_layer15=63.475, loss_interctc_layer21=73.413, loss=71.902, backward_time=0.238, grad_norm=82.167, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.150, optim0_lr0=7.776e-05, train_time=1.429 +[gpua006:0/64] 2024-02-10 09:21:51,861 (trainer:756) INFO: 27epoch:train:7001-7100batch: iter_time=8.270e-05, forward_time=0.143, loss_ctc=84.407, loss_interctc_layer6=88.903, loss_interctc_layer12=74.653, loss_interctc_layer15=69.001, loss_interctc_layer21=87.332, loss=80.859, backward_time=0.209, grad_norm=84.157, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.775e-05, train_time=1.550 +[gpua006:0/64] 2024-02-10 09:24:22,424 (trainer:756) INFO: 27epoch:train:7101-7200batch: iter_time=8.841e-05, forward_time=0.144, loss_ctc=86.571, loss_interctc_layer6=83.738, loss_interctc_layer12=70.029, loss_interctc_layer15=64.385, loss_interctc_layer21=88.146, loss=78.574, backward_time=0.210, grad_norm=72.217, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.774e-05, train_time=1.506 +[gpua006:0/64] 2024-02-10 09:26:21,381 (trainer:756) INFO: 27epoch:train:7201-7300batch: iter_time=2.541e-04, forward_time=0.144, loss_ctc=77.463, loss_interctc_layer6=77.924, loss_interctc_layer12=64.230, loss_interctc_layer15=58.652, loss_interctc_layer21=80.214, loss=71.697, backward_time=0.210, grad_norm=62.815, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.773e-05, train_time=1.189 +[gpua006:0/64] 2024-02-10 09:29:16,672 (trainer:756) INFO: 27epoch:train:7301-7400batch: iter_time=8.169e-05, forward_time=0.143, loss_ctc=78.028, loss_interctc_layer6=86.721, loss_interctc_layer12=72.420, loss_interctc_layer15=66.652, loss_interctc_layer21=80.543, loss=76.873, backward_time=0.209, grad_norm=116.568, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.773e-05, train_time=1.753 +[gpua006:0/64] 2024-02-10 09:31:18,231 (trainer:756) INFO: 27epoch:train:7401-7500batch: iter_time=7.889e-05, forward_time=0.142, loss_ctc=84.053, loss_interctc_layer6=85.998, loss_interctc_layer12=71.290, loss_interctc_layer15=65.338, loss_interctc_layer21=86.826, loss=78.701, backward_time=0.209, grad_norm=70.726, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.772e-05, train_time=1.215 +[gpua006:0/64] 2024-02-10 09:31:38,282 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-10 09:31:56,940 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 09:32:00,341 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 09:32:00,341 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-10 09:32:00,346 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 09:38:27,293 (trainer:756) INFO: 27epoch:train:7501-7600batch: iter_time=3.161, forward_time=0.145, loss_ctc=72.539, loss_interctc_layer6=82.945, loss_interctc_layer12=69.448, loss_interctc_layer15=63.933, loss_interctc_layer21=75.032, loss=72.779, backward_time=0.210, grad_norm=74.938, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.771e-05, train_time=4.290 +[gpua006:0/64] 2024-02-10 09:40:26,868 (trainer:756) INFO: 27epoch:train:7601-7700batch: iter_time=8.144e-05, forward_time=0.142, loss_ctc=78.533, loss_interctc_layer6=81.982, loss_interctc_layer12=67.917, loss_interctc_layer15=62.103, loss_interctc_layer21=81.215, loss=74.350, backward_time=0.208, grad_norm=141.502, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.770e-05, train_time=1.196 +[gpua006:0/64] 2024-02-10 09:42:56,174 (trainer:756) INFO: 27epoch:train:7701-7800batch: iter_time=8.146e-05, forward_time=0.214, loss_ctc=83.516, loss_interctc_layer6=84.465, loss_interctc_layer12=69.852, loss_interctc_layer15=63.957, loss_interctc_layer21=86.432, loss=77.644, backward_time=0.269, grad_norm=70.088, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=7.769e-05, train_time=1.493 +[gpua006:0/64] 2024-02-10 09:45:07,039 (trainer:756) INFO: 27epoch:train:7801-7900batch: iter_time=8.136e-05, forward_time=0.142, loss_ctc=74.478, loss_interctc_layer6=89.642, loss_interctc_layer12=75.146, loss_interctc_layer15=69.087, loss_interctc_layer21=76.741, loss=77.019, backward_time=0.209, grad_norm=88.024, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.768e-05, train_time=1.308 +[gpua006:0/64] 2024-02-10 09:47:24,272 (trainer:756) INFO: 27epoch:train:7901-8000batch: iter_time=8.470e-05, forward_time=0.146, loss_ctc=81.641, loss_interctc_layer6=85.286, loss_interctc_layer12=70.554, loss_interctc_layer15=64.634, loss_interctc_layer21=84.283, loss=77.280, backward_time=0.211, grad_norm=61.449, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.767e-05, train_time=1.371 +[gpua006:0/64] 2024-02-10 09:50:20,480 (trainer:756) INFO: 27epoch:train:8001-8100batch: iter_time=9.580e-05, forward_time=0.148, loss_ctc=72.762, loss_interctc_layer6=82.895, loss_interctc_layer12=68.982, loss_interctc_layer15=63.164, loss_interctc_layer21=75.261, loss=72.613, backward_time=0.208, grad_norm=87.691, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.766e-05, train_time=1.763 +[gpua006:0/64] 2024-02-10 09:52:29,632 (trainer:756) INFO: 27epoch:train:8101-8200batch: iter_time=8.355e-05, forward_time=0.145, loss_ctc=77.005, loss_interctc_layer6=88.812, loss_interctc_layer12=73.678, loss_interctc_layer15=67.708, loss_interctc_layer21=79.616, loss=77.364, backward_time=0.209, grad_norm=75.245, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.765e-05, train_time=1.291 +[gpua006:0/64] 2024-02-10 09:54:40,123 (trainer:756) INFO: 27epoch:train:8201-8300batch: iter_time=9.253e-05, forward_time=0.144, loss_ctc=74.491, loss_interctc_layer6=81.879, loss_interctc_layer12=68.238, loss_interctc_layer15=62.821, loss_interctc_layer21=76.892, loss=72.864, backward_time=0.209, grad_norm=65.951, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.764e-05, train_time=1.304 +[gpua006:0/64] 2024-02-10 09:56:46,005 (trainer:756) INFO: 27epoch:train:8301-8400batch: iter_time=8.267e-05, forward_time=0.146, loss_ctc=77.061, loss_interctc_layer6=92.056, loss_interctc_layer12=77.064, loss_interctc_layer15=70.904, loss_interctc_layer21=79.786, loss=79.374, backward_time=0.215, grad_norm=75.899, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.763e-05, train_time=1.259 +[gpua006:0/64] 2024-02-10 09:59:25,258 (trainer:756) INFO: 27epoch:train:8401-8500batch: iter_time=8.217e-05, forward_time=0.142, loss_ctc=85.328, loss_interctc_layer6=81.367, loss_interctc_layer12=68.200, loss_interctc_layer15=62.610, loss_interctc_layer21=87.910, loss=77.083, backward_time=0.208, grad_norm=73.136, clip=100.000, loss_scale=3.549e+31, optim_step_time=0.136, optim0_lr0=7.762e-05, train_time=1.593 +[gpua006:0/64] 2024-02-10 10:01:19,347 (trainer:756) INFO: 27epoch:train:8501-8600batch: iter_time=8.194e-05, forward_time=0.142, loss_ctc=71.291, loss_interctc_layer6=81.751, loss_interctc_layer12=67.642, loss_interctc_layer15=61.932, loss_interctc_layer21=73.740, loss=71.271, backward_time=0.210, grad_norm=65.516, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.761e-05, train_time=1.141 +[gpua006:0/64] 2024-02-10 10:04:15,997 (trainer:756) INFO: 27epoch:train:8601-8700batch: iter_time=8.327e-05, forward_time=0.210, loss_ctc=76.242, loss_interctc_layer6=82.529, loss_interctc_layer12=68.832, loss_interctc_layer15=63.322, loss_interctc_layer21=78.778, loss=73.941, backward_time=0.294, grad_norm=70.195, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=7.760e-05, train_time=1.766 +[gpua006:0/64] 2024-02-10 10:04:25,554 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 10:05:28,191 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-10 10:05:46,561 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 10:05:50,194 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 10:05:50,194 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-10 10:05:50,197 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 10:11:46,791 (trainer:756) INFO: 27epoch:train:8701-8800batch: iter_time=3.254, forward_time=0.144, loss_ctc=74.314, loss_interctc_layer6=85.449, loss_interctc_layer12=70.694, loss_interctc_layer15=64.674, loss_interctc_layer21=76.968, loss=74.420, backward_time=0.209, grad_norm=95.924, clip=100.000, loss_scale=2.213e+31, optim_step_time=0.137, optim0_lr0=7.759e-05, train_time=4.507 +[gpua006:0/64] 2024-02-10 10:13:41,984 (trainer:756) INFO: 27epoch:train:8801-8900batch: iter_time=7.942e-05, forward_time=0.143, loss_ctc=80.512, loss_interctc_layer6=86.323, loss_interctc_layer12=72.297, loss_interctc_layer15=66.594, loss_interctc_layer21=83.204, loss=77.786, backward_time=0.210, grad_norm=72.637, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.135, optim0_lr0=7.758e-05, train_time=1.152 +[gpua006:0/64] 2024-02-10 10:16:00,755 (trainer:756) INFO: 27epoch:train:8901-9000batch: iter_time=8.786e-05, forward_time=0.143, loss_ctc=74.569, loss_interctc_layer6=80.918, loss_interctc_layer12=66.960, loss_interctc_layer15=61.251, loss_interctc_layer21=77.095, loss=72.159, backward_time=0.211, grad_norm=67.824, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.757e-05, train_time=1.387 +[gpua006:0/64] 2024-02-10 10:18:46,750 (trainer:756) INFO: 27epoch:train:9001-9100batch: iter_time=8.500e-05, forward_time=0.143, loss_ctc=81.670, loss_interctc_layer6=85.620, loss_interctc_layer12=71.019, loss_interctc_layer15=65.056, loss_interctc_layer21=84.522, loss=77.578, backward_time=0.210, grad_norm=64.386, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.756e-05, train_time=1.660 +[gpua006:0/64] 2024-02-10 10:21:14,958 (trainer:756) INFO: 27epoch:train:9101-9200batch: iter_time=8.857e-05, forward_time=0.145, loss_ctc=88.165, loss_interctc_layer6=93.169, loss_interctc_layer12=77.672, loss_interctc_layer15=71.420, loss_interctc_layer21=90.989, loss=84.283, backward_time=0.210, grad_norm=70.358, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.755e-05, train_time=1.482 +[gpua006:0/64] 2024-02-10 10:23:05,221 (trainer:756) INFO: 27epoch:train:9201-9300batch: iter_time=8.194e-05, forward_time=0.165, loss_ctc=62.636, loss_interctc_layer6=78.557, loss_interctc_layer12=65.246, loss_interctc_layer15=59.845, loss_interctc_layer21=64.405, loss=66.138, backward_time=0.210, grad_norm=73.506, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.754e-05, train_time=1.102 +[gpua006:0/64] 2024-02-10 10:25:52,179 (trainer:756) INFO: 27epoch:train:9301-9400batch: iter_time=8.802e-05, forward_time=0.145, loss_ctc=86.338, loss_interctc_layer6=92.730, loss_interctc_layer12=76.972, loss_interctc_layer15=70.583, loss_interctc_layer21=89.226, loss=83.170, backward_time=0.209, grad_norm=63.926, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.753e-05, train_time=1.669 +[gpua006:0/64] 2024-02-10 10:28:22,821 (trainer:756) INFO: 27epoch:train:9401-9500batch: iter_time=8.460e-05, forward_time=0.273, loss_ctc=66.838, loss_interctc_layer6=81.795, loss_interctc_layer12=68.096, loss_interctc_layer15=62.594, loss_interctc_layer21=68.939, loss=69.652, backward_time=0.241, grad_norm=67.421, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.752e-05, train_time=1.506 +[gpua006:0/64] 2024-02-10 10:30:30,180 (trainer:756) INFO: 27epoch:train:9501-9600batch: iter_time=8.344e-05, forward_time=0.149, loss_ctc=79.182, loss_interctc_layer6=88.038, loss_interctc_layer12=73.816, loss_interctc_layer15=68.033, loss_interctc_layer21=81.906, loss=78.195, backward_time=0.212, grad_norm=84.313, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.751e-05, train_time=1.272 +[gpua006:0/64] 2024-02-10 10:33:07,566 (trainer:756) INFO: 27epoch:train:9601-9700batch: iter_time=8.711e-05, forward_time=0.142, loss_ctc=82.084, loss_interctc_layer6=83.057, loss_interctc_layer12=69.693, loss_interctc_layer15=63.977, loss_interctc_layer21=84.571, loss=76.677, backward_time=0.208, grad_norm=77.773, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.750e-05, train_time=1.575 +[gpua006:0/64] 2024-02-10 10:36:00,416 (trainer:756) INFO: 27epoch:train:9701-9800batch: iter_time=8.664e-05, forward_time=0.143, loss_ctc=73.056, loss_interctc_layer6=77.647, loss_interctc_layer12=63.975, loss_interctc_layer15=58.322, loss_interctc_layer21=75.640, loss=69.728, backward_time=0.209, grad_norm=59.383, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.749e-05, train_time=1.728 +[gpua006:0/64] 2024-02-10 10:38:55,269 (trainer:756) INFO: 27epoch:train:9801-9900batch: iter_time=8.877e-05, forward_time=0.143, loss_ctc=75.594, loss_interctc_layer6=85.805, loss_interctc_layer12=71.615, loss_interctc_layer15=65.743, loss_interctc_layer21=78.009, loss=75.353, backward_time=0.208, grad_norm=73.461, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.748e-05, train_time=1.748 +[gpua006:0/64] 2024-02-10 10:39:51,030 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 10:41:08,170 (trainer:756) INFO: 27epoch:train:9901-10000batch: iter_time=8.716e-05, forward_time=0.142, loss_ctc=80.707, loss_interctc_layer6=85.708, loss_interctc_layer12=71.021, loss_interctc_layer15=64.985, loss_interctc_layer21=83.407, loss=77.166, backward_time=0.215, grad_norm=66.244, clip=100.000, loss_scale=1.537e+31, optim_step_time=0.137, optim0_lr0=7.747e-05, train_time=1.329 +[gpua006:0/64] 2024-02-10 10:41:28,199 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-10 10:41:46,823 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 10:41:50,203 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 10:41:50,203 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-10 10:41:50,241 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 10:48:22,986 (trainer:756) INFO: 27epoch:train:10001-10100batch: iter_time=3.194, forward_time=0.184, loss_ctc=72.694, loss_interctc_layer6=83.380, loss_interctc_layer12=69.855, loss_interctc_layer15=64.425, loss_interctc_layer21=75.266, loss=73.124, backward_time=0.221, grad_norm=77.123, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.746e-05, train_time=4.348 +[gpua006:0/64] 2024-02-10 10:50:19,766 (trainer:756) INFO: 27epoch:train:10101-10200batch: iter_time=8.345e-05, forward_time=0.141, loss_ctc=77.905, loss_interctc_layer6=81.889, loss_interctc_layer12=67.896, loss_interctc_layer15=62.071, loss_interctc_layer21=80.685, loss=74.089, backward_time=0.209, grad_norm=68.435, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.745e-05, train_time=1.168 +[gpua006:0/64] 2024-02-10 10:53:03,123 (trainer:756) INFO: 27epoch:train:10201-10300batch: iter_time=8.364e-05, forward_time=0.142, loss_ctc=83.491, loss_interctc_layer6=85.064, loss_interctc_layer12=70.395, loss_interctc_layer15=64.388, loss_interctc_layer21=86.311, loss=77.930, backward_time=0.209, grad_norm=68.926, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.744e-05, train_time=1.633 +[gpua006:0/64] 2024-02-10 10:55:02,139 (trainer:756) INFO: 27epoch:train:10301-10400batch: iter_time=8.403e-05, forward_time=0.143, loss_ctc=73.843, loss_interctc_layer6=88.722, loss_interctc_layer12=74.217, loss_interctc_layer15=68.330, loss_interctc_layer21=76.085, loss=76.239, backward_time=0.210, grad_norm=122.500, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.743e-05, train_time=1.190 +[gpua006:0/64] 2024-02-10 10:58:07,788 (trainer:756) INFO: 27epoch:train:10401-10500batch: iter_time=8.532e-05, forward_time=0.143, loss_ctc=81.282, loss_interctc_layer6=85.472, loss_interctc_layer12=70.561, loss_interctc_layer15=64.503, loss_interctc_layer21=83.938, loss=77.151, backward_time=0.209, grad_norm=63.978, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.742e-05, train_time=1.856 +[gpua006:0/64] 2024-02-10 11:01:49,823 (trainer:756) INFO: 27epoch:train:10501-10600batch: iter_time=8.632e-05, forward_time=0.145, loss_ctc=71.512, loss_interctc_layer6=82.251, loss_interctc_layer12=68.456, loss_interctc_layer15=62.786, loss_interctc_layer21=73.790, loss=71.759, backward_time=0.208, grad_norm=61.091, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.741e-05, train_time=2.220 +[gpua006:0/64] 2024-02-10 11:03:36,461 (trainer:756) INFO: 27epoch:train:10601-10700batch: iter_time=8.727e-05, forward_time=0.143, loss_ctc=76.221, loss_interctc_layer6=87.986, loss_interctc_layer12=72.925, loss_interctc_layer15=66.873, loss_interctc_layer21=78.748, loss=76.551, backward_time=0.210, grad_norm=64.278, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.740e-05, train_time=1.066 +[gpua006:0/64] 2024-02-10 11:05:58,618 (trainer:756) INFO: 27epoch:train:10701-10800batch: iter_time=8.223e-05, forward_time=0.203, loss_ctc=74.170, loss_interctc_layer6=82.024, loss_interctc_layer12=68.452, loss_interctc_layer15=63.072, loss_interctc_layer21=76.494, loss=72.842, backward_time=0.256, grad_norm=71.269, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=7.739e-05, train_time=1.421 +[gpua006:0/64] 2024-02-10 11:08:17,659 (trainer:756) INFO: 27epoch:train:10801-10900batch: iter_time=8.224e-05, forward_time=0.152, loss_ctc=77.096, loss_interctc_layer6=91.901, loss_interctc_layer12=77.065, loss_interctc_layer15=70.811, loss_interctc_layer21=79.578, loss=79.290, backward_time=0.213, grad_norm=74.950, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.739e-05, train_time=1.391 +[gpua006:0/64] 2024-02-10 11:10:31,637 (trainer:756) INFO: 27epoch:train:10901-11000batch: iter_time=8.119e-05, forward_time=0.142, loss_ctc=86.720, loss_interctc_layer6=80.241, loss_interctc_layer12=67.070, loss_interctc_layer15=61.285, loss_interctc_layer21=89.753, loss=77.014, backward_time=0.209, grad_norm=99.191, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.738e-05, train_time=1.340 +[gpua006:0/64] 2024-02-10 11:12:56,529 (trainer:756) INFO: 27epoch:train:11001-11100batch: iter_time=7.800e-05, forward_time=0.142, loss_ctc=70.398, loss_interctc_layer6=81.299, loss_interctc_layer12=67.371, loss_interctc_layer15=61.717, loss_interctc_layer21=72.737, loss=70.705, backward_time=0.210, grad_norm=68.226, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.737e-05, train_time=1.448 +[gpua006:0/64] 2024-02-10 11:15:51,614 (trainer:756) INFO: 27epoch:train:11101-11200batch: iter_time=8.634e-05, forward_time=0.142, loss_ctc=75.512, loss_interctc_layer6=81.783, loss_interctc_layer12=68.169, loss_interctc_layer15=62.664, loss_interctc_layer21=77.891, loss=73.204, backward_time=0.208, grad_norm=73.996, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.736e-05, train_time=1.751 +[gpua006:0/64] 2024-02-10 11:17:49,051 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-10 11:18:07,325 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 11:18:10,698 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 11:18:10,698 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-10 11:18:10,703 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 11:24:20,146 (trainer:756) INFO: 27epoch:train:11201-11300batch: iter_time=3.339, forward_time=0.143, loss_ctc=81.114, loss_interctc_layer6=85.414, loss_interctc_layer12=70.850, loss_interctc_layer15=64.833, loss_interctc_layer21=83.467, loss=77.136, backward_time=0.208, grad_norm=98.297, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.735e-05, train_time=5.085 +[gpua006:0/64] 2024-02-10 11:25:58,015 (trainer:756) INFO: 27epoch:train:11301-11400batch: iter_time=8.279e-05, forward_time=0.142, loss_ctc=86.192, loss_interctc_layer6=86.866, loss_interctc_layer12=72.699, loss_interctc_layer15=67.046, loss_interctc_layer21=89.060, loss=80.373, backward_time=0.210, grad_norm=68.834, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.734e-05, train_time=0.978 +[gpua006:0/64] 2024-02-10 11:28:07,920 (trainer:756) INFO: 27epoch:train:11401-11500batch: iter_time=8.139e-05, forward_time=0.142, loss_ctc=78.628, loss_interctc_layer6=81.575, loss_interctc_layer12=67.722, loss_interctc_layer15=61.872, loss_interctc_layer21=81.335, loss=74.227, backward_time=0.210, grad_norm=120.296, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.733e-05, train_time=1.299 +[gpua006:0/64] 2024-02-10 11:30:51,763 (trainer:756) INFO: 27epoch:train:11501-11600batch: iter_time=8.014e-05, forward_time=0.212, loss_ctc=84.581, loss_interctc_layer6=85.342, loss_interctc_layer12=70.804, loss_interctc_layer15=64.842, loss_interctc_layer21=87.583, loss=78.630, backward_time=0.236, grad_norm=94.008, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.145, optim0_lr0=7.732e-05, train_time=1.638 +[gpua006:0/64] 2024-02-10 11:33:49,459 (trainer:756) INFO: 27epoch:train:11601-11700batch: iter_time=8.693e-05, forward_time=0.158, loss_ctc=90.857, loss_interctc_layer6=92.706, loss_interctc_layer12=77.131, loss_interctc_layer15=70.837, loss_interctc_layer21=93.910, loss=85.088, backward_time=0.220, grad_norm=94.177, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.731e-05, train_time=1.777 +[gpua006:0/64] 2024-02-10 11:35:45,796 (trainer:756) INFO: 27epoch:train:11701-11800batch: iter_time=8.098e-05, forward_time=0.145, loss_ctc=64.612, loss_interctc_layer6=78.210, loss_interctc_layer12=65.049, loss_interctc_layer15=59.597, loss_interctc_layer21=66.531, loss=66.800, backward_time=0.211, grad_norm=53.948, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.730e-05, train_time=1.162 +[gpua006:0/64] 2024-02-10 11:37:34,392 (trainer:756) INFO: 27epoch:train:11801-11900batch: iter_time=7.522e-05, forward_time=0.143, loss_ctc=88.669, loss_interctc_layer6=92.283, loss_interctc_layer12=76.650, loss_interctc_layer15=70.154, loss_interctc_layer21=91.614, loss=83.874, backward_time=0.211, grad_norm=66.339, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.729e-05, train_time=1.087 +[gpua006:0/64] 2024-02-10 11:40:24,637 (trainer:756) INFO: 27epoch:train:11901-12000batch: iter_time=8.049e-05, forward_time=0.141, loss_ctc=70.635, loss_interctc_layer6=81.994, loss_interctc_layer12=68.255, loss_interctc_layer15=62.799, loss_interctc_layer21=72.819, loss=71.300, backward_time=0.208, grad_norm=60.416, clip=100.000, loss_scale=1.501e+31, optim_step_time=0.137, optim0_lr0=7.728e-05, train_time=1.702 +[gpua006:0/64] 2024-02-10 11:42:06,372 (trainer:756) INFO: 27epoch:train:12001-12100batch: iter_time=8.245e-05, forward_time=0.143, loss_ctc=83.179, loss_interctc_layer6=87.510, loss_interctc_layer12=73.313, loss_interctc_layer15=67.664, loss_interctc_layer21=86.096, loss=79.553, backward_time=0.210, grad_norm=84.364, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.727e-05, train_time=1.018 +[gpua006:0/64] 2024-02-10 11:44:27,667 (trainer:756) INFO: 27epoch:train:12101-12200batch: iter_time=8.787e-05, forward_time=0.142, loss_ctc=84.603, loss_interctc_layer6=82.781, loss_interctc_layer12=69.048, loss_interctc_layer15=63.613, loss_interctc_layer21=87.518, loss=77.513, backward_time=0.209, grad_norm=74.968, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.726e-05, train_time=1.413 +[gpua006:0/64] 2024-02-10 11:46:37,662 (trainer:756) INFO: 27epoch:train:12201-12300batch: iter_time=8.127e-05, forward_time=0.142, loss_ctc=77.010, loss_interctc_layer6=78.211, loss_interctc_layer12=64.425, loss_interctc_layer15=58.858, loss_interctc_layer21=79.601, loss=71.621, backward_time=0.210, grad_norm=78.368, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.725e-05, train_time=1.300 +[gpua006:0/64] 2024-02-10 11:49:25,789 (trainer:756) INFO: 27epoch:train:12301-12400batch: iter_time=8.487e-05, forward_time=0.165, loss_ctc=76.329, loss_interctc_layer6=85.047, loss_interctc_layer12=71.139, loss_interctc_layer15=65.395, loss_interctc_layer21=78.853, loss=75.353, backward_time=0.213, grad_norm=103.234, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.724e-05, train_time=1.681 +[gpua006:0/64] 2024-02-10 11:51:51,578 (trainer:756) INFO: 27epoch:train:12401-12500batch: iter_time=8.056e-05, forward_time=0.232, loss_ctc=84.610, loss_interctc_layer6=85.729, loss_interctc_layer12=71.068, loss_interctc_layer15=64.926, loss_interctc_layer21=87.595, loss=78.786, backward_time=0.258, grad_norm=62.558, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.723e-05, train_time=1.457 +[gpua006:0/64] 2024-02-10 11:52:11,608 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-10 11:52:30,687 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 11:52:34,061 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 11:52:34,061 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-10 11:52:34,097 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 11:59:05,755 (trainer:756) INFO: 27epoch:train:12501-12600batch: iter_time=3.260, forward_time=0.143, loss_ctc=72.517, loss_interctc_layer6=82.992, loss_interctc_layer12=69.543, loss_interctc_layer15=64.077, loss_interctc_layer21=74.997, loss=72.825, backward_time=0.211, grad_norm=63.704, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.722e-05, train_time=4.342 +[gpua006:0/64] 2024-02-10 12:00:59,361 (trainer:756) INFO: 27epoch:train:12601-12700batch: iter_time=8.373e-05, forward_time=0.142, loss_ctc=77.489, loss_interctc_layer6=81.621, loss_interctc_layer12=67.645, loss_interctc_layer15=61.856, loss_interctc_layer21=80.280, loss=73.778, backward_time=0.210, grad_norm=68.821, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.721e-05, train_time=1.136 +[gpua006:0/64] 2024-02-10 12:02:57,974 (trainer:756) INFO: 27epoch:train:12701-12800batch: iter_time=8.497e-05, forward_time=0.142, loss_ctc=83.225, loss_interctc_layer6=84.211, loss_interctc_layer12=69.676, loss_interctc_layer15=63.791, loss_interctc_layer21=86.135, loss=77.408, backward_time=0.209, grad_norm=72.631, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.720e-05, train_time=1.186 +[gpua006:0/64] 2024-02-10 12:05:56,109 (trainer:756) INFO: 27epoch:train:12801-12900batch: iter_time=9.118e-05, forward_time=0.143, loss_ctc=73.282, loss_interctc_layer6=88.837, loss_interctc_layer12=74.170, loss_interctc_layer15=68.096, loss_interctc_layer21=75.585, loss=75.994, backward_time=0.210, grad_norm=81.185, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.719e-05, train_time=1.781 +[gpua006:0/64] 2024-02-10 12:08:11,954 (trainer:756) INFO: 27epoch:train:12901-13000batch: iter_time=8.657e-05, forward_time=0.143, loss_ctc=81.141, loss_interctc_layer6=84.531, loss_interctc_layer12=69.881, loss_interctc_layer15=63.943, loss_interctc_layer21=83.814, loss=76.662, backward_time=0.210, grad_norm=54.546, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.718e-05, train_time=1.358 +[gpua006:0/64] 2024-02-10 12:10:27,778 (trainer:756) INFO: 27epoch:train:13001-13100batch: iter_time=7.877e-05, forward_time=0.144, loss_ctc=71.182, loss_interctc_layer6=81.736, loss_interctc_layer12=67.814, loss_interctc_layer15=62.113, loss_interctc_layer21=73.548, loss=71.279, backward_time=0.209, grad_norm=62.690, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.717e-05, train_time=1.359 +[gpua006:0/64] 2024-02-10 12:12:25,184 (trainer:756) INFO: 27epoch:train:13101-13200batch: iter_time=8.493e-05, forward_time=0.141, loss_ctc=75.418, loss_interctc_layer6=87.682, loss_interctc_layer12=72.708, loss_interctc_layer15=66.644, loss_interctc_layer21=78.044, loss=76.099, backward_time=0.209, grad_norm=69.225, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.716e-05, train_time=1.174 +[gpua006:0/64] 2024-02-10 12:14:25,301 (trainer:756) INFO: 27epoch:train:13201-13300batch: iter_time=8.256e-05, forward_time=0.143, loss_ctc=73.512, loss_interctc_layer6=81.118, loss_interctc_layer12=67.396, loss_interctc_layer15=61.941, loss_interctc_layer21=75.958, loss=71.985, backward_time=0.210, grad_norm=77.179, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.715e-05, train_time=1.201 +[gpua006:0/64] 2024-02-10 12:17:00,487 (trainer:756) INFO: 27epoch:train:13301-13400batch: iter_time=8.831e-05, forward_time=0.168, loss_ctc=76.775, loss_interctc_layer6=91.646, loss_interctc_layer12=76.708, loss_interctc_layer15=70.611, loss_interctc_layer21=79.419, loss=79.032, backward_time=0.262, grad_norm=80.744, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.715e-05, train_time=1.552 +[gpua006:0/64] 2024-02-10 12:19:29,620 (trainer:756) INFO: 27epoch:train:13401-13500batch: iter_time=8.886e-05, forward_time=0.167, loss_ctc=85.255, loss_interctc_layer6=80.200, loss_interctc_layer12=66.681, loss_interctc_layer15=61.689, loss_interctc_layer21=88.430, loss=76.451, backward_time=0.218, grad_norm=77.178, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.714e-05, train_time=1.491 +[gpua006:0/64] 2024-02-10 12:21:54,822 (trainer:756) INFO: 27epoch:train:13501-13600batch: iter_time=8.438e-05, forward_time=0.142, loss_ctc=71.090, loss_interctc_layer6=81.225, loss_interctc_layer12=67.395, loss_interctc_layer15=61.808, loss_interctc_layer21=73.540, loss=71.012, backward_time=0.210, grad_norm=73.049, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.713e-05, train_time=1.452 +[gpua006:0/64] 2024-02-10 12:24:13,620 (trainer:756) INFO: 27epoch:train:13601-13700batch: iter_time=8.668e-05, forward_time=0.142, loss_ctc=75.779, loss_interctc_layer6=81.653, loss_interctc_layer12=67.898, loss_interctc_layer15=62.385, loss_interctc_layer21=78.215, loss=73.186, backward_time=0.210, grad_norm=64.324, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.712e-05, train_time=1.388 +[gpua006:0/64] 2024-02-10 12:25:26,628 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-10 12:25:44,950 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 12:25:48,662 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 12:25:48,662 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-10 12:25:48,665 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 12:31:19,830 (trainer:756) INFO: 27epoch:train:13701-13800batch: iter_time=3.045, forward_time=0.201, loss_ctc=75.908, loss_interctc_layer6=85.154, loss_interctc_layer12=70.405, loss_interctc_layer15=64.330, loss_interctc_layer21=78.673, loss=74.894, backward_time=0.218, grad_norm=61.322, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.711e-05, train_time=4.262 +[gpua006:0/64] 2024-02-10 12:33:27,351 (trainer:756) INFO: 27epoch:train:13801-13900batch: iter_time=8.234e-05, forward_time=0.144, loss_ctc=84.477, loss_interctc_layer6=85.841, loss_interctc_layer12=71.717, loss_interctc_layer15=66.045, loss_interctc_layer21=87.262, loss=79.068, backward_time=0.212, grad_norm=87.727, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.710e-05, train_time=1.275 +[gpua006:0/64] 2024-02-10 12:35:36,650 (trainer:756) INFO: 27epoch:train:13901-14000batch: iter_time=8.648e-05, forward_time=0.142, loss_ctc=77.944, loss_interctc_layer6=80.655, loss_interctc_layer12=66.706, loss_interctc_layer15=61.142, loss_interctc_layer21=80.709, loss=73.431, backward_time=0.210, grad_norm=66.013, clip=100.000, loss_scale=3.002e+31, optim_step_time=0.136, optim0_lr0=7.709e-05, train_time=1.293 +[gpua006:0/64] 2024-02-10 12:37:29,711 (trainer:756) INFO: 27epoch:train:14001-14100batch: iter_time=8.799e-05, forward_time=0.143, loss_ctc=83.661, loss_interctc_layer6=84.645, loss_interctc_layer12=69.936, loss_interctc_layer15=64.018, loss_interctc_layer21=86.675, loss=77.787, backward_time=0.211, grad_norm=69.571, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.708e-05, train_time=1.130 +[gpua006:0/64] 2024-02-10 12:40:51,113 (trainer:756) INFO: 27epoch:train:14101-14200batch: iter_time=9.284e-05, forward_time=0.144, loss_ctc=91.546, loss_interctc_layer6=93.407, loss_interctc_layer12=77.726, loss_interctc_layer15=71.481, loss_interctc_layer21=94.701, loss=85.772, backward_time=0.209, grad_norm=88.081, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.707e-05, train_time=2.014 +[gpua006:0/64] 2024-02-10 12:43:10,794 (trainer:756) INFO: 27epoch:train:14201-14300batch: iter_time=8.968e-05, forward_time=0.142, loss_ctc=64.242, loss_interctc_layer6=77.969, loss_interctc_layer12=64.761, loss_interctc_layer15=59.388, loss_interctc_layer21=66.095, loss=66.491, backward_time=0.209, grad_norm=54.969, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.706e-05, train_time=1.397 +[gpua006:0/64] 2024-02-10 12:46:06,942 (trainer:756) INFO: 27epoch:train:14301-14400batch: iter_time=8.152e-05, forward_time=0.215, loss_ctc=89.093, loss_interctc_layer6=92.482, loss_interctc_layer12=76.739, loss_interctc_layer15=70.301, loss_interctc_layer21=92.061, loss=84.135, backward_time=0.255, grad_norm=72.020, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=7.705e-05, train_time=1.760 +[gpua006:0/64] 2024-02-10 12:48:25,606 (trainer:756) INFO: 27epoch:train:14401-14500batch: iter_time=8.399e-05, forward_time=0.143, loss_ctc=70.886, loss_interctc_layer6=82.218, loss_interctc_layer12=68.554, loss_interctc_layer15=63.170, loss_interctc_layer21=72.978, loss=71.561, backward_time=0.210, grad_norm=93.605, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.704e-05, train_time=1.388 +[gpua006:0/64] 2024-02-10 12:50:54,531 (trainer:756) INFO: 27epoch:train:14501-14600batch: iter_time=8.819e-05, forward_time=0.142, loss_ctc=83.061, loss_interctc_layer6=88.483, loss_interctc_layer12=74.159, loss_interctc_layer15=68.404, loss_interctc_layer21=86.065, loss=80.034, backward_time=0.209, grad_norm=73.870, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.703e-05, train_time=1.489 +[gpua006:0/64] 2024-02-10 12:53:23,200 (trainer:756) INFO: 27epoch:train:14601-14700batch: iter_time=9.178e-05, forward_time=0.142, loss_ctc=83.628, loss_interctc_layer6=82.229, loss_interctc_layer12=68.301, loss_interctc_layer15=63.073, loss_interctc_layer21=86.555, loss=76.757, backward_time=0.207, grad_norm=69.187, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.702e-05, train_time=1.486 +[gpua006:0/64] 2024-02-10 12:55:55,991 (trainer:756) INFO: 27epoch:train:14701-14800batch: iter_time=8.436e-05, forward_time=0.142, loss_ctc=76.603, loss_interctc_layer6=78.061, loss_interctc_layer12=64.097, loss_interctc_layer15=58.584, loss_interctc_layer21=79.351, loss=71.339, backward_time=0.209, grad_norm=67.221, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.701e-05, train_time=1.528 +[gpua006:0/64] 2024-02-10 12:58:38,004 (trainer:756) INFO: 27epoch:train:14801-14900batch: iter_time=8.623e-05, forward_time=0.142, loss_ctc=76.616, loss_interctc_layer6=85.560, loss_interctc_layer12=71.240, loss_interctc_layer15=65.574, loss_interctc_layer21=78.991, loss=75.596, backward_time=0.209, grad_norm=70.551, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.700e-05, train_time=1.620 +[gpua006:0/64] 2024-02-10 13:01:00,781 (trainer:756) INFO: 27epoch:train:14901-15000batch: iter_time=8.615e-05, forward_time=0.142, loss_ctc=84.574, loss_interctc_layer6=85.488, loss_interctc_layer12=70.639, loss_interctc_layer15=64.582, loss_interctc_layer21=87.538, loss=78.564, backward_time=0.209, grad_norm=65.946, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.699e-05, train_time=1.428 +[gpua006:0/64] 2024-02-10 13:32:10,089 (trainer:355) INFO: 27epoch results: [train] iter_time=0.249, forward_time=0.155, loss_ctc=79.253, loss_interctc_layer6=85.169, loss_interctc_layer12=70.913, loss_interctc_layer15=65.161, loss_interctc_layer21=81.861, loss=76.472, backward_time=0.216, grad_norm=76.613, clip=100.000, loss_scale=1.600e+31, optim_step_time=0.137, optim0_lr0=7.771e-05, train_time=1.630, time=6 hours, 47 minutes and 50.96 seconds, total_count=405000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=44.344, cer_ctc=0.205, loss_interctc_layer6=49.708, cer_interctc_layer6=0.222, loss_interctc_layer12=37.050, cer_interctc_layer12=0.156, loss_interctc_layer15=32.744, cer_interctc_layer15=0.131, loss_interctc_layer21=46.827, cer_interctc_layer21=0.219, loss=42.135, time=30 minutes and 45.23 seconds, total_count=126117, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-10 13:32:29,516 (trainer:410) INFO: The best model has been updated: valid.cer_ctc, valid.total_count +[gpua006:0/64] 2024-02-10 13:32:29,619 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/19epoch.pth, exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/22epoch.pth +[gpua006:0/64] 2024-02-10 13:32:29,619 (trainer:289) INFO: 28/45epoch started. Estimated time to finish: 5 days, 21 hours and 12 minutes +[gpua006:0/64] 2024-02-10 13:32:29,691 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-10 13:32:47,356 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-10 13:32:50,656 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-10 13:32:50,656 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-10 13:32:50,660 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-10 13:39:17,327 (trainer:756) INFO: 28epoch:train:1-100batch: iter_time=2.739, forward_time=0.187, loss_ctc=81.443, loss_interctc_layer6=89.783, loss_interctc_layer12=75.992, loss_interctc_layer15=70.384, loss_interctc_layer21=83.992, loss=80.319, backward_time=0.225, grad_norm=82.385, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.698e-05, train_time=4.076 +[gpua006:0/64] 2024-02-10 13:41:14,666 (trainer:756) INFO: 28epoch:train:101-200batch: iter_time=2.966e-04, forward_time=0.251, loss_ctc=90.140, loss_interctc_layer6=98.701, loss_interctc_layer12=83.100, loss_interctc_layer15=77.014, loss_interctc_layer21=93.118, loss=88.415, backward_time=0.268, grad_norm=81.019, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=7.697e-05, train_time=1.171 +[gpua006:0/64] 2024-02-10 13:41:57,508 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-10 13:43:30,264 (trainer:756) INFO: 28epoch:train:201-300batch: iter_time=9.195e-05, forward_time=0.142, loss_ctc=76.952, loss_interctc_layer6=87.654, loss_interctc_layer12=73.661, loss_interctc_layer15=67.892, loss_interctc_layer21=79.565, loss=77.145, backward_time=0.209, grad_norm=67.677, clip=100.000, loss_scale=2.868e+31, optim_step_time=0.136, optim0_lr0=7.696e-05, train_time=1.358 +[gpua006:0/64] 2024-02-10 13:45:39,676 (trainer:756) INFO: 28epoch:train:301-400batch: iter_time=9.006e-05, forward_time=0.142, loss_ctc=76.808, loss_interctc_layer6=85.775, loss_interctc_layer12=71.418, loss_interctc_layer15=65.726, loss_interctc_layer21=79.316, loss=75.809, backward_time=0.209, grad_norm=71.145, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.695e-05, train_time=1.294 +[gpua006:0/64] 2024-02-10 13:48:06,973 (trainer:756) INFO: 28epoch:train:401-500batch: iter_time=8.953e-05, forward_time=0.233, loss_ctc=105.566, loss_interctc_layer6=105.624, loss_interctc_layer12=87.810, loss_interctc_layer15=81.034, loss_interctc_layer21=109.326, loss=97.872, backward_time=0.284, grad_norm=86.447, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=7.695e-05, train_time=1.473 +[gpua006:0/64] 2024-02-10 13:50:12,014 (trainer:756) INFO: 28epoch:train:501-600batch: iter_time=8.901e-05, forward_time=0.142, loss_ctc=88.170, loss_interctc_layer6=95.192, loss_interctc_layer12=79.729, loss_interctc_layer15=73.521, loss_interctc_layer21=91.051, loss=85.533, backward_time=0.209, grad_norm=80.035, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.694e-05, train_time=1.248 +[gpua006:0/64] 2024-02-10 13:52:08,318 (trainer:756) INFO: 28epoch:train:601-700batch: iter_time=8.997e-05, forward_time=0.142, loss_ctc=69.956, loss_interctc_layer6=78.201, loss_interctc_layer12=64.852, loss_interctc_layer15=59.352, loss_interctc_layer21=72.420, loss=68.956, backward_time=0.208, grad_norm=92.394, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.693e-05, train_time=1.165 +[gpua006:0/64] 2024-02-10 13:54:31,779 (trainer:756) INFO: 28epoch:train:701-800batch: iter_time=9.708e-05, forward_time=0.203, loss_ctc=69.834, loss_interctc_layer6=79.136, loss_interctc_layer12=66.027, loss_interctc_layer15=60.643, loss_interctc_layer21=72.110, loss=69.550, backward_time=0.267, grad_norm=149.067, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.692e-05, train_time=1.435 +[gpua006:0/64] 2024-02-10 13:56:49,401 (trainer:756) INFO: 28epoch:train:801-900batch: iter_time=9.260e-05, forward_time=0.157, loss_ctc=81.283, loss_interctc_layer6=83.091, loss_interctc_layer12=69.005, loss_interctc_layer15=63.338, loss_interctc_layer21=84.207, loss=76.185, backward_time=0.226, grad_norm=62.579, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.691e-05, train_time=1.376 +[gpua006:0/64] 2024-02-10 13:59:06,003 (trainer:756) INFO: 28epoch:train:901-1000batch: iter_time=8.523e-05, forward_time=0.142, loss_ctc=63.957, loss_interctc_layer6=76.245, loss_interctc_layer12=64.189, loss_interctc_layer15=59.117, loss_interctc_layer21=66.053, loss=65.912, backward_time=0.207, grad_norm=61.038, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.690e-05, train_time=1.364 +srun: Job step aborted: Waiting up to 32 seconds for job step to finish.