diff --git "a/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.3.log" "b/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.3.log" new file mode 100644--- /dev/null +++ "b/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.3.log" @@ -0,0 +1,4563 @@ +# Running on gpua006.delta.ncsa.illinois.edu +# Started at Sun Feb 11 20:14:26 CST 2024 +# SLURMD_NODENAME=gpua006 +# SLURM_CLUSTER_NAME=delta +# SLURM_CONF=/var/spool/slurmd/conf-cache/slurm.conf +# SLURM_CPUS_ON_NODE=64 +# SLURM_CPUS_PER_TASK=64 +# SLURM_EXPORT_ENV=PATH +# SLURM_GET_USER_ENV=1 +# SLURM_GPUS_ON_NODE=4 +# SLURM_GTIDS=0 +# SLURM_JOBID=2973985 +# SLURM_JOB_ACCOUNT=bbjs-delta-gpu +# SLURM_JOB_CPUS_PER_NODE='64(x16)' +# SLURM_JOB_END_TIME=1707876852 +# SLURM_JOB_GID=202 +# SLURM_JOB_GPUS=0,1,2,3 +# SLURM_JOB_ID=2973985 +# SLURM_JOB_NAME=exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.log +# SLURM_JOB_NODELIST='gpua[006-007,012,016,033,038-040,049,054-055,057,079-080,085,089]' +# SLURM_JOB_NUM_NODES=16 +# SLURM_JOB_PARTITION=gpuA100x4 +# SLURM_JOB_QOS=bbjs-delta-gpu +# SLURM_JOB_RESERVATION=bbjs +# SLURM_JOB_START_TIME=1707704052 +# SLURM_JOB_UID=68077 +# SLURM_JOB_USER=peng6 +# SLURM_LOCALID=0 +# SLURM_MEM_PER_NODE=240000 +# SLURM_MPI_TYPE=pmi2 +# SLURM_NNODES=16 +# SLURM_NODEID=0 +# SLURM_NODELIST='gpua[006-007,012,016,033,038-040,049,054-055,057,079-080,085,089]' +# SLURM_NODE_ALIASES='(null)' +# SLURM_OPEN_MODE=a +# SLURM_PRIO_PROCESS=0 +# SLURM_PROCID=0 +# SLURM_SUBMIT_DIR=/scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1 +# SLURM_SUBMIT_HOST=dt-login03.delta.ncsa.illinois.edu +# SLURM_TASKS_PER_NODE='1(x16)' +# SLURM_TASK_PID=741649 +# SLURM_TOPOLOGY_ADDR=ss00.ss05.gpua006 +# SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.node +# SLURM_WORKING_CLUSTER=delta:dt-sched:6817:9984:109 +# srun --export=ALL python3 -m espnet2.bin.s2t_train --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +GpuFreq=control_disabled +/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_st/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +ats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_methats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +od file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +[gpua006:0/64] 2024-02-11 20:15:16,297 (distributed_c10d:319) INFO: Added key: store_based_barrier_key:1 to store for rank: 0 +[gpua006:0/64] 2024-02-11 20:15:26,353 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:15:36,376 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:15:46,382 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:15:56,403 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:16:06,416 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:16:16,502 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:16:26,553 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:16:36,616 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:16:46,714 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:16:56,793 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:17:06,864 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:17:16,888 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=56, timeout=0:30:00) +/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +[gpua006:0/64] 2024-02-11 20:17:26,903 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=60, timeout=0:30:00) +[gpua006:0/64] 2024-02-11 20:17:36,955 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=60, timeout=0:30:00) +/scratch/bbjs/peng6/espnet-owsm-ctc-2/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc-2/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc-2/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_c80091c5-d019-4a3d-89a0-e206b6eb306d +[gpua006:0/64] 2024-02-11 20:17:45,858 (distributed_c10d:353) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 64 nodes. +[gpua006:0/64] 2024-02-11 20:17:45,916 (s2t:420) INFO: Vocabulary size: 50002 +[gpua006:0/64] 2024-02-11 20:17:57,861 (abs_task:1270) INFO: pytorch.version=1.13.1, cuda.available=True, cudnn.version=8500, cudnn.benchmark=False, cudnn.deterministic=True +[gpua006:0/64] 2024-02-11 20:17:57,872 (abs_task:1271) INFO: Model structure: +ESPnetS2TCTCModel( + (frontend): DefaultFrontend( + (stft): Stft(n_fft=512, win_length=400, hop_length=160, center=True, normalized=False, onesided=True) + (frontend): Frontend() + (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False) + ) + (specaug): SpecAug( + (freq_mask): MaskAlongAxis(mask_width_range=[0, 27], num_mask=2, axis=freq) + (time_mask): MaskAlongAxisVariableMaxWidth(mask_width_ratio_range=[0.0, 0.05], num_mask=10, axis=time) + ) + (normalize): GlobalMVN(stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz, norm_means=True, norm_vars=True) + (encoder): EBranchformerCTCEncoder( + (embed): Conv2dSubsampling8( + (conv): Sequential( + (0): Conv2d(1, 1024, kernel_size=(3, 3), stride=(2, 2)) + (1): ReLU() + (2): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2)) + (3): ReLU() + (4): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2)) + (5): ReLU() + ) + (out): Linear(in_features=9216, out_features=1024, bias=True) + (pos_enc): PositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (encoders): MultiSequential( + (0): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (1): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (2): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (3): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (4): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (5): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (6): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (7): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (8): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (9): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (10): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (11): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (12): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (13): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (14): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (15): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (16): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (17): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (18): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (19): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (20): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (21): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (22): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (23): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (24): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (25): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + (26): EBranchformerEncoderLayer( + (attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (cgmlp): ConvolutionalGatingMLP( + (channel_proj1): Sequential( + (0): Linear(in_features=1024, out_features=4096, bias=True) + (1): GELU(approximate='none') + ) + (csgu): ConvolutionalSpatialGatingUnit( + (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True) + (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (act): Identity() + (dropout): Dropout(p=0.1, inplace=False) + ) + (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=1024, out_features=4096, bias=True) + (w_2): Linear(in_features=4096, out_features=1024, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (cross_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=1024, out_features=1024, bias=True) + (linear_k): Linear(in_features=1024, out_features=1024, bias=True) + (linear_v): Linear(in_features=1024, out_features=1024, bias=True) + (linear_out): Linear(in_features=1024, out_features=1024, bias=True) + (dropout): Identity() + ) + (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048) + (merge_proj): Linear(in_features=2048, out_features=1024, bias=True) + ) + ) + (after_norm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True) + (conditioning_layer): Linear(in_features=50002, out_features=1024, bias=True) + ) + (prompt_encoder): TransformerEncoder( + (encoders): MultiSequential( + (0): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): EncoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=512, out_features=512, bias=True) + (linear_k): Linear(in_features=512, out_features=512, bias=True) + (linear_v): Linear(in_features=512, out_features=512, bias=True) + (linear_out): Linear(in_features=512, out_features=512, bias=True) + (dropout): Identity() + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=512, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=512, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((512,), eps=1e-12, elementwise_affine=True) + ) + (embed): Embedding(50002, 512) + (pos_enc): PositionalEncoding( + (dropout): Dropout(p=0.0, inplace=False) + ) + (embed_proj): Linear(in_features=512, out_features=1024, bias=True) + (prompt_proj): Linear(in_features=512, out_features=1024, bias=True) + (ctc): CTC( + (ctc_lo): Linear(in_features=1024, out_features=50002, bias=True) + (ctc_loss): CTCLoss() + ) +) + +Model summary: + Class Name: ESPnetS2TCTCModel + Total Number of model parameters: 1.01 B + Number of trainable parameters: 1.01 B (100.0%) + Size: 4.02 GB + Type: torch.float32 +[gpua006:0/64] 2024-02-11 20:17:57,873 (abs_task:1274) INFO: Optimizer: +AdamW ( +Parameter Group 0 + amsgrad: False + betas: [0.9, 0.98] + capturable: False + eps: 1e-06 + foreach: None + initial_lr: 0.0002 + lr: 1.6666666666666667e-09 + maximize: False + weight_decay: 0.0 +) +[gpua006:0/64] 2024-02-11 20:17:57,873 (abs_task:1275) INFO: Scheduler: PiecewiseLinearWarmupLR(warmup_steps_list=[0, 30000, 60000], warmup_lr_list=[0.0, 5e-05, 0.0002]) +[gpua006:0/64] 2024-02-11 20:17:57,874 (abs_task:1284) INFO: Saving the configuration in exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/config.yaml +[gpua006:0/64] 2024-02-11 20:18:03,575 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 20:18:04,567 (abs_task:1660) INFO: [valid] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/dev_v3/wav.scp", "type": "kaldi_ark"} + text_prev: {"path": "dump/raw/dev_v3/text.prev", "type": "text"} + text_ctc: {"path": "dump/raw/dev_v3/text.ctc", "type": "text"} + text: {"path": "dump/raw/dev_v3/text", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 20:18:04,567 (abs_task:1661) INFO: [valid] Batch sampler: UnsortedBatchSampler(N-batch=4671, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/valid/speech_shape, +[gpua006:0/64] 2024-02-11 20:18:04,568 (abs_task:1662) INFO: [valid] mini-batch sizes summary: N-batch=4671, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-11 20:18:36,546 (trainer:167) INFO: The training was resumed using exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/checkpoint.pth +gpua006:741723:741723 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:741723:741723 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:741723:741723 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:741723:741723 [0] NCCL INFO cudaDriverVersion 12020 +NCCL version 2.14.3+cuda11.7 +[gpua006:0/64] 2024-02-11 20:18:44,456 (trainer:301) INFO: 32/45epoch started +[gpua006:0/64] 2024-02-11 20:18:44,495 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-11 20:19:02,979 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 20:19:06,554 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 20:19:06,554 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-11 20:19:06,558 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +gpua039:3725078:3725078 [0] NCCL INFO cudaDriverVersion 12020 +gpua039:3725078:3725078 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3725078:3725078 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3725078:3725078 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3725078:3725153 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3725078:3725153 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3725078:3725153 [0] NCCL INFO Using network AWS Libfabric +gpua039:3725078:3725153 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua039:3725078:3725153 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3725078:3725153 [0] NCCL INFO Trees [0] 25/28/-1->24->16 [1] 25/-1/-1->24->21 +gpua039:3725078:3725153 [0] NCCL INFO Channel 00/0 : 23[c7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Channel 01/0 : 23[c7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Channel 00/0 : 24[7000] -> 25[46000] via P2P/IPC/read +gpua039:3725078:3725153 [0] NCCL INFO Channel 01/0 : 24[7000] -> 25[46000] via P2P/IPC/read +gpua039:3725078:3725153 [0] NCCL INFO Connected all rings +gpua039:3725078:3725153 [0] NCCL INFO Channel 01/0 : 21[46000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Channel 00/0 : 24[7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Channel 00/0 : 16[7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Channel 00/0 : 24[7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Channel 00/0 : 28[7000] -> 24[7000] [receive] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Channel 01/0 : 24[7000] -> 21[46000] [send] via NET/AWS Libfabric/1 +gpua039:3725078:3725153 [0] NCCL INFO Connected all trees +gpua039:3725078:3725153 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3725078:3725153 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3725078:3725153 [0] NCCL INFO comm 0x55f2b4f4d870 rank 24 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua039:3725080:3725080 [2] NCCL INFO cudaDriverVersion 12020 +gpua039:3725080:3725080 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3725080:3725080 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3725080:3725080 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3725080:3725151 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3725080:3725151 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3725080:3725151 [2] NCCL INFO Using network AWS Libfabric +gpua039:3725080:3725151 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua039:3725080:3725151 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3725080:3725151 [2] NCCL INFO Trees [0] 27/-1/-1->26->25 [1] 27/-1/-1->26->25 +gpua039:3725080:3725151 [2] NCCL INFO Channel 00/0 : 26[85000] -> 27[c7000] via P2P/IPC/read +gpua039:3725080:3725151 [2] NCCL INFO Channel 01/0 : 26[85000] -> 27[c7000] via P2P/IPC/read +gpua039:3725080:3725151 [2] NCCL INFO Connected all rings +gpua039:3725080:3725151 [2] NCCL INFO Channel 00/0 : 26[85000] -> 25[46000] via P2P/IPC/read +gpua039:3725080:3725151 [2] NCCL INFO Channel 01/0 : 26[85000] -> 25[46000] via P2P/IPC/read +gpua039:3725080:3725151 [2] NCCL INFO Connected all trees +gpua039:3725080:3725151 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3725080:3725151 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3725080:3725151 [2] NCCL INFO comm 0x5641bf8ca160 rank 26 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua039:3725081:3725081 [3] NCCL INFO cudaDriverVersion 12020 +gpua039:3725081:3725081 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3725081:3725081 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3725081:3725081 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3725081:3725152 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3725081:3725152 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3725081:3725152 [3] NCCL INFO Using network AWS Libfabric +gpua039:3725081:3725152 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua039:3725081:3725152 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3725081:3725152 [3] NCCL INFO Trees [0] -1/-1/-1->27->26 [1] -1/-1/-1->27->26 +gpua039:3725081:3725152 [3] NCCL INFO Channel 00/0 : 27[c7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua039:3725081:3725152 [3] NCCL INFO Channel 01/0 : 27[c7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua039:3725081:3725152 [3] NCCL INFO Connected all rings +gpua039:3725081:3725152 [3] NCCL INFO Channel 00/0 : 27[c7000] -> 26[85000] via P2P/IPC/read +gpua039:3725081:3725152 [3] NCCL INFO Channel 01/0 : 27[c7000] -> 26[85000] via P2P/IPC/read +gpua039:3725081:3725152 [3] NCCL INFO Connected all trees +gpua039:3725081:3725152 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3725081:3725152 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3725081:3725152 [3] NCCL INFO comm 0x5639d6ba8ad0 rank 27 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua039:3725079:3725079 [1] NCCL INFO cudaDriverVersion 12020 +gpua039:3725079:3725079 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.39<0> +gpua039:3725079:3725079 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua039:3725079:3725079 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua039:3725079:3725154 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua039:3725079:3725154 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua039:3725079:3725154 [1] NCCL INFO Using network AWS Libfabric +gpua039:3725079:3725154 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua039:3725079:3725154 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua039:3725079:3725154 [1] NCCL INFO Trees [0] 26/20/-1->25->24 [1] 26/-1/-1->25->24 +gpua039:3725079:3725154 [1] NCCL INFO Channel 00/0 : 25[46000] -> 26[85000] via P2P/IPC/read +gpua039:3725079:3725154 [1] NCCL INFO Channel 01/0 : 25[46000] -> 26[85000] via P2P/IPC/read +gpua039:3725079:3725154 [1] NCCL INFO Connected all rings +gpua039:3725079:3725154 [1] NCCL INFO Channel 00/0 : 20[7000] -> 25[46000] [receive] via NET/AWS Libfabric/1 +gpua039:3725079:3725154 [1] NCCL INFO Channel 00/0 : 25[46000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua039:3725079:3725154 [1] NCCL INFO Channel 00/0 : 25[46000] -> 24[7000] via P2P/IPC/read +gpua039:3725079:3725154 [1] NCCL INFO Channel 01/0 : 25[46000] -> 24[7000] via P2P/IPC/read +gpua039:3725079:3725154 [1] NCCL INFO Connected all trees +gpua039:3725079:3725154 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua039:3725079:3725154 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua039:3725079:3725154 [1] NCCL INFO comm 0x55fd738f7350 rank 25 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua007:1643153:1643153 [2] NCCL INFO cudaDriverVersion 12020 +gpua007:1643153:1643153 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.7<0> +gpua007:1643153:1643153 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua007:1643153:1643153 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua007:1643153:1643210 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua007:1643153:1643210 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua007:1643153:1643210 [2] NCCL INFO Using network AWS Libfabric +gpua007:1643153:1643210 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua007:1643153:1643210 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua007:1643153:1643210 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5 +gpua007:1643153:1643210 [2] NCCL INFO Channel 00/0 : 6[85000] -> 7[c7000] via P2P/IPC/read +gpua007:1643153:1643210 [2] NCCL INFO Channel 01/0 : 6[85000] -> 7[c7000] via P2P/IPC/read +gpua007:1643153:1643210 [2] NCCL INFO Connected all rings +gpua007:1643153:1643210 [2] NCCL INFO Channel 00/0 : 6[85000] -> 5[46000] via P2P/IPC/read +gpua007:1643153:1643210 [2] NCCL INFO Channel 01/0 : 6[85000] -> 5[46000] via P2P/IPC/read +gpua007:1643153:1643210 [2] NCCL INFO Connected all trees +gpua007:1643153:1643210 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua007:1643153:1643210 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua007:1643153:1643210 [2] NCCL INFO comm 0x563c47dd8720 rank 6 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua055:183041:183041 [3] NCCL INFO cudaDriverVersion 12020 +gpua055:183041:183041 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:183041:183041 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:183041:183041 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:183041:183109 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua055:183041:183109 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:183041:183109 [3] NCCL INFO Using network AWS Libfabric +gpua055:183041:183109 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua055:183041:183109 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:183041:183109 [3] NCCL INFO Trees [0] -1/-1/-1->43->42 [1] -1/-1/-1->43->42 +gpua055:183041:183109 [3] NCCL INFO Channel 00/0 : 43[c7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua055:183041:183109 [3] NCCL INFO Channel 01/0 : 43[c7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua007:1643151:1643151 [0] NCCL INFO cudaDriverVersion 12020 +gpua007:1643151:1643151 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.7<0> +gpua007:1643151:1643151 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua007:1643151:1643151 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua007:1643151:1643212 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua007:1643151:1643212 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua007:1643151:1643212 [0] NCCL INFO Using network AWS Libfabric +gpua007:1643151:1643212 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua007:1643151:1643212 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua007:1643151:1643212 [0] NCCL INFO Trees [0] 5/-1/-1->4->9 [1] 5/0/-1->4->12 +gpua007:1643151:1643212 [0] NCCL INFO Channel 00/0 : 3[c7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua007:1643151:1643212 [0] NCCL INFO Channel 01/0 : 3[c7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua055:183041:183109 [3] NCCL INFO Connected all rings +gpua055:183041:183109 [3] NCCL INFO Channel 00/0 : 43[c7000] -> 42[85000] via P2P/IPC/read +gpua055:183041:183109 [3] NCCL INFO Channel 01/0 : 43[c7000] -> 42[85000] via P2P/IPC/read +gpua055:183041:183109 [3] NCCL INFO Connected all trees +gpua055:183041:183109 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:183041:183109 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:183041:183109 [3] NCCL INFO comm 0x555c5c0f2f80 rank 43 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua007:1643151:1643212 [0] NCCL INFO Channel 00/0 : 4[7000] -> 5[46000] via P2P/IPC/read +gpua007:1643151:1643212 [0] NCCL INFO Channel 01/0 : 4[7000] -> 5[46000] via P2P/IPC/read +gpua007:1643151:1643212 [0] NCCL INFO Connected all rings +gpua007:1643151:1643212 [0] NCCL INFO Channel 01/0 : 0[7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua007:1643151:1643212 [0] NCCL INFO Channel 00/0 : 4[7000] -> 9[46000] [send] via NET/AWS Libfabric/1 +gpua007:1643151:1643212 [0] NCCL INFO Channel 01/0 : 4[7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua007:1643151:1643212 [0] NCCL INFO Channel 01/0 : 12[7000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua007:1643151:1643212 [0] NCCL INFO Channel 00/0 : 9[46000] -> 4[7000] [receive] via NET/AWS Libfabric/1 +gpua007:1643151:1643212 [0] NCCL INFO Channel 01/0 : 4[7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua007:1643151:1643212 [0] NCCL INFO Connected all trees +gpua007:1643151:1643212 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:741723:741790 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:741723:741790 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:741723:741790 [0] NCCL INFO Using network AWS Libfabric +gpua006:741723:741790 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua006:741723:741790 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:741723:741790 [0] NCCL INFO Channel 00/02 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 +gpua006:741723:741790 [0] NCCL INFO Channel 01/02 : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 +gpua006:741723:741790 [0] NCCL INFO Trees [0] 1/32/-1->0->-1 [1] 1/-1/-1->0->4 +gpua006:741723:741790 [0] NCCL INFO Channel 00/0 : 63[c7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:741723:741790 [0] NCCL INFO Channel 01/0 : 63[c7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:741723:741790 [0] NCCL INFO Channel 00/0 : 0[7000] -> 1[46000] via P2P/IPC/read +gpua007:1643151:1643212 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua007:1643151:1643212 [0] NCCL INFO comm 0x56041fb4bf60 rank 4 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua006:741723:741790 [0] NCCL INFO Channel 01/0 : 0[7000] -> 1[46000] via P2P/IPC/read +gpua006:741723:741790 [0] NCCL INFO Connected all rings +gpua006:741723:741790 [0] NCCL INFO Channel 01/0 : 0[7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua006:741723:741790 [0] NCCL INFO Channel 00/0 : 32[7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:741723:741790 [0] NCCL INFO Channel 00/0 : 0[7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua006:741723:741790 [0] NCCL INFO Channel 01/0 : 4[7000] -> 0[7000] [receive] via NET/AWS Libfabric/1 +gpua006:741723:741790 [0] NCCL INFO Connected all trees +gpua006:741723:741790 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:741723:741790 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua006:741723:741790 [0] NCCL INFO comm 0x5628fb4d3e80 rank 0 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua055:183040:183040 [2] NCCL INFO cudaDriverVersion 12020 +gpua055:183040:183040 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:183040:183040 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:183040:183040 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:183040:183110 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua055:183040:183110 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:183040:183110 [2] NCCL INFO Using network AWS Libfabric +gpua055:183040:183110 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua055:183040:183110 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:183040:183110 [2] NCCL INFO Trees [0] 43/-1/-1->42->41 [1] 43/-1/-1->42->41 +gpua055:183040:183110 [2] NCCL INFO Channel 00/0 : 42[85000] -> 43[c7000] via P2P/IPC/read +gpua055:183040:183110 [2] NCCL INFO Channel 01/0 : 42[85000] -> 43[c7000] via P2P/IPC/read +gpua055:183040:183110 [2] NCCL INFO Connected all rings +gpua007:1643154:1643154 [3] NCCL INFO cudaDriverVersion 12020 +gpua007:1643154:1643154 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.7<0> +gpua007:1643154:1643154 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua007:1643154:1643154 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua007:1643154:1643211 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua007:1643154:1643211 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua007:1643154:1643211 [3] NCCL INFO Using network AWS Libfabric +gpua007:1643154:1643211 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua007:1643154:1643211 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua007:1643154:1643211 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6 +gpua007:1643154:1643211 [3] NCCL INFO Channel 00/0 : 7[c7000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua007:1643154:1643211 [3] NCCL INFO Channel 01/0 : 7[c7000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua057:3948781:3948781 [3] NCCL INFO cudaDriverVersion 12020 +gpua057:3948781:3948781 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3948781:3948781 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3948781:3948781 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3948781:3948850 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3948781:3948850 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3948781:3948850 [3] NCCL INFO Using network AWS Libfabric +gpua057:3948781:3948850 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua057:3948781:3948850 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3948781:3948850 [3] NCCL INFO Trees [0] -1/-1/-1->47->46 [1] -1/-1/-1->47->46 +gpua057:3948781:3948850 [3] NCCL INFO Channel 00/0 : 47[c7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua057:3948781:3948850 [3] NCCL INFO Channel 01/0 : 47[c7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua055:183040:183110 [2] NCCL INFO Channel 00/0 : 42[85000] -> 41[46000] via P2P/IPC/read +gpua055:183040:183110 [2] NCCL INFO Channel 01/0 : 42[85000] -> 41[46000] via P2P/IPC/read +gpua055:183040:183110 [2] NCCL INFO Connected all trees +gpua055:183040:183110 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:183040:183110 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:183040:183110 [2] NCCL INFO comm 0x55d1d949f320 rank 42 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua007:1643154:1643211 [3] NCCL INFO Connected all rings +gpua007:1643154:1643211 [3] NCCL INFO Channel 00/0 : 7[c7000] -> 6[85000] via P2P/IPC/read +gpua007:1643154:1643211 [3] NCCL INFO Channel 01/0 : 7[c7000] -> 6[85000] via P2P/IPC/read +gpua007:1643154:1643211 [3] NCCL INFO Connected all trees +gpua007:1643154:1643211 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua007:1643154:1643211 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua007:1643154:1643211 [3] NCCL INFO comm 0x55bf9993ac30 rank 7 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua057:3948781:3948850 [3] NCCL INFO Connected all rings +gpua057:3948781:3948850 [3] NCCL INFO Channel 00/0 : 47[c7000] -> 46[85000] via P2P/IPC/read +gpua057:3948781:3948850 [3] NCCL INFO Channel 01/0 : 47[c7000] -> 46[85000] via P2P/IPC/read +gpua057:3948781:3948850 [3] NCCL INFO Connected all trees +gpua057:3948781:3948850 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3948781:3948850 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3948781:3948850 [3] NCCL INFO comm 0x55a943954a60 rank 47 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua055:183039:183039 [1] NCCL INFO cudaDriverVersion 12020 +gpua055:183039:183039 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:183039:183039 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:183039:183039 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:183039:183108 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua055:183039:183108 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:183039:183108 [1] NCCL INFO Using network AWS Libfabric +gpua055:183039:183108 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua055:183039:183108 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:183039:183108 [1] NCCL INFO Trees [0] 42/36/-1->41->40 [1] 42/-1/-1->41->40 +gpua055:183039:183108 [1] NCCL INFO Channel 00/0 : 41[46000] -> 42[85000] via P2P/IPC/read +gpua055:183039:183108 [1] NCCL INFO Channel 01/0 : 41[46000] -> 42[85000] via P2P/IPC/read +gpua055:183039:183108 [1] NCCL INFO Connected all rings +gpua055:183039:183108 [1] NCCL INFO Channel 00/0 : 36[7000] -> 41[46000] [receive] via NET/AWS Libfabric/1 +gpua055:183039:183108 [1] NCCL INFO Channel 00/0 : 41[46000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua055:183039:183108 [1] NCCL INFO Channel 00/0 : 41[46000] -> 40[7000] via P2P/IPC/read +gpua055:183039:183108 [1] NCCL INFO Channel 01/0 : 41[46000] -> 40[7000] via P2P/IPC/read +gpua055:183039:183108 [1] NCCL INFO Connected all trees +gpua055:183039:183108 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:183039:183108 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:183039:183108 [1] NCCL INFO comm 0x55e364a262a0 rank 41 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua038:242424:242424 [3] NCCL INFO cudaDriverVersion 12020 +gpua038:242424:242424 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:242424:242424 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:242424:242424 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:242424:242508 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:242424:242508 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:242424:242508 [3] NCCL INFO Using network AWS Libfabric +gpua038:242424:242508 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua038:242424:242508 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:242424:242508 [3] NCCL INFO Trees [0] -1/-1/-1->23->22 [1] -1/-1/-1->23->22 +gpua038:242424:242508 [3] NCCL INFO Channel 00/0 : 23[c7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua038:242424:242508 [3] NCCL INFO Channel 01/0 : 23[c7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua054:483904:483904 [1] NCCL INFO cudaDriverVersion 12020 +gpua054:483904:483904 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:483904:483904 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua054:483904:483904 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:483904:483980 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:483904:483980 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:483904:483980 [1] NCCL INFO Using network AWS Libfabric +gpua054:483904:483980 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua054:483904:483980 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:483904:483980 [1] NCCL INFO Trees [0] 38/-1/-1->37->36 [1] 38/40/-1->37->36 +gpua054:483904:483980 [1] NCCL INFO Channel 00/0 : 37[46000] -> 38[85000] via P2P/IPC/read +gpua054:483904:483980 [1] NCCL INFO Channel 01/0 : 37[46000] -> 38[85000] via P2P/IPC/read +gpua054:483904:483980 [1] NCCL INFO Connected all rings +gpua012:2787975:2787975 [3] NCCL INFO cudaDriverVersion 12020 +gpua012:2787975:2787975 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua012:2787975:2787975 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2787975:2787975 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2787975:2788041 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2787975:2788041 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2787975:2788041 [3] NCCL INFO Using network AWS Libfabric +gpua012:2787975:2788041 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua012:2787975:2788041 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2787975:2788041 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] -1/-1/-1->11->10 +gpua012:2787975:2788041 [3] NCCL INFO Channel 00/0 : 11[c7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua012:2787975:2788041 [3] NCCL INFO Channel 01/0 : 11[c7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757850:3757850 [1] NCCL INFO cudaDriverVersion 12020 +gpua079:3757850:3757850 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3757850:3757850 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3757850:3757850 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3757850:3757906 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3757850:3757906 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3757850:3757906 [1] NCCL INFO Using network AWS Libfabric +gpua079:3757850:3757906 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua079:3757850:3757906 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3757850:3757906 [1] NCCL INFO Trees [0] 50/40/-1->49->48 [1] 50/-1/-1->49->48 +gpua079:3757850:3757906 [1] NCCL INFO Channel 00/0 : 49[46000] -> 50[85000] via P2P/IPC/read +gpua079:3757850:3757906 [1] NCCL INFO Channel 01/0 : 49[46000] -> 50[85000] via P2P/IPC/read +gpua038:242424:242508 [3] NCCL INFO Connected all rings +gpua038:242424:242508 [3] NCCL INFO Channel 00/0 : 23[c7000] -> 22[85000] via P2P/IPC/read +gpua038:242424:242508 [3] NCCL INFO Channel 01/0 : 23[c7000] -> 22[85000] via P2P/IPC/read +gpua038:242424:242508 [3] NCCL INFO Connected all trees +gpua038:242424:242508 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua038:242424:242508 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:242424:242508 [3] NCCL INFO comm 0x557ed448ca10 rank 23 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua054:483904:483980 [1] NCCL INFO Channel 01/0 : 37[46000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua054:483904:483980 [1] NCCL INFO Channel 01/0 : 40[7000] -> 37[46000] [receive] via NET/AWS Libfabric/1 +gpua054:483904:483980 [1] NCCL INFO Channel 00/0 : 37[46000] -> 36[7000] via P2P/IPC/read +gpua054:483904:483980 [1] NCCL INFO Channel 01/0 : 37[46000] -> 36[7000] via P2P/IPC/read +gpua054:483904:483980 [1] NCCL INFO Connected all trees +gpua054:483904:483980 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:483904:483980 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:483904:483980 [1] NCCL INFO comm 0x562e360f4910 rank 37 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua054:483905:483905 [2] NCCL INFO cudaDriverVersion 12020 +gpua054:483905:483905 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:483905:483905 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2787975:2788041 [3] NCCL INFO Connected all rings +gpua012:2787975:2788041 [3] NCCL INFO Channel 00/0 : 11[c7000] -> 10[85000] via P2P/IPC/read +gpua012:2787975:2788041 [3] NCCL INFO Channel 01/0 : 11[c7000] -> 10[85000] via P2P/IPC/read +gpua012:2787975:2788041 [3] NCCL INFO Connected all trees +gpua012:2787975:2788041 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua012:2787975:2788041 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2787975:2788041 [3] NCCL INFO comm 0x5584de3f1da0 rank 11 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua057:3948778:3948778 [0] NCCL INFO cudaDriverVersion 12020 +gpua057:3948778:3948778 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3948778:3948778 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3948778:3948778 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3948778:3948847 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3948778:3948847 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3948778:3948847 [0] NCCL INFO Using network AWS Libfabric +gpua057:3948778:3948847 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua057:3948778:3948847 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3948778:3948847 [0] NCCL INFO Trees [0] 45/-1/-1->44->40 [1] 45/36/-1->44->29 +gpua057:3948778:3948847 [0] NCCL INFO Channel 00/0 : 43[c7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3757850:3757906 [1] NCCL INFO Connected all rings +gpua079:3757850:3757906 [1] NCCL INFO Channel 00/0 : 40[7000] -> 49[46000] [receive] via NET/AWS Libfabric/1 +gpua079:3757850:3757906 [1] NCCL INFO Channel 00/0 : 49[46000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757850:3757906 [1] NCCL INFO Channel 00/0 : 49[46000] -> 48[7000] via P2P/IPC/read +gpua079:3757850:3757906 [1] NCCL INFO Channel 01/0 : 49[46000] -> 48[7000] via P2P/IPC/read +gpua079:3757850:3757906 [1] NCCL INFO Connected all trees +gpua079:3757850:3757906 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3757850:3757906 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3757850:3757906 [1] NCCL INFO comm 0x55d6b7ef75e0 rank 49 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua038:242421:242421 [0] NCCL INFO cudaDriverVersion 12020 +gpua038:242421:242421 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:242421:242421 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:242421:242421 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:242421:242505 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:242421:242505 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:242421:242505 [0] NCCL INFO Using network AWS Libfabric +gpua038:242421:242505 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua038:242421:242505 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:242421:242505 [0] NCCL INFO Trees [0] 21/-1/-1->20->25 [1] 21/16/-1->20->13 +gpua038:242421:242505 [0] NCCL INFO Channel 00/0 : 19[c7000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Channel 01/0 : 19[c7000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua006:741726:741726 [3] NCCL INFO cudaDriverVersion 12020 +gpua006:741726:741726 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:741726:741726 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:741726:741726 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:741726:741791 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:741726:741791 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:741726:741791 [3] NCCL INFO Using network AWS Libfabric +gpua006:741726:741791 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua006:741726:741791 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:741726:741791 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 +gpua006:741726:741791 [3] NCCL INFO Channel 00/0 : 3[c7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua006:741726:741791 [3] NCCL INFO Channel 01/0 : 3[c7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua033:1707824:1707824 [0] NCCL INFO cudaDriverVersion 12020 +gpua033:1707824:1707824 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua033:1707824:1707824 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1707824:1707824 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1707824:1707898 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1707824:1707898 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1707824:1707898 [0] NCCL INFO Using network AWS Libfabric +gpua033:1707824:1707898 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua033:1707824:1707898 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1707824:1707898 [0] NCCL INFO Trees [0] 17/24/-1->16->33 [1] 17/-1/-1->16->20 +gpua033:1707824:1707898 [0] NCCL INFO Channel 00/0 : 15[c7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua054:483905:483905 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:483905:483979 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:483905:483979 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:483905:483979 [2] NCCL INFO Using network AWS Libfabric +gpua054:483905:483979 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua054:483905:483979 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:483905:483979 [2] NCCL INFO Trees [0] 39/-1/-1->38->37 [1] 39/-1/-1->38->37 +gpua054:483905:483979 [2] NCCL INFO Channel 00/0 : 38[85000] -> 39[c7000] via P2P/IPC/read +gpua054:483905:483979 [2] NCCL INFO Channel 01/0 : 38[85000] -> 39[c7000] via P2P/IPC/read +gpua054:483905:483979 [2] NCCL INFO Connected all rings +gpua054:483905:483979 [2] NCCL INFO Channel 00/0 : 38[85000] -> 37[46000] via P2P/IPC/read +gpua054:483905:483979 [2] NCCL INFO Channel 01/0 : 38[85000] -> 37[46000] via P2P/IPC/read +gpua049:4111892:4111892 [1] NCCL INFO cudaDriverVersion 12020 +gpua049:4111892:4111892 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua049:4111892:4111892 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:4111892:4111892 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:4111892:4111964 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:4111892:4111964 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:4111892:4111964 [1] NCCL INFO Using network AWS Libfabric +gpua049:4111892:4111964 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua049:4111892:4111964 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:4111892:4111964 [1] NCCL INFO Trees [0] 34/16/-1->33->32 [1] 34/-1/-1->33->32 +gpua049:4111892:4111964 [1] NCCL INFO Channel 00/0 : 33[46000] -> 34[85000] via P2P/IPC/read +gpua049:4111892:4111964 [1] NCCL INFO Channel 01/0 : 33[46000] -> 34[85000] via P2P/IPC/read +gpua057:3948778:3948847 [0] NCCL INFO Channel 01/0 : 43[c7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3948778:3948847 [0] NCCL INFO Channel 00/0 : 44[7000] -> 45[46000] via P2P/IPC/read +gpua057:3948778:3948847 [0] NCCL INFO Channel 01/0 : 44[7000] -> 45[46000] via P2P/IPC/read +gpua057:3948778:3948847 [0] NCCL INFO Connected all rings +gpua057:3948778:3948847 [0] NCCL INFO Channel 00/0 : 40[7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3948778:3948847 [0] NCCL INFO Channel 01/0 : 36[7000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3948778:3948847 [0] NCCL INFO Channel 01/0 : 29[46000] -> 44[7000] [receive] via NET/AWS Libfabric/1 +gpua057:3948778:3948847 [0] NCCL INFO Channel 01/0 : 44[7000] -> 29[46000] [send] via NET/AWS Libfabric/1 +gpua057:3948778:3948847 [0] NCCL INFO Channel 01/0 : 44[7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua057:3948778:3948847 [0] NCCL INFO Channel 00/0 : 44[7000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua040:3922553:3922553 [0] NCCL INFO cudaDriverVersion 12020 +gpua040:3922553:3922553 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3922553:3922553 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3922553:3922553 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3922553:3922631 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua040:3922553:3922631 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3922553:3922631 [0] NCCL INFO Using network AWS Libfabric +gpua040:3922553:3922631 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua040:3922553:3922631 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3922553:3922631 [0] NCCL INFO Trees [0] 29/-1/-1->28->24 [1] 29/12/-1->28->60 +gpua040:3922553:3922631 [0] NCCL INFO Channel 00/0 : 27[c7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3757849:3757849 [0] NCCL INFO cudaDriverVersion 12020 +gpua079:3757849:3757849 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3757849:3757849 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3757849:3757849 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3757849:3757907 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3757849:3757907 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3757849:3757907 [0] NCCL INFO Using network AWS Libfabric +gpua079:3757849:3757907 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua079:3757849:3757907 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3757849:3757907 [0] NCCL INFO Trees [0] 49/56/-1->48->32 [1] 49/-1/-1->48->52 +gpua079:3757849:3757907 [0] NCCL INFO Channel 00/0 : 47[c7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Channel 00/0 : 20[7000] -> 21[46000] via P2P/IPC/read +gpua038:242421:242505 [0] NCCL INFO Channel 01/0 : 20[7000] -> 21[46000] via P2P/IPC/read +gpua038:242421:242505 [0] NCCL INFO Connected all rings +gpua038:242421:242505 [0] NCCL INFO Channel 01/0 : 16[7000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Channel 00/0 : 20[7000] -> 25[46000] [send] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Channel 01/0 : 13[46000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Channel 01/0 : 20[7000] -> 13[46000] [send] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Channel 00/0 : 25[46000] -> 20[7000] [receive] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Channel 01/0 : 20[7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO Connected all trees +gpua038:242421:242505 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:913542:913542 [2] NCCL INFO cudaDriverVersion 12020 +gpua089:913542:913542 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:913542:913542 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:913542:913542 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:913542:913611 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:913542:913611 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua089:913542:913611 [2] NCCL INFO Using network AWS Libfabric +gpua089:913542:913611 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua089:913542:913611 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:913542:913611 [2] NCCL INFO Trees [0] 63/-1/-1->62->61 [1] 63/-1/-1->62->61 +gpua089:913542:913611 [2] NCCL INFO Channel 00/0 : 62[85000] -> 63[c7000] via P2P/IPC/read +gpua089:913542:913611 [2] NCCL INFO Channel 01/0 : 62[85000] -> 63[c7000] via P2P/IPC/read +gpua089:913542:913611 [2] NCCL INFO Connected all rings +gpua006:741726:741791 [3] NCCL INFO Connected all rings +gpua006:741726:741791 [3] NCCL INFO Channel 00/0 : 3[c7000] -> 2[85000] via P2P/IPC/read +gpua006:741726:741791 [3] NCCL INFO Channel 01/0 : 3[c7000] -> 2[85000] via P2P/IPC/read +gpua006:741726:741791 [3] NCCL INFO Connected all trees +gpua006:741726:741791 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:741726:741791 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua006:741726:741791 [3] NCCL INFO comm 0x56487fe7b2a0 rank 3 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua033:1707824:1707898 [0] NCCL INFO Channel 01/0 : 15[c7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1707824:1707898 [0] NCCL INFO Channel 00/0 : 16[7000] -> 17[46000] via P2P/IPC/read +gpua033:1707824:1707898 [0] NCCL INFO Channel 01/0 : 16[7000] -> 17[46000] via P2P/IPC/read +gpua033:1707824:1707898 [0] NCCL INFO Connected all rings +gpua033:1707824:1707898 [0] NCCL INFO Channel 01/0 : 16[7000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua033:1707824:1707898 [0] NCCL INFO Channel 00/0 : 16[7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua033:1707824:1707898 [0] NCCL INFO Channel 00/0 : 16[7000] -> 33[46000] [send] via NET/AWS Libfabric/1 +gpua033:1707824:1707898 [0] NCCL INFO Channel 00/0 : 33[46000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1707824:1707898 [0] NCCL INFO Channel 00/0 : 24[7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua033:1707824:1707898 [0] NCCL INFO Channel 01/0 : 20[7000] -> 16[7000] [receive] via NET/AWS Libfabric/1 +gpua054:483905:483979 [2] NCCL INFO Connected all trees +gpua054:483905:483979 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:483905:483979 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:483905:483979 [2] NCCL INFO comm 0x55c7d1072e90 rank 38 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua012:2787974:2787974 [2] NCCL INFO cudaDriverVersion 12020 +gpua012:2787974:2787974 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua012:2787974:2787974 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2787974:2787974 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2787974:2788044 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2787974:2788044 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2787974:2788044 [2] NCCL INFO Using network AWS Libfabric +gpua012:2787974:2788044 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua012:2787974:2788044 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2787974:2788044 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9 +gpua012:2787974:2788044 [2] NCCL INFO Channel 00/0 : 10[85000] -> 11[c7000] via P2P/IPC/read +gpua012:2787974:2788044 [2] NCCL INFO Channel 01/0 : 10[85000] -> 11[c7000] via P2P/IPC/read +gpua049:4111892:4111964 [1] NCCL INFO Connected all rings +gpua049:4111892:4111964 [1] NCCL INFO Channel 00/0 : 16[7000] -> 33[46000] [receive] via NET/AWS Libfabric/1 +gpua049:4111892:4111964 [1] NCCL INFO Channel 00/0 : 33[46000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua049:4111892:4111964 [1] NCCL INFO Channel 00/0 : 33[46000] -> 32[7000] via P2P/IPC/read +gpua049:4111892:4111964 [1] NCCL INFO Channel 01/0 : 33[46000] -> 32[7000] via P2P/IPC/read +gpua049:4111892:4111964 [1] NCCL INFO Connected all trees +gpua049:4111892:4111964 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:4111892:4111964 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:4111892:4111964 [1] NCCL INFO comm 0x56102c3982a0 rank 33 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua049:4111891:4111891 [0] NCCL INFO cudaDriverVersion 12020 +gpua049:4111891:4111891 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua057:3948778:3948847 [0] NCCL INFO Connected all trees +gpua057:3948778:3948847 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3948778:3948847 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3948778:3948847 [0] NCCL INFO comm 0x55a4cff267f0 rank 44 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua080:3333275:3333275 [3] NCCL INFO cudaDriverVersion 12020 +gpua080:3333275:3333275 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua080:3333275:3333275 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3333275:3333275 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3333275:3333350 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3333275:3333350 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3333275:3333350 [3] NCCL INFO Using network AWS Libfabric +gpua080:3333275:3333350 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua080:3333275:3333350 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3333275:3333350 [3] NCCL INFO Trees [0] -1/-1/-1->55->54 [1] -1/-1/-1->55->54 +gpua080:3333275:3333350 [3] NCCL INFO Channel 00/0 : 55[c7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua080:3333275:3333350 [3] NCCL INFO Channel 01/0 : 55[c7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua040:3922553:3922631 [0] NCCL INFO Channel 01/0 : 27[c7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3922553:3922631 [0] NCCL INFO Channel 00/0 : 28[7000] -> 29[46000] via P2P/IPC/read +gpua040:3922553:3922631 [0] NCCL INFO Channel 01/0 : 28[7000] -> 29[46000] via P2P/IPC/read +gpua040:3922553:3922631 [0] NCCL INFO Connected all rings +gpua040:3922553:3922631 [0] NCCL INFO Channel 00/0 : 24[7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3922553:3922631 [0] NCCL INFO Channel 01/0 : 12[7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3922553:3922631 [0] NCCL INFO Channel 01/0 : 60[7000] -> 28[7000] [receive] via NET/AWS Libfabric/1 +gpua040:3922553:3922631 [0] NCCL INFO Channel 01/0 : 28[7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua040:3922553:3922631 [0] NCCL INFO Channel 01/0 : 28[7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua040:3922553:3922631 [0] NCCL INFO Channel 00/0 : 28[7000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757849:3757907 [0] NCCL INFO Channel 01/0 : 47[c7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3757849:3757907 [0] NCCL INFO Channel 00/0 : 48[7000] -> 49[46000] via P2P/IPC/read +gpua079:3757849:3757907 [0] NCCL INFO Channel 01/0 : 48[7000] -> 49[46000] via P2P/IPC/read +gpua079:3757849:3757907 [0] NCCL INFO Connected all rings +gpua079:3757849:3757907 [0] NCCL INFO Channel 01/0 : 48[7000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757849:3757907 [0] NCCL INFO Channel 00/0 : 48[7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757849:3757907 [0] NCCL INFO Channel 00/0 : 32[7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3757849:3757907 [0] NCCL INFO Channel 00/0 : 48[7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757849:3757907 [0] NCCL INFO Channel 00/0 : 56[7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua079:3757849:3757907 [0] NCCL INFO Channel 01/0 : 52[7000] -> 48[7000] [receive] via NET/AWS Libfabric/1 +gpua038:242421:242505 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:242421:242505 [0] NCCL INFO comm 0x55e71b2c7960 rank 20 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua089:913542:913611 [2] NCCL INFO Channel 00/0 : 62[85000] -> 61[46000] via P2P/IPC/read +gpua089:913542:913611 [2] NCCL INFO Channel 01/0 : 62[85000] -> 61[46000] via P2P/IPC/read +gpua089:913542:913611 [2] NCCL INFO Connected all trees +gpua089:913542:913611 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:913542:913611 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua089:913542:913611 [2] NCCL INFO comm 0x5641e4c994a0 rank 62 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua089:913541:913541 [1] NCCL INFO cudaDriverVersion 12020 +gpua089:913541:913541 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:913541:913541 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:913541:913541 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:913541:913612 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:913541:913612 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:741725:741725 [2] NCCL INFO cudaDriverVersion 12020 +gpua006:741725:741725 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:741725:741725 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:741725:741725 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:741725:741793 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:741725:741793 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:741725:741793 [2] NCCL INFO Using network AWS Libfabric +gpua006:741725:741793 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua006:741725:741793 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:741725:741793 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 +gpua006:741725:741793 [2] NCCL INFO Channel 00/0 : 2[85000] -> 3[c7000] via P2P/IPC/read +gpua006:741725:741793 [2] NCCL INFO Channel 01/0 : 2[85000] -> 3[c7000] via P2P/IPC/read +gpua006:741725:741793 [2] NCCL INFO Connected all rings +gpua033:1707824:1707898 [0] NCCL INFO Connected all trees +gpua033:1707824:1707898 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1707824:1707898 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1707824:1707898 [0] NCCL INFO comm 0x558abbb369f0 rank 16 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua033:1707827:1707827 [3] NCCL INFO cudaDriverVersion 12020 +gpua033:1707827:1707827 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua033:1707827:1707827 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1707827:1707827 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1707827:1707897 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1707827:1707897 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1707827:1707897 [3] NCCL INFO Using network AWS Libfabric +gpua033:1707827:1707897 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua054:483903:483903 [0] NCCL INFO cudaDriverVersion 12020 +gpua054:483903:483903 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:483903:483903 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua054:483903:483903 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:483903:483977 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:483903:483977 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:483903:483977 [0] NCCL INFO Using network AWS Libfabric +gpua054:483903:483977 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua054:483903:483977 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:483903:483977 [0] NCCL INFO Trees [0] 37/-1/-1->36->41 [1] 37/32/-1->36->44 +gpua054:483903:483977 [0] NCCL INFO Channel 00/0 : 35[c7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:483903:483977 [0] NCCL INFO Channel 01/0 : 35[c7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2787974:2788044 [2] NCCL INFO Connected all rings +gpua012:2787974:2788044 [2] NCCL INFO Channel 00/0 : 10[85000] -> 9[46000] via P2P/IPC/read +gpua012:2787974:2788044 [2] NCCL INFO Channel 01/0 : 10[85000] -> 9[46000] via P2P/IPC/read +gpua012:2787974:2788044 [2] NCCL INFO Connected all trees +gpua012:2787974:2788044 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua012:2787974:2788044 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2787974:2788044 [2] NCCL INFO comm 0x563bbe1358e0 rank 10 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua049:4111891:4111891 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:4111891:4111891 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:4111891:4111963 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:4111891:4111963 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:4111891:4111963 [0] NCCL INFO Using network AWS Libfabric +gpua049:4111891:4111963 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua049:4111891:4111963 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:4111891:4111963 [0] NCCL INFO Trees [0] 33/48/-1->32->0 [1] 33/-1/-1->32->36 +gpua049:4111891:4111963 [0] NCCL INFO Channel 00/0 : 31[c7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 01/0 : 31[c7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 00/0 : 32[7000] -> 33[46000] via P2P/IPC/read +gpua080:3333275:3333350 [3] NCCL INFO Connected all rings +gpua080:3333275:3333350 [3] NCCL INFO Channel 00/0 : 55[c7000] -> 54[85000] via P2P/IPC/read +gpua080:3333275:3333350 [3] NCCL INFO Channel 01/0 : 55[c7000] -> 54[85000] via P2P/IPC/read +gpua080:3333275:3333350 [3] NCCL INFO Connected all trees +gpua080:3333275:3333350 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua080:3333275:3333350 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3333275:3333350 [3] NCCL INFO comm 0x55745f903ac0 rank 55 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua040:3922553:3922631 [0] NCCL INFO Connected all trees +gpua040:3922553:3922631 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3922553:3922631 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3922553:3922631 [0] NCCL INFO comm 0x55e025fd1d80 rank 28 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua079:3757849:3757907 [0] NCCL INFO Connected all trees +gpua079:3757849:3757907 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3757849:3757907 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3757849:3757907 [0] NCCL INFO comm 0x55c5f6605310 rank 48 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua089:913541:913612 [1] NCCL INFO Using network AWS Libfabric +gpua089:913541:913612 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua089:913541:913612 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:913541:913612 [1] NCCL INFO Trees [0] 62/-1/-1->61->60 [1] 62/-1/-1->61->60 +gpua089:913541:913612 [1] NCCL INFO Channel 00/0 : 61[46000] -> 62[85000] via P2P/IPC/read +gpua089:913541:913612 [1] NCCL INFO Channel 01/0 : 61[46000] -> 62[85000] via P2P/IPC/read +gpua089:913541:913612 [1] NCCL INFO Connected all rings +gpua089:913541:913612 [1] NCCL INFO Channel 00/0 : 61[46000] -> 60[7000] via P2P/IPC/read +gpua089:913541:913612 [1] NCCL INFO Channel 01/0 : 61[46000] -> 60[7000] via P2P/IPC/read +gpua089:913541:913612 [1] NCCL INFO Connected all trees +gpua089:913541:913612 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:913541:913612 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:4081113:4081113 [3] NCCL INFO cudaDriverVersion 12020 +gpua085:4081113:4081113 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:4081113:4081113 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:4081113:4081113 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:4081113:4081178 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua085:4081113:4081178 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:4081113:4081178 [3] NCCL INFO Using network AWS Libfabric +gpua085:4081113:4081178 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua085:4081113:4081178 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:4081113:4081178 [3] NCCL INFO Trees [0] -1/-1/-1->59->58 [1] -1/-1/-1->59->58 +gpua085:4081113:4081178 [3] NCCL INFO Channel 00/0 : 59[c7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua085:4081113:4081178 [3] NCCL INFO Channel 01/0 : 59[c7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua006:741725:741793 [2] NCCL INFO Channel 00/0 : 2[85000] -> 1[46000] via P2P/IPC/read +gpua006:741725:741793 [2] NCCL INFO Channel 01/0 : 2[85000] -> 1[46000] via P2P/IPC/read +gpua006:741725:741793 [2] NCCL INFO Connected all trees +gpua006:741725:741793 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:741725:741793 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua006:741725:741793 [2] NCCL INFO comm 0x55b3f40d0e00 rank 2 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua033:1707827:1707897 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1707827:1707897 [3] NCCL INFO Trees [0] -1/-1/-1->19->18 [1] -1/-1/-1->19->18 +gpua033:1707827:1707897 [3] NCCL INFO Channel 00/0 : 19[c7000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua033:1707827:1707897 [3] NCCL INFO Channel 01/0 : 19[c7000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua033:1707827:1707897 [3] NCCL INFO Connected all rings +gpua033:1707827:1707897 [3] NCCL INFO Channel 00/0 : 19[c7000] -> 18[85000] via P2P/IPC/read +gpua033:1707827:1707897 [3] NCCL INFO Channel 01/0 : 19[c7000] -> 18[85000] via P2P/IPC/read +gpua033:1707827:1707897 [3] NCCL INFO Connected all trees +gpua033:1707827:1707897 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1707827:1707897 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1707827:1707897 [3] NCCL INFO comm 0x55cf31aeda70 rank 19 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua054:483903:483977 [0] NCCL INFO Channel 00/0 : 36[7000] -> 37[46000] via P2P/IPC/read +gpua054:483903:483977 [0] NCCL INFO Channel 01/0 : 36[7000] -> 37[46000] via P2P/IPC/read +gpua054:483903:483977 [0] NCCL INFO Connected all rings +gpua054:483903:483977 [0] NCCL INFO Channel 01/0 : 32[7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:483903:483977 [0] NCCL INFO Channel 00/0 : 36[7000] -> 41[46000] [send] via NET/AWS Libfabric/1 +gpua054:483903:483977 [0] NCCL INFO Channel 01/0 : 36[7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua054:483903:483977 [0] NCCL INFO Channel 01/0 : 44[7000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:483903:483977 [0] NCCL INFO Channel 00/0 : 41[46000] -> 36[7000] [receive] via NET/AWS Libfabric/1 +gpua054:483903:483977 [0] NCCL INFO Channel 01/0 : 36[7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua054:483903:483977 [0] NCCL INFO Connected all trees +gpua054:483903:483977 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua012:2787972:2787972 [0] NCCL INFO cudaDriverVersion 12020 +gpua012:2787972:2787972 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua012:2787972:2787972 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2787972:2787972 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2787972:2788042 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2787972:2788042 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2787972:2788042 [0] NCCL INFO Using network AWS Libfabric +gpua012:2787972:2788042 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua012:2787972:2788042 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2787972:2788042 [0] NCCL INFO Trees [0] 9/12/-1->8->17 [1] 9/-1/-1->8->5 +gpua012:2787972:2788042 [0] NCCL INFO Channel 00/0 : 7[c7000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2787972:2788042 [0] NCCL INFO Channel 01/0 : 7[c7000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua016:645598:645598 [3] NCCL INFO cudaDriverVersion 12020 +gpua016:645598:645598 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:645598:645598 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:645598:645598 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:645598:645662 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:645598:645662 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:645598:645662 [3] NCCL INFO Using network AWS Libfabric +gpua016:645598:645662 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua016:645598:645662 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:645598:645662 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] -1/-1/-1->15->14 +gpua016:645598:645662 [3] NCCL INFO Channel 00/0 : 15[c7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua016:645598:645662 [3] NCCL INFO Channel 01/0 : 15[c7000] -> 16[7000] [send] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 01/0 : 32[7000] -> 33[46000] via P2P/IPC/read +gpua049:4111891:4111963 [0] NCCL INFO Connected all rings +gpua049:4111891:4111963 [0] NCCL INFO Channel 01/0 : 32[7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 00/0 : 32[7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 00/0 : 0[7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 00/0 : 32[7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 00/0 : 48[7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Channel 01/0 : 36[7000] -> 32[7000] [receive] via NET/AWS Libfabric/1 +gpua049:4111891:4111963 [0] NCCL INFO Connected all trees +gpua049:4111891:4111963 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3922555:3922555 [2] NCCL INFO cudaDriverVersion 12020 +gpua040:3922555:3922555 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3922555:3922555 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3922555:3922555 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3922555:3922633 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua040:3922555:3922633 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3922555:3922633 [2] NCCL INFO Using network AWS Libfabric +gpua040:3922555:3922633 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua040:3922555:3922633 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3922555:3922633 [2] NCCL INFO Trees [0] 31/-1/-1->30->29 [1] 31/-1/-1->30->29 +gpua040:3922555:3922633 [2] NCCL INFO Channel 00/0 : 30[85000] -> 31[c7000] via P2P/IPC/read +gpua040:3922555:3922633 [2] NCCL INFO Channel 01/0 : 30[85000] -> 31[c7000] via P2P/IPC/read +gpua089:913541:913612 [1] NCCL INFO comm 0x55fc24929250 rank 61 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua085:4081113:4081178 [3] NCCL INFO Connected all rings +gpua085:4081113:4081178 [3] NCCL INFO Channel 00/0 : 59[c7000] -> 58[85000] via P2P/IPC/read +gpua085:4081113:4081178 [3] NCCL INFO Channel 01/0 : 59[c7000] -> 58[85000] via P2P/IPC/read +gpua085:4081113:4081178 [3] NCCL INFO Connected all trees +gpua085:4081113:4081178 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:4081113:4081178 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:4081113:4081178 [3] NCCL INFO comm 0x555bbae1dff0 rank 59 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua085:4081111:4081111 [1] NCCL INFO cudaDriverVersion 12020 +gpua085:4081111:4081111 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:4081111:4081111 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:4081111:4081111 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:4081111:4081179 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:741724:741724 [1] NCCL INFO cudaDriverVersion 12020 +gpua006:741724:741724 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.6<0> +gpua006:741724:741724 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua006:741724:741724 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua006:741724:741792 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua006:741724:741792 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua006:741724:741792 [1] NCCL INFO Using network AWS Libfabric +gpua006:741724:741792 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua006:741724:741792 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua006:741724:741792 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 +gpua006:741724:741792 [1] NCCL INFO Channel 00/0 : 1[46000] -> 2[85000] via P2P/IPC/read +gpua006:741724:741792 [1] NCCL INFO Channel 01/0 : 1[46000] -> 2[85000] via P2P/IPC/read +gpua006:741724:741792 [1] NCCL INFO Connected all rings +gpua033:1707825:1707825 [1] NCCL INFO cudaDriverVersion 12020 +gpua033:1707825:1707825 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua033:1707825:1707825 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1707825:1707825 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1707825:1707899 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1707825:1707899 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1707825:1707899 [1] NCCL INFO Using network AWS Libfabric +gpua033:1707825:1707899 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua033:1707825:1707899 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1707825:1707899 [1] NCCL INFO Trees [0] 18/8/-1->17->16 [1] 18/-1/-1->17->16 +gpua033:1707825:1707899 [1] NCCL INFO Channel 00/0 : 17[46000] -> 18[85000] via P2P/IPC/read +gpua033:1707825:1707899 [1] NCCL INFO Channel 01/0 : 17[46000] -> 18[85000] via P2P/IPC/read +gpua054:483903:483977 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:483903:483977 [0] NCCL INFO comm 0x5565ef5ba2a0 rank 36 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua012:2787972:2788042 [0] NCCL INFO Channel 00/0 : 8[7000] -> 9[46000] via P2P/IPC/read +gpua012:2787972:2788042 [0] NCCL INFO Channel 01/0 : 8[7000] -> 9[46000] via P2P/IPC/read +gpua012:2787972:2788042 [0] NCCL INFO Connected all rings +gpua012:2787972:2788042 [0] NCCL INFO Channel 01/0 : 5[46000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2787972:2788042 [0] NCCL INFO Channel 00/0 : 8[7000] -> 12[7000] [send] via NET/AWS Libfabric/1 +gpua012:2787972:2788042 [0] NCCL INFO Channel 00/0 : 8[7000] -> 17[46000] [send] via NET/AWS Libfabric/1 +gpua012:2787972:2788042 [0] NCCL INFO Channel 00/0 : 17[46000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2787972:2788042 [0] NCCL INFO Channel 00/0 : 12[7000] -> 8[7000] [receive] via NET/AWS Libfabric/1 +gpua012:2787972:2788042 [0] NCCL INFO Channel 01/0 : 8[7000] -> 5[46000] [send] via NET/AWS Libfabric/1 +gpua012:2787972:2788042 [0] NCCL INFO Connected all trees +gpua012:2787972:2788042 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua016:645598:645662 [3] NCCL INFO Connected all rings +gpua016:645598:645662 [3] NCCL INFO Channel 00/0 : 15[c7000] -> 14[85000] via P2P/IPC/read +gpua016:645598:645662 [3] NCCL INFO Channel 01/0 : 15[c7000] -> 14[85000] via P2P/IPC/read +gpua016:645598:645662 [3] NCCL INFO Connected all trees +gpua016:645598:645662 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua016:645598:645662 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:645598:645662 [3] NCCL INFO comm 0x55e2ac580f80 rank 15 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua049:4111891:4111963 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:4111891:4111963 [0] NCCL INFO comm 0x55a0a8023d20 rank 32 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua040:3922555:3922633 [2] NCCL INFO Connected all rings +gpua040:3922555:3922633 [2] NCCL INFO Channel 00/0 : 30[85000] -> 29[46000] via P2P/IPC/read +gpua040:3922555:3922633 [2] NCCL INFO Channel 01/0 : 30[85000] -> 29[46000] via P2P/IPC/read +gpua040:3922555:3922633 [2] NCCL INFO Connected all trees +gpua040:3922555:3922633 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3922555:3922633 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3922555:3922633 [2] NCCL INFO comm 0x55de12cc0970 rank 30 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua085:4081111:4081179 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:4081111:4081179 [1] NCCL INFO Using network AWS Libfabric +gpua085:4081111:4081179 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua085:4081111:4081179 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:4081111:4081179 [1] NCCL INFO Trees [0] 58/52/-1->57->56 [1] 58/-1/-1->57->56 +gpua085:4081111:4081179 [1] NCCL INFO Channel 00/0 : 57[46000] -> 58[85000] via P2P/IPC/read +gpua085:4081111:4081179 [1] NCCL INFO Channel 01/0 : 57[46000] -> 58[85000] via P2P/IPC/read +gpua085:4081111:4081179 [1] NCCL INFO Connected all rings +gpua085:4081111:4081179 [1] NCCL INFO Channel 00/0 : 52[7000] -> 57[46000] [receive] via NET/AWS Libfabric/1 +gpua085:4081111:4081179 [1] NCCL INFO Channel 00/0 : 57[46000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua085:4081111:4081179 [1] NCCL INFO Channel 00/0 : 57[46000] -> 56[7000] via P2P/IPC/read +gpua006:741724:741792 [1] NCCL INFO Channel 00/0 : 1[46000] -> 0[7000] via P2P/IPC/read +gpua006:741724:741792 [1] NCCL INFO Channel 01/0 : 1[46000] -> 0[7000] via P2P/IPC/read +gpua006:741724:741792 [1] NCCL INFO Connected all trees +gpua006:741724:741792 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua006:741724:741792 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua006:741724:741792 [1] NCCL INFO comm 0x560539192390 rank 1 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua033:1707825:1707899 [1] NCCL INFO Connected all rings +gpua033:1707825:1707899 [1] NCCL INFO Channel 00/0 : 8[7000] -> 17[46000] [receive] via NET/AWS Libfabric/1 +gpua033:1707825:1707899 [1] NCCL INFO Channel 00/0 : 17[46000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua033:1707825:1707899 [1] NCCL INFO Channel 00/0 : 17[46000] -> 16[7000] via P2P/IPC/read +gpua033:1707825:1707899 [1] NCCL INFO Channel 01/0 : 17[46000] -> 16[7000] via P2P/IPC/read +gpua033:1707825:1707899 [1] NCCL INFO Connected all trees +gpua033:1707825:1707899 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1707825:1707899 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1707825:1707899 [1] NCCL INFO comm 0x55819473e770 rank 17 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua033:1707826:1707826 [2] NCCL INFO cudaDriverVersion 12020 +gpua033:1707826:1707826 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.33<0> +gpua012:2787972:2788042 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2787972:2788042 [0] NCCL INFO comm 0x55633872e670 rank 8 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua016:645595:645595 [0] NCCL INFO cudaDriverVersion 12020 +gpua016:645595:645595 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:645595:645595 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:645595:645595 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:645595:645665 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:645595:645665 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:645595:645665 [0] NCCL INFO Using network AWS Libfabric +gpua016:645595:645665 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua016:645595:645665 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:645595:645665 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] 13/4/-1->12->28 +gpua016:645595:645665 [0] NCCL INFO Channel 00/0 : 11[c7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua016:645595:645665 [0] NCCL INFO Channel 01/0 : 11[c7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua049:4111893:4111893 [2] NCCL INFO cudaDriverVersion 12020 +gpua049:4111893:4111893 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua049:4111893:4111893 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:4111893:4111893 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:4111893:4111966 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:4111893:4111966 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:4111893:4111966 [2] NCCL INFO Using network AWS Libfabric +gpua049:4111893:4111966 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua049:4111893:4111966 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:4111893:4111966 [2] NCCL INFO Trees [0] 35/-1/-1->34->33 [1] 35/-1/-1->34->33 +gpua049:4111893:4111966 [2] NCCL INFO Channel 00/0 : 34[85000] -> 35[c7000] via P2P/IPC/read +gpua049:4111893:4111966 [2] NCCL INFO Channel 01/0 : 34[85000] -> 35[c7000] via P2P/IPC/read +gpua057:3948780:3948780 [2] NCCL INFO cudaDriverVersion 12020 +gpua057:3948780:3948780 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3948780:3948780 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3948780:3948780 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3948780:3948848 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3948780:3948848 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3948780:3948848 [2] NCCL INFO Using network AWS Libfabric +gpua057:3948780:3948848 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua057:3948780:3948848 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3948780:3948848 [2] NCCL INFO Trees [0] 47/-1/-1->46->45 [1] 47/-1/-1->46->45 +gpua057:3948780:3948848 [2] NCCL INFO Channel 00/0 : 46[85000] -> 47[c7000] via P2P/IPC/read +gpua057:3948780:3948848 [2] NCCL INFO Channel 01/0 : 46[85000] -> 47[c7000] via P2P/IPC/read +gpua080:3333273:3333273 [1] NCCL INFO cudaDriverVersion 12020 +gpua080:3333273:3333273 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua080:3333273:3333273 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3333273:3333273 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3333273:3333351 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3333273:3333351 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3333273:3333351 [1] NCCL INFO Using network AWS Libfabric +gpua080:3333273:3333351 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua080:3333273:3333351 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3333273:3333351 [1] NCCL INFO Trees [0] 54/-1/-1->53->52 [1] 54/56/-1->53->52 +gpua080:3333273:3333351 [1] NCCL INFO Channel 00/0 : 53[46000] -> 54[85000] via P2P/IPC/read +gpua080:3333273:3333351 [1] NCCL INFO Channel 01/0 : 53[46000] -> 54[85000] via P2P/IPC/read +gpua040:3922554:3922554 [1] NCCL INFO cudaDriverVersion 12020 +gpua040:3922554:3922554 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3922554:3922554 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3922554:3922554 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3922554:3922632 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua040:3922554:3922632 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3922554:3922632 [1] NCCL INFO Using network AWS Libfabric +gpua040:3922554:3922632 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua040:3922554:3922632 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3922554:3922632 [1] NCCL INFO Trees [0] 30/-1/-1->29->28 [1] 30/44/-1->29->28 +gpua040:3922554:3922632 [1] NCCL INFO Channel 00/0 : 29[46000] -> 30[85000] via P2P/IPC/read +gpua040:3922554:3922632 [1] NCCL INFO Channel 01/0 : 29[46000] -> 30[85000] via P2P/IPC/read +gpua085:4081111:4081179 [1] NCCL INFO Channel 01/0 : 57[46000] -> 56[7000] via P2P/IPC/read +gpua085:4081111:4081179 [1] NCCL INFO Connected all trees +gpua085:4081111:4081179 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:4081111:4081179 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:4081111:4081179 [1] NCCL INFO comm 0x55b7a137b570 rank 57 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua033:1707826:1707826 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua033:1707826:1707826 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua033:1707826:1707900 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua033:1707826:1707900 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua033:1707826:1707900 [2] NCCL INFO Using network AWS Libfabric +gpua033:1707826:1707900 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua033:1707826:1707900 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua033:1707826:1707900 [2] NCCL INFO Trees [0] 19/-1/-1->18->17 [1] 19/-1/-1->18->17 +gpua033:1707826:1707900 [2] NCCL INFO Channel 00/0 : 18[85000] -> 19[c7000] via P2P/IPC/read +gpua033:1707826:1707900 [2] NCCL INFO Channel 01/0 : 18[85000] -> 19[c7000] via P2P/IPC/read +gpua033:1707826:1707900 [2] NCCL INFO Connected all rings +gpua033:1707826:1707900 [2] NCCL INFO Channel 00/0 : 18[85000] -> 17[46000] via P2P/IPC/read +gpua016:645595:645665 [0] NCCL INFO Channel 00/0 : 12[7000] -> 13[46000] via P2P/IPC/read +gpua016:645595:645665 [0] NCCL INFO Channel 01/0 : 12[7000] -> 13[46000] via P2P/IPC/read +gpua016:645595:645665 [0] NCCL INFO Connected all rings +gpua016:645595:645665 [0] NCCL INFO Channel 00/0 : 8[7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua016:645595:645665 [0] NCCL INFO Channel 01/0 : 4[7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua016:645595:645665 [0] NCCL INFO Channel 01/0 : 12[7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua016:645595:645665 [0] NCCL INFO Channel 01/0 : 28[7000] -> 12[7000] [receive] via NET/AWS Libfabric/1 +gpua016:645595:645665 [0] NCCL INFO Channel 01/0 : 12[7000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua016:645595:645665 [0] NCCL INFO Channel 00/0 : 12[7000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua016:645595:645665 [0] NCCL INFO Connected all trees +gpua016:645595:645665 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:4111893:4111966 [2] NCCL INFO Connected all rings +gpua049:4111893:4111966 [2] NCCL INFO Channel 00/0 : 34[85000] -> 33[46000] via P2P/IPC/read +gpua049:4111893:4111966 [2] NCCL INFO Channel 01/0 : 34[85000] -> 33[46000] via P2P/IPC/read +gpua049:4111893:4111966 [2] NCCL INFO Connected all trees +gpua049:4111893:4111966 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:4111893:4111966 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:4111893:4111966 [2] NCCL INFO comm 0x560d97a60e10 rank 34 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua057:3948780:3948848 [2] NCCL INFO Connected all rings +gpua057:3948780:3948848 [2] NCCL INFO Channel 00/0 : 46[85000] -> 45[46000] via P2P/IPC/read +gpua057:3948780:3948848 [2] NCCL INFO Channel 01/0 : 46[85000] -> 45[46000] via P2P/IPC/read +gpua057:3948780:3948848 [2] NCCL INFO Connected all trees +gpua057:3948780:3948848 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3948780:3948848 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3948780:3948848 [2] NCCL INFO comm 0x560b20526a20 rank 46 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua080:3333273:3333351 [1] NCCL INFO Connected all rings +gpua080:3333273:3333351 [1] NCCL INFO Channel 01/0 : 53[46000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua080:3333273:3333351 [1] NCCL INFO Channel 01/0 : 56[7000] -> 53[46000] [receive] via NET/AWS Libfabric/1 +gpua080:3333273:3333351 [1] NCCL INFO Channel 00/0 : 53[46000] -> 52[7000] via P2P/IPC/read +gpua080:3333273:3333351 [1] NCCL INFO Channel 01/0 : 53[46000] -> 52[7000] via P2P/IPC/read +gpua080:3333273:3333351 [1] NCCL INFO Connected all trees +gpua080:3333273:3333351 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua080:3333273:3333351 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3333273:3333351 [1] NCCL INFO comm 0x55718f2428c0 rank 53 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua080:3333272:3333272 [0] NCCL INFO cudaDriverVersion 12020 +gpua080:3333272:3333272 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua040:3922554:3922632 [1] NCCL INFO Connected all rings +gpua040:3922554:3922632 [1] NCCL INFO Channel 01/0 : 29[46000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua040:3922554:3922632 [1] NCCL INFO Channel 01/0 : 44[7000] -> 29[46000] [receive] via NET/AWS Libfabric/1 +gpua040:3922554:3922632 [1] NCCL INFO Channel 00/0 : 29[46000] -> 28[7000] via P2P/IPC/read +gpua040:3922554:3922632 [1] NCCL INFO Channel 01/0 : 29[46000] -> 28[7000] via P2P/IPC/read +gpua040:3922554:3922632 [1] NCCL INFO Connected all trees +gpua040:3922554:3922632 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3922554:3922632 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3922554:3922632 [1] NCCL INFO comm 0x563cc461a850 rank 29 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua089:913543:913543 [3] NCCL INFO cudaDriverVersion 12020 +gpua089:913543:913543 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:913543:913543 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:913543:913543 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:913543:913609 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:913543:913609 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua089:913543:913609 [3] NCCL INFO Using network AWS Libfabric +gpua089:913543:913609 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua089:913543:913609 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:913543:913609 [3] NCCL INFO Trees [0] -1/-1/-1->63->62 [1] -1/-1/-1->63->62 +gpua089:913543:913609 [3] NCCL INFO Channel 00/0 : 63[c7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua089:913543:913609 [3] NCCL INFO Channel 01/0 : 63[c7000] -> 0[7000] [send] via NET/AWS Libfabric/1 +gpua085:4081112:4081112 [2] NCCL INFO cudaDriverVersion 12020 +gpua085:4081112:4081112 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:4081112:4081112 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:4081112:4081112 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:4081112:4081177 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua085:4081112:4081177 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:4081112:4081177 [2] NCCL INFO Using network AWS Libfabric +gpua085:4081112:4081177 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua085:4081112:4081177 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:4081112:4081177 [2] NCCL INFO Trees [0] 59/-1/-1->58->57 [1] 59/-1/-1->58->57 +gpua085:4081112:4081177 [2] NCCL INFO Channel 00/0 : 58[85000] -> 59[c7000] via P2P/IPC/read +gpua085:4081112:4081177 [2] NCCL INFO Channel 01/0 : 58[85000] -> 59[c7000] via P2P/IPC/read +gpua033:1707826:1707900 [2] NCCL INFO Channel 01/0 : 18[85000] -> 17[46000] via P2P/IPC/read +gpua033:1707826:1707900 [2] NCCL INFO Connected all trees +gpua033:1707826:1707900 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua033:1707826:1707900 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua033:1707826:1707900 [2] NCCL INFO comm 0x55ef33513e50 rank 18 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua016:645595:645665 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:645595:645665 [0] NCCL INFO comm 0x55c1b3ec9fb0 rank 12 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua049:4111894:4111894 [3] NCCL INFO cudaDriverVersion 12020 +gpua049:4111894:4111894 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.49<0> +gpua049:4111894:4111894 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua049:4111894:4111894 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua049:4111894:4111965 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua049:4111894:4111965 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua049:4111894:4111965 [3] NCCL INFO Using network AWS Libfabric +gpua049:4111894:4111965 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua049:4111894:4111965 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua049:4111894:4111965 [3] NCCL INFO Trees [0] -1/-1/-1->35->34 [1] -1/-1/-1->35->34 +gpua049:4111894:4111965 [3] NCCL INFO Channel 00/0 : 35[c7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua049:4111894:4111965 [3] NCCL INFO Channel 01/0 : 35[c7000] -> 36[7000] [send] via NET/AWS Libfabric/1 +gpua057:3948779:3948779 [1] NCCL INFO cudaDriverVersion 12020 +gpua057:3948779:3948779 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.57<0> +gpua057:3948779:3948779 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua057:3948779:3948779 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua057:3948779:3948849 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua057:3948779:3948849 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua057:3948779:3948849 [1] NCCL INFO Using network AWS Libfabric +gpua057:3948779:3948849 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua057:3948779:3948849 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua057:3948779:3948849 [1] NCCL INFO Trees [0] 46/-1/-1->45->44 [1] 46/52/-1->45->44 +gpua057:3948779:3948849 [1] NCCL INFO Channel 00/0 : 45[46000] -> 46[85000] via P2P/IPC/read +gpua057:3948779:3948849 [1] NCCL INFO Channel 01/0 : 45[46000] -> 46[85000] via P2P/IPC/read +gpua080:3333272:3333272 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3333272:3333272 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3333272:3333352 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3333272:3333352 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3333272:3333352 [0] NCCL INFO Using network AWS Libfabric +gpua080:3333272:3333352 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua080:3333272:3333352 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3333272:3333352 [0] NCCL INFO Trees [0] 53/-1/-1->52->57 [1] 53/48/-1->52->45 +gpua080:3333272:3333352 [0] NCCL INFO Channel 00/0 : 51[c7000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Channel 01/0 : 51[c7000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Channel 00/0 : 52[7000] -> 53[46000] via P2P/IPC/read +gpua055:183038:183038 [0] NCCL INFO cudaDriverVersion 12020 +gpua055:183038:183038 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0> +gpua055:183038:183038 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua055:183038:183038 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua055:183038:183107 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua055:183038:183107 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua055:183038:183107 [0] NCCL INFO Using network AWS Libfabric +gpua055:183038:183107 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua055:183038:183107 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua055:183038:183107 [0] NCCL INFO Trees [0] 41/44/-1->40->49 [1] 41/-1/-1->40->37 +gpua055:183038:183107 [0] NCCL INFO Channel 00/0 : 39[c7000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:183038:183107 [0] NCCL INFO Channel 01/0 : 39[c7000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua089:913543:913609 [3] NCCL INFO Connected all rings +gpua089:913543:913609 [3] NCCL INFO Channel 00/0 : 63[c7000] -> 62[85000] via P2P/IPC/read +gpua089:913543:913609 [3] NCCL INFO Channel 01/0 : 63[c7000] -> 62[85000] via P2P/IPC/read +gpua089:913543:913609 [3] NCCL INFO Connected all trees +gpua089:913543:913609 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:913543:913609 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua089:913543:913609 [3] NCCL INFO comm 0x563aab0370b0 rank 63 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua085:4081112:4081177 [2] NCCL INFO Connected all rings +gpua085:4081112:4081177 [2] NCCL INFO Channel 00/0 : 58[85000] -> 57[46000] via P2P/IPC/read +gpua085:4081112:4081177 [2] NCCL INFO Channel 01/0 : 58[85000] -> 57[46000] via P2P/IPC/read +gpua085:4081112:4081177 [2] NCCL INFO Connected all trees +gpua085:4081112:4081177 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:4081112:4081177 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:4081112:4081177 [2] NCCL INFO comm 0x5641dbc89900 rank 58 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua016:645596:645596 [1] NCCL INFO cudaDriverVersion 12020 +gpua016:645596:645596 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:645596:645596 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:645596:645596 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:645596:645664 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:645596:645664 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:645596:645664 [1] NCCL INFO Using network AWS Libfabric +gpua016:645596:645664 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua016:645596:645664 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:645596:645664 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/20/-1->13->12 +gpua016:645596:645664 [1] NCCL INFO Channel 00/0 : 13[46000] -> 14[85000] via P2P/IPC/read +gpua016:645596:645664 [1] NCCL INFO Channel 01/0 : 13[46000] -> 14[85000] via P2P/IPC/read +gpua016:645596:645664 [1] NCCL INFO Connected all rings +gpua049:4111894:4111965 [3] NCCL INFO Connected all rings +gpua049:4111894:4111965 [3] NCCL INFO Channel 00/0 : 35[c7000] -> 34[85000] via P2P/IPC/read +gpua049:4111894:4111965 [3] NCCL INFO Channel 01/0 : 35[c7000] -> 34[85000] via P2P/IPC/read +gpua049:4111894:4111965 [3] NCCL INFO Connected all trees +gpua049:4111894:4111965 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua049:4111894:4111965 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua049:4111894:4111965 [3] NCCL INFO comm 0x55ed5ae6a1a0 rank 35 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua057:3948779:3948849 [1] NCCL INFO Connected all rings +gpua057:3948779:3948849 [1] NCCL INFO Channel 01/0 : 45[46000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua057:3948779:3948849 [1] NCCL INFO Channel 01/0 : 52[7000] -> 45[46000] [receive] via NET/AWS Libfabric/1 +gpua057:3948779:3948849 [1] NCCL INFO Channel 00/0 : 45[46000] -> 44[7000] via P2P/IPC/read +gpua057:3948779:3948849 [1] NCCL INFO Channel 01/0 : 45[46000] -> 44[7000] via P2P/IPC/read +gpua057:3948779:3948849 [1] NCCL INFO Connected all trees +gpua057:3948779:3948849 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua057:3948779:3948849 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua057:3948779:3948849 [1] NCCL INFO comm 0x55bf1adf7750 rank 45 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua080:3333272:3333352 [0] NCCL INFO Channel 01/0 : 52[7000] -> 53[46000] via P2P/IPC/read +gpua080:3333272:3333352 [0] NCCL INFO Connected all rings +gpua080:3333272:3333352 [0] NCCL INFO Channel 01/0 : 48[7000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Channel 00/0 : 52[7000] -> 57[46000] [send] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Channel 01/0 : 45[46000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Channel 01/0 : 52[7000] -> 45[46000] [send] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Channel 00/0 : 57[46000] -> 52[7000] [receive] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Channel 01/0 : 52[7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua080:3333272:3333352 [0] NCCL INFO Connected all trees +gpua080:3333272:3333352 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua055:183038:183107 [0] NCCL INFO Channel 00/0 : 40[7000] -> 41[46000] via P2P/IPC/read +gpua055:183038:183107 [0] NCCL INFO Channel 01/0 : 40[7000] -> 41[46000] via P2P/IPC/read +gpua055:183038:183107 [0] NCCL INFO Connected all rings +gpua055:183038:183107 [0] NCCL INFO Channel 01/0 : 37[46000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:183038:183107 [0] NCCL INFO Channel 00/0 : 40[7000] -> 44[7000] [send] via NET/AWS Libfabric/1 +gpua055:183038:183107 [0] NCCL INFO Channel 00/0 : 40[7000] -> 49[46000] [send] via NET/AWS Libfabric/1 +gpua055:183038:183107 [0] NCCL INFO Channel 00/0 : 49[46000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:183038:183107 [0] NCCL INFO Channel 00/0 : 44[7000] -> 40[7000] [receive] via NET/AWS Libfabric/1 +gpua055:183038:183107 [0] NCCL INFO Channel 01/0 : 40[7000] -> 37[46000] [send] via NET/AWS Libfabric/1 +gpua055:183038:183107 [0] NCCL INFO Connected all trees +gpua055:183038:183107 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:913540:913540 [0] NCCL INFO cudaDriverVersion 12020 +gpua089:913540:913540 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.89<0> +gpua089:913540:913540 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua089:913540:913540 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua089:913540:913610 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua089:913540:913610 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua089:913540:913610 [0] NCCL INFO Using network AWS Libfabric +gpua089:913540:913610 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua089:913540:913610 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua089:913540:913610 [0] NCCL INFO Trees [0] 61/-1/-1->60->56 [1] 61/28/-1->60->-1 +gpua089:913540:913610 [0] NCCL INFO Channel 00/0 : 59[c7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua089:913540:913610 [0] NCCL INFO Channel 01/0 : 59[c7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua085:4081110:4081110 [0] NCCL INFO cudaDriverVersion 12020 +gpua085:4081110:4081110 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.85<0> +gpua085:4081110:4081110 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua085:4081110:4081110 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua085:4081110:4081180 [0] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua085:4081110:4081180 [0] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua085:4081110:4081180 [0] NCCL INFO Using network AWS Libfabric +gpua085:4081110:4081180 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000 +gpua085:4081110:4081180 [0] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua085:4081110:4081180 [0] NCCL INFO Trees [0] 57/60/-1->56->48 [1] 57/-1/-1->56->53 +gpua085:4081110:4081180 [0] NCCL INFO Channel 00/0 : 55[c7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua016:645596:645664 [1] NCCL INFO Channel 01/0 : 13[46000] -> 20[7000] [send] via NET/AWS Libfabric/1 +gpua016:645596:645664 [1] NCCL INFO Channel 01/0 : 20[7000] -> 13[46000] [receive] via NET/AWS Libfabric/1 +gpua016:645596:645664 [1] NCCL INFO Channel 00/0 : 13[46000] -> 12[7000] via P2P/IPC/read +gpua016:645596:645664 [1] NCCL INFO Channel 01/0 : 13[46000] -> 12[7000] via P2P/IPC/read +gpua016:645596:645664 [1] NCCL INFO Connected all trees +gpua016:645596:645664 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua016:645596:645664 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:645596:645664 [1] NCCL INFO comm 0x55987be07b40 rank 13 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua007:1643152:1643152 [1] NCCL INFO cudaDriverVersion 12020 +gpua007:1643152:1643152 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.7<0> +gpua007:1643152:1643152 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua007:1643152:1643152 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua007:1643152:1643213 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua007:1643152:1643213 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua007:1643152:1643213 [1] NCCL INFO Using network AWS Libfabric +gpua007:1643152:1643213 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua007:1643152:1643213 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua007:1643152:1643213 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/8/-1->5->4 +gpua007:1643152:1643213 [1] NCCL INFO Channel 00/0 : 5[46000] -> 6[85000] via P2P/IPC/read +gpua007:1643152:1643213 [1] NCCL INFO Channel 01/0 : 5[46000] -> 6[85000] via P2P/IPC/read +gpua080:3333272:3333352 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3333272:3333352 [0] NCCL INFO comm 0x55b709b34f60 rank 52 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua055:183038:183107 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua055:183038:183107 [0] NCCL INFO comm 0x5647a38f5ad0 rank 40 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua089:913540:913610 [0] NCCL INFO Channel 00/0 : 60[7000] -> 61[46000] via P2P/IPC/read +gpua089:913540:913610 [0] NCCL INFO Channel 01/0 : 60[7000] -> 61[46000] via P2P/IPC/read +gpua089:913540:913610 [0] NCCL INFO Connected all rings +gpua089:913540:913610 [0] NCCL INFO Channel 00/0 : 56[7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua089:913540:913610 [0] NCCL INFO Channel 01/0 : 28[7000] -> 60[7000] [receive] via NET/AWS Libfabric/1 +gpua089:913540:913610 [0] NCCL INFO Channel 01/0 : 60[7000] -> 28[7000] [send] via NET/AWS Libfabric/1 +gpua089:913540:913610 [0] NCCL INFO Channel 00/0 : 60[7000] -> 56[7000] [send] via NET/AWS Libfabric/1 +gpua089:913540:913610 [0] NCCL INFO Connected all trees +gpua089:913540:913610 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua089:913540:913610 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua089:913540:913610 [0] NCCL INFO comm 0x5571aef3a6b0 rank 60 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua085:4081110:4081180 [0] NCCL INFO Channel 01/0 : 55[c7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:4081110:4081180 [0] NCCL INFO Channel 00/0 : 56[7000] -> 57[46000] via P2P/IPC/read +gpua085:4081110:4081180 [0] NCCL INFO Channel 01/0 : 56[7000] -> 57[46000] via P2P/IPC/read +gpua085:4081110:4081180 [0] NCCL INFO Connected all rings +gpua085:4081110:4081180 [0] NCCL INFO Channel 01/0 : 53[46000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:4081110:4081180 [0] NCCL INFO Channel 00/0 : 56[7000] -> 60[7000] [send] via NET/AWS Libfabric/1 +gpua085:4081110:4081180 [0] NCCL INFO Channel 00/0 : 48[7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:4081110:4081180 [0] NCCL INFO Channel 00/0 : 56[7000] -> 48[7000] [send] via NET/AWS Libfabric/1 +gpua085:4081110:4081180 [0] NCCL INFO Channel 00/0 : 60[7000] -> 56[7000] [receive] via NET/AWS Libfabric/1 +gpua085:4081110:4081180 [0] NCCL INFO Channel 01/0 : 56[7000] -> 53[46000] [send] via NET/AWS Libfabric/1 +gpua016:645597:645597 [2] NCCL INFO cudaDriverVersion 12020 +gpua016:645597:645597 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0> +gpua016:645597:645597 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua016:645597:645597 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua016:645597:645663 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua016:645597:645663 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua016:645597:645663 [2] NCCL INFO Using network AWS Libfabric +gpua016:645597:645663 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua016:645597:645663 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua016:645597:645663 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 +gpua016:645597:645663 [2] NCCL INFO Channel 00/0 : 14[85000] -> 15[c7000] via P2P/IPC/read +gpua016:645597:645663 [2] NCCL INFO Channel 01/0 : 14[85000] -> 15[c7000] via P2P/IPC/read +gpua016:645597:645663 [2] NCCL INFO Connected all rings +gpua007:1643152:1643213 [1] NCCL INFO Connected all rings +gpua007:1643152:1643213 [1] NCCL INFO Channel 01/0 : 5[46000] -> 8[7000] [send] via NET/AWS Libfabric/1 +gpua007:1643152:1643213 [1] NCCL INFO Channel 01/0 : 8[7000] -> 5[46000] [receive] via NET/AWS Libfabric/1 +gpua007:1643152:1643213 [1] NCCL INFO Channel 00/0 : 5[46000] -> 4[7000] via P2P/IPC/read +gpua007:1643152:1643213 [1] NCCL INFO Channel 01/0 : 5[46000] -> 4[7000] via P2P/IPC/read +gpua007:1643152:1643213 [1] NCCL INFO Connected all trees +gpua007:1643152:1643213 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua007:1643152:1643213 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua007:1643152:1643213 [1] NCCL INFO comm 0x562671ed6010 rank 5 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua085:4081110:4081180 [0] NCCL INFO Connected all trees +gpua085:4081110:4081180 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua085:4081110:4081180 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua085:4081110:4081180 [0] NCCL INFO comm 0x557e0d14bb10 rank 56 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE +gpua016:645597:645663 [2] NCCL INFO Channel 00/0 : 14[85000] -> 13[46000] via P2P/IPC/read +gpua016:645597:645663 [2] NCCL INFO Channel 01/0 : 14[85000] -> 13[46000] via P2P/IPC/read +gpua016:645597:645663 [2] NCCL INFO Connected all trees +gpua016:645597:645663 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua016:645597:645663 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua016:645597:645663 [2] NCCL INFO comm 0x556e6f071670 rank 14 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua038:242422:242422 [1] NCCL INFO cudaDriverVersion 12020 +gpua038:242422:242422 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:242422:242422 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:242422:242422 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:242422:242507 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:242422:242507 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:242422:242507 [1] NCCL INFO Using network AWS Libfabric +gpua038:242422:242507 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua038:242422:242507 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:242422:242507 [1] NCCL INFO Trees [0] 22/-1/-1->21->20 [1] 22/24/-1->21->20 +gpua038:242422:242507 [1] NCCL INFO Channel 00/0 : 21[46000] -> 22[85000] via P2P/IPC/read +gpua038:242422:242507 [1] NCCL INFO Channel 01/0 : 21[46000] -> 22[85000] via P2P/IPC/read +gpua038:242422:242507 [1] NCCL INFO Connected all rings +gpua079:3757852:3757852 [3] NCCL INFO cudaDriverVersion 12020 +gpua079:3757852:3757852 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3757852:3757852 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3757852:3757852 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3757852:3757908 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3757852:3757908 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3757852:3757908 [3] NCCL INFO Using network AWS Libfabric +gpua079:3757852:3757908 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua079:3757852:3757908 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3757852:3757908 [3] NCCL INFO Trees [0] -1/-1/-1->51->50 [1] -1/-1/-1->51->50 +gpua079:3757852:3757908 [3] NCCL INFO Channel 00/0 : 51[c7000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757852:3757908 [3] NCCL INFO Channel 01/0 : 51[c7000] -> 52[7000] [send] via NET/AWS Libfabric/1 +gpua038:242422:242507 [1] NCCL INFO Channel 01/0 : 21[46000] -> 24[7000] [send] via NET/AWS Libfabric/1 +gpua038:242422:242507 [1] NCCL INFO Channel 01/0 : 24[7000] -> 21[46000] [receive] via NET/AWS Libfabric/1 +gpua038:242422:242507 [1] NCCL INFO Channel 00/0 : 21[46000] -> 20[7000] via P2P/IPC/read +gpua038:242422:242507 [1] NCCL INFO Channel 01/0 : 21[46000] -> 20[7000] via P2P/IPC/read +gpua038:242422:242507 [1] NCCL INFO Connected all trees +gpua038:242422:242507 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua038:242422:242507 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:242422:242507 [1] NCCL INFO comm 0x55cc2f9155a0 rank 21 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua079:3757852:3757908 [3] NCCL INFO Connected all rings +gpua079:3757852:3757908 [3] NCCL INFO Channel 00/0 : 51[c7000] -> 50[85000] via P2P/IPC/read +gpua079:3757852:3757908 [3] NCCL INFO Channel 01/0 : 51[c7000] -> 50[85000] via P2P/IPC/read +gpua079:3757852:3757908 [3] NCCL INFO Connected all trees +gpua079:3757852:3757908 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3757852:3757908 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3757852:3757908 [3] NCCL INFO comm 0x56558f477920 rank 51 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua038:242423:242423 [2] NCCL INFO cudaDriverVersion 12020 +gpua038:242423:242423 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.38<0> +gpua038:242423:242423 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua038:242423:242423 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua038:242423:242506 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua038:242423:242506 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua038:242423:242506 [2] NCCL INFO Using network AWS Libfabric +gpua038:242423:242506 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua038:242423:242506 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua038:242423:242506 [2] NCCL INFO Trees [0] 23/-1/-1->22->21 [1] 23/-1/-1->22->21 +gpua038:242423:242506 [2] NCCL INFO Channel 00/0 : 22[85000] -> 23[c7000] via P2P/IPC/read +gpua038:242423:242506 [2] NCCL INFO Channel 01/0 : 22[85000] -> 23[c7000] via P2P/IPC/read +gpua038:242423:242506 [2] NCCL INFO Connected all rings +gpua054:483906:483906 [3] NCCL INFO cudaDriverVersion 12020 +gpua054:483906:483906 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.54<0> +gpua054:483906:483906 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua054:483906:483906 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua054:483906:483978 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua054:483906:483978 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua054:483906:483978 [3] NCCL INFO Using network AWS Libfabric +gpua054:483906:483978 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua054:483906:483978 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua054:483906:483978 [3] NCCL INFO Trees [0] -1/-1/-1->39->38 [1] -1/-1/-1->39->38 +gpua054:483906:483978 [3] NCCL INFO Channel 00/0 : 39[c7000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua054:483906:483978 [3] NCCL INFO Channel 01/0 : 39[c7000] -> 40[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757851:3757851 [2] NCCL INFO cudaDriverVersion 12020 +gpua079:3757851:3757851 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0> +gpua079:3757851:3757851 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua079:3757851:3757851 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua079:3757851:3757905 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua079:3757851:3757905 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua079:3757851:3757905 [2] NCCL INFO Using network AWS Libfabric +gpua079:3757851:3757905 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua079:3757851:3757905 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua079:3757851:3757905 [2] NCCL INFO Trees [0] 51/-1/-1->50->49 [1] 51/-1/-1->50->49 +gpua079:3757851:3757905 [2] NCCL INFO Channel 00/0 : 50[85000] -> 51[c7000] via P2P/IPC/read +gpua079:3757851:3757905 [2] NCCL INFO Channel 01/0 : 50[85000] -> 51[c7000] via P2P/IPC/read +gpua038:242423:242506 [2] NCCL INFO Channel 00/0 : 22[85000] -> 21[46000] via P2P/IPC/read +gpua038:242423:242506 [2] NCCL INFO Channel 01/0 : 22[85000] -> 21[46000] via P2P/IPC/read +gpua038:242423:242506 [2] NCCL INFO Connected all trees +gpua038:242423:242506 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua038:242423:242506 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua038:242423:242506 [2] NCCL INFO comm 0x55718af3a950 rank 22 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua054:483906:483978 [3] NCCL INFO Connected all rings +gpua054:483906:483978 [3] NCCL INFO Channel 00/0 : 39[c7000] -> 38[85000] via P2P/IPC/read +gpua054:483906:483978 [3] NCCL INFO Channel 01/0 : 39[c7000] -> 38[85000] via P2P/IPC/read +gpua054:483906:483978 [3] NCCL INFO Connected all trees +gpua054:483906:483978 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua054:483906:483978 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua054:483906:483978 [3] NCCL INFO comm 0x55e3f4242c40 rank 39 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua040:3922556:3922556 [3] NCCL INFO cudaDriverVersion 12020 +gpua040:3922556:3922556 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.40<0> +gpua040:3922556:3922556 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua040:3922556:3922556 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua040:3922556:3922634 [3] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua040:3922556:3922634 [3] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua040:3922556:3922634 [3] NCCL INFO Using network AWS Libfabric +gpua040:3922556:3922634 [3] NCCL INFO Setting affinity for GPU 3 to ffff +gpua040:3922556:3922634 [3] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua040:3922556:3922634 [3] NCCL INFO Trees [0] -1/-1/-1->31->30 [1] -1/-1/-1->31->30 +gpua040:3922556:3922634 [3] NCCL INFO Channel 00/0 : 31[c7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua040:3922556:3922634 [3] NCCL INFO Channel 01/0 : 31[c7000] -> 32[7000] [send] via NET/AWS Libfabric/1 +gpua079:3757851:3757905 [2] NCCL INFO Connected all rings +gpua079:3757851:3757905 [2] NCCL INFO Channel 00/0 : 50[85000] -> 49[46000] via P2P/IPC/read +gpua079:3757851:3757905 [2] NCCL INFO Channel 01/0 : 50[85000] -> 49[46000] via P2P/IPC/read +gpua079:3757851:3757905 [2] NCCL INFO Connected all trees +gpua079:3757851:3757905 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua079:3757851:3757905 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua079:3757851:3757905 [2] NCCL INFO comm 0x564127980990 rank 50 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +gpua040:3922556:3922634 [3] NCCL INFO Connected all rings +gpua040:3922556:3922634 [3] NCCL INFO Channel 00/0 : 31[c7000] -> 30[85000] via P2P/IPC/read +gpua040:3922556:3922634 [3] NCCL INFO Channel 01/0 : 31[c7000] -> 30[85000] via P2P/IPC/read +gpua040:3922556:3922634 [3] NCCL INFO Connected all trees +gpua040:3922556:3922634 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua040:3922556:3922634 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua040:3922556:3922634 [3] NCCL INFO comm 0x55d17e122530 rank 31 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE +gpua012:2787973:2787973 [1] NCCL INFO cudaDriverVersion 12020 +gpua012:2787973:2787973 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.12<0> +gpua012:2787973:2787973 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua012:2787973:2787973 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua012:2787973:2788043 [1] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua012:2787973:2788043 [1] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua012:2787973:2788043 [1] NCCL INFO Using network AWS Libfabric +gpua012:2787973:2788043 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000 +gpua012:2787973:2788043 [1] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua012:2787973:2788043 [1] NCCL INFO Trees [0] 10/4/-1->9->8 [1] 10/-1/-1->9->8 +gpua012:2787973:2788043 [1] NCCL INFO Channel 00/0 : 9[46000] -> 10[85000] via P2P/IPC/read +gpua012:2787973:2788043 [1] NCCL INFO Channel 01/0 : 9[46000] -> 10[85000] via P2P/IPC/read +gpua012:2787973:2788043 [1] NCCL INFO Connected all rings +gpua012:2787973:2788043 [1] NCCL INFO Channel 00/0 : 4[7000] -> 9[46000] [receive] via NET/AWS Libfabric/1 +gpua012:2787973:2788043 [1] NCCL INFO Channel 00/0 : 9[46000] -> 4[7000] [send] via NET/AWS Libfabric/1 +gpua012:2787973:2788043 [1] NCCL INFO Channel 00/0 : 9[46000] -> 8[7000] via P2P/IPC/read +gpua012:2787973:2788043 [1] NCCL INFO Channel 01/0 : 9[46000] -> 8[7000] via P2P/IPC/read +gpua012:2787973:2788043 [1] NCCL INFO Connected all trees +gpua012:2787973:2788043 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua012:2787973:2788043 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua012:2787973:2788043 [1] NCCL INFO comm 0x561b200a8400 rank 9 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE +gpua080:3333274:3333274 [2] NCCL INFO cudaDriverVersion 12020 +gpua080:3333274:3333274 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0> +gpua080:3333274:3333274 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v6 symbol. +gpua080:3333274:3333274 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin symbol (v4 or v5). +gpua080:3333274:3333353 [2] NCCL INFO NET/OFI Using aws-ofi-nccl 1.6.0 +gpua080:3333274:3333353 [2] NCCL INFO NET/OFI Selected Provider is cxi (found 2 nics) +gpua080:3333274:3333353 [2] NCCL INFO Using network AWS Libfabric +gpua080:3333274:3333353 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000 +gpua080:3333274:3333353 [2] NCCL INFO NCCL_CROSS_NIC set by environment to 1. +gpua080:3333274:3333353 [2] NCCL INFO Trees [0] 55/-1/-1->54->53 [1] 55/-1/-1->54->53 +gpua080:3333274:3333353 [2] NCCL INFO Channel 00/0 : 54[85000] -> 55[c7000] via P2P/IPC/read +gpua080:3333274:3333353 [2] NCCL INFO Channel 01/0 : 54[85000] -> 55[c7000] via P2P/IPC/read +gpua080:3333274:3333353 [2] NCCL INFO Connected all rings +gpua080:3333274:3333353 [2] NCCL INFO Channel 00/0 : 54[85000] -> 53[46000] via P2P/IPC/read +gpua080:3333274:3333353 [2] NCCL INFO Channel 01/0 : 54[85000] -> 53[46000] via P2P/IPC/read +gpua080:3333274:3333353 [2] NCCL INFO Connected all trees +gpua080:3333274:3333353 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512 +gpua080:3333274:3333353 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +gpua080:3333274:3333353 [2] NCCL INFO comm 0x55c65287b820 rank 54 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE +[gpua006:0/64] 2024-02-11 20:27:38,123 (distributed:1027) INFO: Reducer buckets have been rebuilt in this iteration. +[gpua006:0/64] 2024-02-11 20:29:13,004 (trainer:756) INFO: 32epoch:train:1-100batch: iter_time=5.203, forward_time=0.244, loss_ctc=71.557, loss_interctc_layer6=78.572, loss_interctc_layer12=64.794, loss_interctc_layer15=59.313, loss_interctc_layer21=74.332, loss=69.714, backward_time=0.224, grad_norm=186.838, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.185e-05, train_time=6.284 +[gpua006:0/64] 2024-02-11 20:31:01,936 (trainer:756) INFO: 32epoch:train:101-200batch: iter_time=9.700e-05, forward_time=0.143, loss_ctc=75.751, loss_interctc_layer6=78.644, loss_interctc_layer12=65.496, loss_interctc_layer15=60.167, loss_interctc_layer21=78.427, loss=71.697, backward_time=0.208, grad_norm=71.980, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.184e-05, train_time=1.089 +[gpua006:0/64] 2024-02-11 20:33:15,939 (trainer:756) INFO: 32epoch:train:201-300batch: iter_time=1.069e-04, forward_time=0.141, loss_ctc=74.156, loss_interctc_layer6=77.294, loss_interctc_layer12=64.325, loss_interctc_layer15=59.239, loss_interctc_layer21=76.815, loss=70.366, backward_time=0.206, grad_norm=72.388, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.183e-05, train_time=1.340 +[gpua006:0/64] 2024-02-11 20:35:28,458 (trainer:756) INFO: 32epoch:train:301-400batch: iter_time=9.188e-05, forward_time=0.253, loss_ctc=67.280, loss_interctc_layer6=76.476, loss_interctc_layer12=63.540, loss_interctc_layer15=58.216, loss_interctc_layer21=69.645, loss=67.031, backward_time=0.252, grad_norm=58.269, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.182e-05, train_time=1.325 +[gpua006:0/64] 2024-02-11 20:37:27,094 (trainer:756) INFO: 32epoch:train:401-500batch: iter_time=8.652e-05, forward_time=0.141, loss_ctc=80.076, loss_interctc_layer6=88.091, loss_interctc_layer12=74.063, loss_interctc_layer15=68.077, loss_interctc_layer21=82.894, loss=78.640, backward_time=0.206, grad_norm=72.359, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.182e-05, train_time=1.186 +[gpua006:0/64] 2024-02-11 20:40:33,218 (trainer:756) INFO: 32epoch:train:501-600batch: iter_time=8.761e-05, forward_time=0.141, loss_ctc=76.668, loss_interctc_layer6=83.938, loss_interctc_layer12=69.799, loss_interctc_layer15=64.196, loss_interctc_layer21=79.586, loss=74.837, backward_time=0.204, grad_norm=81.125, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.181e-05, train_time=1.861 +[gpua006:0/64] 2024-02-11 20:43:11,034 (trainer:756) INFO: 32epoch:train:601-700batch: iter_time=8.495e-05, forward_time=0.143, loss_ctc=81.328, loss_interctc_layer6=89.099, loss_interctc_layer12=74.047, loss_interctc_layer15=67.981, loss_interctc_layer21=84.265, loss=79.344, backward_time=0.204, grad_norm=91.524, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.180e-05, train_time=1.578 +[gpua006:0/64] 2024-02-11 20:45:22,698 (trainer:756) INFO: 32epoch:train:701-800batch: iter_time=9.734e-05, forward_time=0.251, loss_ctc=73.746, loss_interctc_layer6=81.376, loss_interctc_layer12=67.385, loss_interctc_layer15=61.583, loss_interctc_layer21=76.350, loss=72.088, backward_time=0.238, grad_norm=72.666, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.179e-05, train_time=1.316 +[gpua006:0/64] 2024-02-11 20:47:26,358 (trainer:756) INFO: 32epoch:train:801-900batch: iter_time=9.847e-05, forward_time=0.141, loss_ctc=87.440, loss_interctc_layer6=85.594, loss_interctc_layer12=71.634, loss_interctc_layer15=65.876, loss_interctc_layer21=90.572, loss=80.223, backward_time=0.206, grad_norm=81.254, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.178e-05, train_time=1.237 +[gpua006:0/64] 2024-02-11 20:50:11,429 (trainer:756) INFO: 32epoch:train:901-1000batch: iter_time=1.061e-04, forward_time=0.141, loss_ctc=81.595, loss_interctc_layer6=94.914, loss_interctc_layer12=79.184, loss_interctc_layer15=72.722, loss_interctc_layer21=84.429, loss=82.569, backward_time=0.204, grad_norm=86.652, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.178e-05, train_time=1.650 +[gpua006:0/64] 2024-02-11 20:52:19,158 (trainer:756) INFO: 32epoch:train:1001-1100batch: iter_time=1.048e-04, forward_time=0.141, loss_ctc=77.916, loss_interctc_layer6=89.366, loss_interctc_layer12=73.994, loss_interctc_layer15=67.944, loss_interctc_layer21=80.595, loss=77.963, backward_time=0.205, grad_norm=84.965, clip=100.000, loss_scale=2.819e+31, optim_step_time=0.136, optim0_lr0=7.177e-05, train_time=1.277 +[gpua006:0/64] 2024-02-11 20:54:56,739 (trainer:756) INFO: 32epoch:train:1101-1200batch: iter_time=8.645e-05, forward_time=0.260, loss_ctc=94.357, loss_interctc_layer6=103.294, loss_interctc_layer12=87.611, loss_interctc_layer15=81.275, loss_interctc_layer21=97.333, loss=92.774, backward_time=0.245, grad_norm=105.857, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=7.176e-05, train_time=1.575 +[gpua006:0/64] 2024-02-11 20:56:32,496 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-11 20:56:51,991 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 20:56:55,510 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 20:56:55,510 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-11 20:56:55,514 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-11 21:05:32,394 (trainer:756) INFO: 32epoch:train:1201-1300batch: iter_time=3.125, forward_time=0.142, loss_ctc=67.303, loss_interctc_layer6=77.964, loss_interctc_layer12=64.450, loss_interctc_layer15=58.874, loss_interctc_layer21=69.550, loss=67.628, backward_time=0.207, grad_norm=65.564, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.175e-05, train_time=6.356 +[gpua006:0/64] 2024-02-11 21:07:22,336 (trainer:756) INFO: 32epoch:train:1301-1400batch: iter_time=8.877e-05, forward_time=0.146, loss_ctc=70.569, loss_interctc_layer6=81.747, loss_interctc_layer12=68.265, loss_interctc_layer15=62.809, loss_interctc_layer21=73.075, loss=71.293, backward_time=0.208, grad_norm=63.477, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.175e-05, train_time=1.099 +[gpua006:0/64] 2024-02-11 21:09:26,721 (trainer:756) INFO: 32epoch:train:1401-1500batch: iter_time=8.271e-05, forward_time=0.143, loss_ctc=81.290, loss_interctc_layer6=80.414, loss_interctc_layer12=66.888, loss_interctc_layer15=61.289, loss_interctc_layer21=84.426, loss=74.861, backward_time=0.208, grad_norm=82.796, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.174e-05, train_time=1.244 +[gpua006:0/64] 2024-02-11 21:12:13,720 (trainer:756) INFO: 32epoch:train:1501-1600batch: iter_time=8.785e-05, forward_time=0.143, loss_ctc=63.771, loss_interctc_layer6=65.588, loss_interctc_layer12=54.167, loss_interctc_layer15=49.415, loss_interctc_layer21=66.206, loss=59.829, backward_time=0.208, grad_norm=67.971, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.173e-05, train_time=1.670 +[gpua006:0/64] 2024-02-11 21:14:39,048 (trainer:756) INFO: 32epoch:train:1601-1700batch: iter_time=9.051e-05, forward_time=0.151, loss_ctc=66.747, loss_interctc_layer6=83.791, loss_interctc_layer12=69.762, loss_interctc_layer15=64.010, loss_interctc_layer21=68.844, loss=70.631, backward_time=0.210, grad_norm=64.996, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.172e-05, train_time=1.453 +[gpua006:0/64] 2024-02-11 21:17:03,072 (trainer:756) INFO: 32epoch:train:1701-1800batch: iter_time=1.056e-04, forward_time=0.249, loss_ctc=82.662, loss_interctc_layer6=86.532, loss_interctc_layer12=72.220, loss_interctc_layer15=66.394, loss_interctc_layer21=85.780, loss=78.718, backward_time=0.246, grad_norm=94.121, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=7.172e-05, train_time=1.440 +[gpua006:0/64] 2024-02-11 21:18:51,874 (trainer:756) INFO: 32epoch:train:1801-1900batch: iter_time=8.503e-05, forward_time=0.147, loss_ctc=77.475, loss_interctc_layer6=83.614, loss_interctc_layer12=69.706, loss_interctc_layer15=63.945, loss_interctc_layer21=80.309, loss=75.010, backward_time=0.210, grad_norm=70.786, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.171e-05, train_time=1.088 +[gpua006:0/64] 2024-02-11 21:20:40,202 (trainer:756) INFO: 32epoch:train:1901-2000batch: iter_time=8.537e-05, forward_time=0.143, loss_ctc=81.380, loss_interctc_layer6=86.584, loss_interctc_layer12=71.575, loss_interctc_layer15=65.515, loss_interctc_layer21=84.073, loss=77.825, backward_time=0.209, grad_norm=67.732, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.170e-05, train_time=1.083 +[gpua006:0/64] 2024-02-11 21:23:31,082 (trainer:756) INFO: 32epoch:train:2001-2100batch: iter_time=9.515e-05, forward_time=0.143, loss_ctc=74.294, loss_interctc_layer6=83.073, loss_interctc_layer12=68.982, loss_interctc_layer15=63.209, loss_interctc_layer21=76.908, loss=73.293, backward_time=0.208, grad_norm=62.505, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.169e-05, train_time=1.709 +[gpua006:0/64] 2024-02-11 21:23:51,733 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-11 21:25:34,692 (trainer:756) INFO: 32epoch:train:2101-2200batch: iter_time=8.944e-05, forward_time=0.143, loss_ctc=83.282, loss_interctc_layer6=86.036, loss_interctc_layer12=71.409, loss_interctc_layer15=65.587, loss_interctc_layer21=86.075, loss=78.478, backward_time=0.208, grad_norm=89.204, clip=100.000, loss_scale=2.438e+31, optim_step_time=0.136, optim0_lr0=7.168e-05, train_time=1.236 +[gpua006:0/64] 2024-02-11 21:27:13,237 (trainer:756) INFO: 32epoch:train:2201-2300batch: iter_time=8.989e-05, forward_time=0.144, loss_ctc=85.317, loss_interctc_layer6=96.770, loss_interctc_layer12=80.490, loss_interctc_layer15=73.993, loss_interctc_layer21=88.059, loss=84.926, backward_time=0.209, grad_norm=72.895, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.168e-05, train_time=0.985 +[gpua006:0/64] 2024-02-11 21:30:00,138 (trainer:756) INFO: 32epoch:train:2301-2400batch: iter_time=8.477e-05, forward_time=0.143, loss_ctc=70.381, loss_interctc_layer6=83.457, loss_interctc_layer12=68.911, loss_interctc_layer15=63.070, loss_interctc_layer21=72.960, loss=71.756, backward_time=0.207, grad_norm=83.560, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.167e-05, train_time=1.669 +[gpua006:0/64] 2024-02-11 21:32:00,147 (trainer:756) INFO: 32epoch:train:2401-2500batch: iter_time=8.416e-05, forward_time=0.147, loss_ctc=90.316, loss_interctc_layer6=102.185, loss_interctc_layer12=86.347, loss_interctc_layer15=79.977, loss_interctc_layer21=93.072, loss=90.379, backward_time=0.208, grad_norm=85.424, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.166e-05, train_time=1.200 +[gpua006:0/64] 2024-02-11 21:32:20,177 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-11 21:32:39,043 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 21:32:42,527 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 21:32:42,527 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-11 21:32:42,531 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-11 21:43:15,628 (trainer:756) INFO: 32epoch:train:2501-2600batch: iter_time=3.115, forward_time=0.230, loss_ctc=73.276, loss_interctc_layer6=76.936, loss_interctc_layer12=63.331, loss_interctc_layer15=57.800, loss_interctc_layer21=76.137, loss=69.496, backward_time=0.228, grad_norm=70.132, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.165e-05, train_time=6.754 +[gpua006:0/64] 2024-02-11 21:45:02,279 (trainer:756) INFO: 32epoch:train:2601-2700batch: iter_time=8.404e-05, forward_time=0.145, loss_ctc=78.237, loss_interctc_layer6=77.606, loss_interctc_layer12=64.366, loss_interctc_layer15=58.872, loss_interctc_layer21=81.248, loss=72.066, backward_time=0.208, grad_norm=81.609, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.165e-05, train_time=1.067 +[gpua006:0/64] 2024-02-11 21:47:04,027 (trainer:756) INFO: 32epoch:train:2701-2800batch: iter_time=8.634e-05, forward_time=0.143, loss_ctc=76.635, loss_interctc_layer6=76.813, loss_interctc_layer12=63.875, loss_interctc_layer15=58.688, loss_interctc_layer21=79.528, loss=71.108, backward_time=0.208, grad_norm=65.909, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.164e-05, train_time=1.217 +[gpua006:0/64] 2024-02-11 21:49:10,225 (trainer:756) INFO: 32epoch:train:2801-2900batch: iter_time=8.250e-05, forward_time=0.144, loss_ctc=71.035, loss_interctc_layer6=75.069, loss_interctc_layer12=62.149, loss_interctc_layer15=56.821, loss_interctc_layer21=73.491, loss=67.713, backward_time=0.210, grad_norm=64.921, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.163e-05, train_time=1.262 +[gpua006:0/64] 2024-02-11 21:51:10,684 (trainer:756) INFO: 32epoch:train:2901-3000batch: iter_time=8.563e-05, forward_time=0.144, loss_ctc=82.246, loss_interctc_layer6=86.191, loss_interctc_layer12=72.071, loss_interctc_layer15=66.350, loss_interctc_layer21=84.969, loss=78.365, backward_time=0.208, grad_norm=73.134, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.162e-05, train_time=1.204 +[gpua006:0/64] 2024-02-11 21:53:02,438 (trainer:756) INFO: 32epoch:train:3001-3100batch: iter_time=8.282e-05, forward_time=0.143, loss_ctc=82.015, loss_interctc_layer6=82.712, loss_interctc_layer12=68.917, loss_interctc_layer15=63.272, loss_interctc_layer21=85.150, loss=76.413, backward_time=0.208, grad_norm=65.045, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.162e-05, train_time=1.117 +[gpua006:0/64] 2024-02-11 21:54:58,289 (trainer:756) INFO: 32epoch:train:3101-3200batch: iter_time=8.313e-05, forward_time=0.147, loss_ctc=87.554, loss_interctc_layer6=88.934, loss_interctc_layer12=73.791, loss_interctc_layer15=67.804, loss_interctc_layer21=90.572, loss=81.731, backward_time=0.209, grad_norm=103.604, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.161e-05, train_time=1.158 +[gpua006:0/64] 2024-02-11 21:57:18,042 (trainer:756) INFO: 32epoch:train:3201-3300batch: iter_time=1.003e-04, forward_time=0.176, loss_ctc=73.130, loss_interctc_layer6=80.817, loss_interctc_layer12=66.630, loss_interctc_layer15=60.769, loss_interctc_layer21=75.647, loss=71.399, backward_time=0.231, grad_norm=59.741, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.160e-05, train_time=1.397 +[gpua006:0/64] 2024-02-11 21:59:49,055 (trainer:756) INFO: 32epoch:train:3301-3400batch: iter_time=8.415e-05, forward_time=0.159, loss_ctc=89.893, loss_interctc_layer6=85.135, loss_interctc_layer12=71.075, loss_interctc_layer15=65.416, loss_interctc_layer21=93.100, loss=80.924, backward_time=0.214, grad_norm=76.355, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.159e-05, train_time=1.510 +[gpua006:0/64] 2024-02-11 22:01:39,967 (trainer:756) INFO: 32epoch:train:3401-3500batch: iter_time=8.446e-05, forward_time=0.144, loss_ctc=87.097, loss_interctc_layer6=94.646, loss_interctc_layer12=78.483, loss_interctc_layer15=71.958, loss_interctc_layer21=90.051, loss=84.447, backward_time=0.208, grad_norm=75.674, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.158e-05, train_time=1.110 +[gpua006:0/64] 2024-02-11 22:03:32,710 (trainer:756) INFO: 32epoch:train:3501-3600batch: iter_time=8.685e-05, forward_time=0.144, loss_ctc=80.299, loss_interctc_layer6=88.372, loss_interctc_layer12=72.798, loss_interctc_layer15=66.604, loss_interctc_layer21=83.042, loss=78.223, backward_time=0.209, grad_norm=89.741, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.158e-05, train_time=1.127 +[gpua006:0/64] 2024-02-11 22:05:17,229 (trainer:756) INFO: 32epoch:train:3601-3700batch: iter_time=8.973e-05, forward_time=0.146, loss_ctc=95.655, loss_interctc_layer6=100.540, loss_interctc_layer12=84.342, loss_interctc_layer15=78.198, loss_interctc_layer21=98.814, loss=91.510, backward_time=0.209, grad_norm=88.686, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.157e-05, train_time=1.045 +[gpua006:0/64] 2024-02-11 22:06:31,158 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-11 22:06:50,344 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 22:06:53,799 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 22:06:53,800 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-11 22:06:53,803 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-11 22:11:48,897 (trainer:756) INFO: 32epoch:train:3701-3800batch: iter_time=2.647, forward_time=0.146, loss_ctc=68.264, loss_interctc_layer6=77.257, loss_interctc_layer12=63.543, loss_interctc_layer15=58.075, loss_interctc_layer21=70.739, loss=67.576, backward_time=0.209, grad_norm=63.779, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.156e-05, train_time=3.916 +[gpua006:0/64] 2024-02-11 22:13:26,869 (trainer:756) INFO: 32epoch:train:3801-3900batch: iter_time=1.042e-04, forward_time=0.144, loss_ctc=70.192, loss_interctc_layer6=81.672, loss_interctc_layer12=67.989, loss_interctc_layer15=62.506, loss_interctc_layer21=72.884, loss=71.048, backward_time=0.211, grad_norm=66.182, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.155e-05, train_time=0.979 +[gpua006:0/64] 2024-02-11 22:15:14,241 (trainer:756) INFO: 32epoch:train:3901-4000batch: iter_time=9.895e-05, forward_time=0.144, loss_ctc=80.538, loss_interctc_layer6=79.578, loss_interctc_layer12=66.019, loss_interctc_layer15=60.575, loss_interctc_layer21=83.669, loss=74.076, backward_time=0.210, grad_norm=85.010, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.155e-05, train_time=1.073 +[gpua006:0/64] 2024-02-11 22:17:28,495 (trainer:756) INFO: 32epoch:train:4001-4100batch: iter_time=9.255e-05, forward_time=0.232, loss_ctc=63.731, loss_interctc_layer6=65.788, loss_interctc_layer12=54.136, loss_interctc_layer15=49.375, loss_interctc_layer21=66.096, loss=59.825, backward_time=0.225, grad_norm=99.967, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.145, optim0_lr0=7.154e-05, train_time=1.341 +[gpua006:0/64] 2024-02-11 22:19:40,001 (trainer:756) INFO: 32epoch:train:4101-4200batch: iter_time=8.902e-05, forward_time=0.144, loss_ctc=66.036, loss_interctc_layer6=83.297, loss_interctc_layer12=69.213, loss_interctc_layer15=63.512, loss_interctc_layer21=68.181, loss=70.048, backward_time=0.208, grad_norm=72.030, clip=100.000, loss_scale=3.631e+31, optim_step_time=0.137, optim0_lr0=7.153e-05, train_time=1.315 +[gpua006:0/64] 2024-02-11 22:21:44,400 (trainer:756) INFO: 32epoch:train:4201-4300batch: iter_time=8.937e-05, forward_time=0.143, loss_ctc=81.720, loss_interctc_layer6=84.679, loss_interctc_layer12=70.146, loss_interctc_layer15=64.436, loss_interctc_layer21=84.828, loss=77.162, backward_time=0.208, grad_norm=63.867, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.152e-05, train_time=1.244 +[gpua006:0/64] 2024-02-11 22:23:42,644 (trainer:756) INFO: 32epoch:train:4301-4400batch: iter_time=8.339e-05, forward_time=0.144, loss_ctc=77.057, loss_interctc_layer6=83.700, loss_interctc_layer12=69.551, loss_interctc_layer15=63.828, loss_interctc_layer21=79.974, loss=74.822, backward_time=0.209, grad_norm=74.427, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.152e-05, train_time=1.182 +[gpua006:0/64] 2024-02-11 22:25:35,259 (trainer:756) INFO: 32epoch:train:4401-4500batch: iter_time=8.197e-05, forward_time=0.143, loss_ctc=81.975, loss_interctc_layer6=86.802, loss_interctc_layer12=71.544, loss_interctc_layer15=65.341, loss_interctc_layer21=85.065, loss=78.145, backward_time=0.210, grad_norm=104.084, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.151e-05, train_time=1.126 +[gpua006:0/64] 2024-02-11 22:27:38,173 (trainer:756) INFO: 32epoch:train:4501-4600batch: iter_time=8.654e-05, forward_time=0.143, loss_ctc=74.115, loss_interctc_layer6=83.070, loss_interctc_layer12=68.838, loss_interctc_layer15=63.008, loss_interctc_layer21=76.827, loss=73.172, backward_time=0.208, grad_norm=70.927, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.150e-05, train_time=1.229 +[gpua006:0/64] 2024-02-11 22:29:18,474 (trainer:756) INFO: 32epoch:train:4601-4700batch: iter_time=8.424e-05, forward_time=0.143, loss_ctc=83.000, loss_interctc_layer6=85.989, loss_interctc_layer12=71.482, loss_interctc_layer15=65.465, loss_interctc_layer21=85.767, loss=78.340, backward_time=0.209, grad_norm=69.597, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.149e-05, train_time=1.003 +[gpua006:0/64] 2024-02-11 22:30:06,089 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-11 22:31:13,523 (trainer:756) INFO: 32epoch:train:4701-4800batch: iter_time=9.238e-05, forward_time=0.222, loss_ctc=85.373, loss_interctc_layer6=96.724, loss_interctc_layer12=80.365, loss_interctc_layer15=73.972, loss_interctc_layer21=88.146, loss=84.916, backward_time=0.237, grad_norm=157.219, clip=100.000, loss_scale=2.971e+31, optim_step_time=0.141, optim0_lr0=7.149e-05, train_time=1.150 +[gpua006:0/64] 2024-02-11 22:33:42,906 (trainer:756) INFO: 32epoch:train:4801-4900batch: iter_time=9.232e-05, forward_time=0.143, loss_ctc=71.186, loss_interctc_layer6=83.777, loss_interctc_layer12=69.086, loss_interctc_layer15=63.414, loss_interctc_layer21=73.823, loss=72.257, backward_time=0.207, grad_norm=76.748, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.148e-05, train_time=1.493 +[gpua006:0/64] 2024-02-11 22:35:39,219 (trainer:756) INFO: 32epoch:train:4901-5000batch: iter_time=8.898e-05, forward_time=0.144, loss_ctc=89.587, loss_interctc_layer6=101.968, loss_interctc_layer12=85.849, loss_interctc_layer15=79.471, loss_interctc_layer21=92.343, loss=89.844, backward_time=0.209, grad_norm=87.831, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.147e-05, train_time=1.163 +[gpua006:0/64] 2024-02-11 22:35:59,362 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-11 22:36:18,067 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 22:36:21,539 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 22:36:21,539 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-11 22:36:21,542 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-11 22:49:39,057 (trainer:756) INFO: 32epoch:train:5001-5100batch: iter_time=2.863, forward_time=0.143, loss_ctc=73.110, loss_interctc_layer6=76.666, loss_interctc_layer12=63.098, loss_interctc_layer15=57.631, loss_interctc_layer21=76.012, loss=69.303, backward_time=0.206, grad_norm=65.662, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.146e-05, train_time=8.399 +[gpua006:0/64] 2024-02-11 22:52:56,318 (trainer:756) INFO: 32epoch:train:5101-5200batch: iter_time=9.128e-05, forward_time=0.143, loss_ctc=78.081, loss_interctc_layer6=77.896, loss_interctc_layer12=64.287, loss_interctc_layer15=58.835, loss_interctc_layer21=81.010, loss=72.022, backward_time=0.206, grad_norm=59.584, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.146e-05, train_time=1.972 +[gpua006:0/64] 2024-02-11 22:56:02,939 (trainer:756) INFO: 32epoch:train:5201-5300batch: iter_time=8.774e-05, forward_time=0.144, loss_ctc=74.117, loss_interctc_layer6=75.832, loss_interctc_layer12=62.852, loss_interctc_layer15=57.629, loss_interctc_layer21=76.837, loss=69.453, backward_time=0.207, grad_norm=99.199, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.145e-05, train_time=1.866 +[gpua006:0/64] 2024-02-11 22:59:17,529 (trainer:756) INFO: 32epoch:train:5301-5400batch: iter_time=9.354e-05, forward_time=0.145, loss_ctc=71.203, loss_interctc_layer6=75.309, loss_interctc_layer12=62.084, loss_interctc_layer15=56.727, loss_interctc_layer21=73.764, loss=67.818, backward_time=0.208, grad_norm=62.210, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.144e-05, train_time=1.946 +[gpua006:0/64] 2024-02-11 23:02:16,244 (trainer:756) INFO: 32epoch:train:5401-5500batch: iter_time=9.186e-05, forward_time=0.146, loss_ctc=82.085, loss_interctc_layer6=86.179, loss_interctc_layer12=71.869, loss_interctc_layer15=66.094, loss_interctc_layer21=84.892, loss=78.224, backward_time=0.207, grad_norm=96.528, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.143e-05, train_time=1.787 +[gpua006:0/64] 2024-02-11 23:05:16,933 (trainer:756) INFO: 32epoch:train:5501-5600batch: iter_time=9.166e-05, forward_time=0.157, loss_ctc=82.610, loss_interctc_layer6=82.956, loss_interctc_layer12=68.663, loss_interctc_layer15=62.951, loss_interctc_layer21=85.782, loss=76.593, backward_time=0.207, grad_norm=113.266, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.143e-05, train_time=1.807 +[gpua006:0/64] 2024-02-11 23:08:21,253 (trainer:756) INFO: 32epoch:train:5601-5700batch: iter_time=9.238e-05, forward_time=0.240, loss_ctc=87.587, loss_interctc_layer6=88.317, loss_interctc_layer12=73.085, loss_interctc_layer15=67.228, loss_interctc_layer21=90.605, loss=81.364, backward_time=0.226, grad_norm=87.180, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.142e-05, train_time=1.843 +[gpua006:0/64] 2024-02-11 23:11:39,059 (trainer:756) INFO: 32epoch:train:5701-5800batch: iter_time=1.032e-04, forward_time=0.144, loss_ctc=73.198, loss_interctc_layer6=80.264, loss_interctc_layer12=66.115, loss_interctc_layer15=60.297, loss_interctc_layer21=75.779, loss=71.131, backward_time=0.207, grad_norm=70.212, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.141e-05, train_time=1.977 +[gpua006:0/64] 2024-02-11 23:14:13,079 (trainer:756) INFO: 32epoch:train:5801-5900batch: iter_time=8.662e-05, forward_time=0.144, loss_ctc=89.200, loss_interctc_layer6=84.881, loss_interctc_layer12=70.832, loss_interctc_layer15=65.099, loss_interctc_layer21=92.392, loss=80.481, backward_time=0.208, grad_norm=93.268, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.140e-05, train_time=1.540 +[gpua006:0/64] 2024-02-11 23:16:28,371 (trainer:756) INFO: 32epoch:train:5901-6000batch: iter_time=8.765e-05, forward_time=0.144, loss_ctc=86.684, loss_interctc_layer6=93.616, loss_interctc_layer12=77.693, loss_interctc_layer15=71.303, loss_interctc_layer21=89.733, loss=83.806, backward_time=0.208, grad_norm=76.105, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.139e-05, train_time=1.353 +[gpua006:0/64] 2024-02-11 23:18:35,391 (trainer:756) INFO: 32epoch:train:6001-6100batch: iter_time=8.175e-05, forward_time=0.146, loss_ctc=80.581, loss_interctc_layer6=88.466, loss_interctc_layer12=73.041, loss_interctc_layer15=66.865, loss_interctc_layer21=83.528, loss=78.496, backward_time=0.208, grad_norm=73.569, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.139e-05, train_time=1.270 +[gpua006:0/64] 2024-02-11 23:21:05,761 (trainer:756) INFO: 32epoch:train:6101-6200batch: iter_time=8.605e-05, forward_time=0.144, loss_ctc=94.181, loss_interctc_layer6=99.298, loss_interctc_layer12=83.509, loss_interctc_layer15=77.151, loss_interctc_layer21=97.117, loss=90.251, backward_time=0.207, grad_norm=102.102, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.138e-05, train_time=1.503 +[gpua006:0/64] 2024-02-11 23:22:40,664 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-11 23:22:59,369 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 23:23:02,845 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 23:23:02,845 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-11 23:23:02,848 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-11 23:31:41,225 (trainer:756) INFO: 32epoch:train:6201-6300batch: iter_time=2.760, forward_time=0.144, loss_ctc=68.183, loss_interctc_layer6=76.723, loss_interctc_layer12=63.105, loss_interctc_layer15=57.664, loss_interctc_layer21=70.568, loss=67.249, backward_time=0.208, grad_norm=86.784, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.137e-05, train_time=6.354 +[gpua006:0/64] 2024-02-11 23:33:18,058 (trainer:756) INFO: 32epoch:train:6301-6400batch: iter_time=8.047e-05, forward_time=0.143, loss_ctc=74.744, loss_interctc_layer6=81.113, loss_interctc_layer12=67.549, loss_interctc_layer15=62.127, loss_interctc_layer21=77.507, loss=72.608, backward_time=0.210, grad_norm=80.922, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.136e-05, train_time=0.968 +[gpua006:0/64] 2024-02-11 23:34:54,400 (trainer:756) INFO: 32epoch:train:6401-6500batch: iter_time=7.684e-05, forward_time=0.145, loss_ctc=83.122, loss_interctc_layer6=80.205, loss_interctc_layer12=66.313, loss_interctc_layer15=60.719, loss_interctc_layer21=86.366, loss=75.345, backward_time=0.209, grad_norm=77.288, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.136e-05, train_time=0.963 +[gpua006:0/64] 2024-02-11 23:36:53,969 (trainer:756) INFO: 32epoch:train:6501-6600batch: iter_time=2.963e-04, forward_time=0.146, loss_ctc=64.009, loss_interctc_layer6=65.313, loss_interctc_layer12=53.638, loss_interctc_layer15=48.877, loss_interctc_layer21=66.443, loss=59.656, backward_time=0.209, grad_norm=54.115, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.135e-05, train_time=1.195 +[gpua006:0/64] 2024-02-11 23:39:05,332 (trainer:756) INFO: 32epoch:train:6601-6700batch: iter_time=8.530e-05, forward_time=0.235, loss_ctc=71.933, loss_interctc_layer6=82.995, loss_interctc_layer12=68.867, loss_interctc_layer15=63.173, loss_interctc_layer21=74.180, loss=72.230, backward_time=0.234, grad_norm=64.519, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=7.134e-05, train_time=1.313 +[gpua006:0/64] 2024-02-11 23:41:34,744 (trainer:756) INFO: 32epoch:train:6701-6800batch: iter_time=9.022e-05, forward_time=0.143, loss_ctc=84.107, loss_interctc_layer6=84.335, loss_interctc_layer12=69.978, loss_interctc_layer15=64.226, loss_interctc_layer21=87.202, loss=77.970, backward_time=0.208, grad_norm=75.510, clip=100.000, loss_scale=3.103e+31, optim_step_time=0.137, optim0_lr0=7.133e-05, train_time=1.494 +[gpua006:0/64] 2024-02-11 23:44:10,374 (trainer:756) INFO: 32epoch:train:6801-6900batch: iter_time=9.004e-05, forward_time=0.143, loss_ctc=83.397, loss_interctc_layer6=83.581, loss_interctc_layer12=69.457, loss_interctc_layer15=63.715, loss_interctc_layer21=86.403, loss=77.311, backward_time=0.209, grad_norm=64.378, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.133e-05, train_time=1.555 +[gpua006:0/64] 2024-02-11 23:46:12,774 (trainer:756) INFO: 32epoch:train:6901-7000batch: iter_time=8.775e-05, forward_time=0.144, loss_ctc=84.836, loss_interctc_layer6=86.523, loss_interctc_layer12=71.310, loss_interctc_layer15=65.070, loss_interctc_layer21=87.991, loss=79.146, backward_time=0.209, grad_norm=69.255, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.132e-05, train_time=1.224 +[gpua006:0/64] 2024-02-11 23:48:39,451 (trainer:756) INFO: 32epoch:train:7001-7100batch: iter_time=8.147e-05, forward_time=0.143, loss_ctc=76.072, loss_interctc_layer6=83.262, loss_interctc_layer12=68.987, loss_interctc_layer15=63.113, loss_interctc_layer21=78.829, loss=74.052, backward_time=0.207, grad_norm=63.569, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.131e-05, train_time=1.468 +[gpua006:0/64] 2024-02-11 23:51:10,508 (trainer:756) INFO: 32epoch:train:7101-7200batch: iter_time=8.500e-05, forward_time=0.144, loss_ctc=86.693, loss_interctc_layer6=85.553, loss_interctc_layer12=70.969, loss_interctc_layer15=65.077, loss_interctc_layer21=89.636, loss=79.585, backward_time=0.207, grad_norm=81.704, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.130e-05, train_time=1.510 +[gpua006:0/64] 2024-02-11 23:53:39,752 (trainer:756) INFO: 32epoch:train:7201-7300batch: iter_time=8.304e-05, forward_time=0.148, loss_ctc=87.311, loss_interctc_layer6=95.332, loss_interctc_layer12=79.252, loss_interctc_layer15=72.817, loss_interctc_layer21=90.205, loss=84.983, backward_time=0.209, grad_norm=80.539, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.130e-05, train_time=1.492 +[gpua006:0/64] 2024-02-11 23:56:09,617 (trainer:756) INFO: 32epoch:train:7301-7400batch: iter_time=8.487e-05, forward_time=0.143, loss_ctc=75.145, loss_interctc_layer6=82.482, loss_interctc_layer12=68.026, loss_interctc_layer15=62.278, loss_interctc_layer21=77.903, loss=73.167, backward_time=0.208, grad_norm=80.471, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.129e-05, train_time=1.498 +[gpua006:0/64] 2024-02-11 23:58:35,269 (trainer:756) INFO: 32epoch:train:7401-7500batch: iter_time=8.428e-05, forward_time=0.144, loss_ctc=92.430, loss_interctc_layer6=101.502, loss_interctc_layer12=85.304, loss_interctc_layer15=78.846, loss_interctc_layer21=95.281, loss=90.673, backward_time=0.208, grad_norm=92.830, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.128e-05, train_time=1.456 +[gpua006:0/64] 2024-02-11 23:58:55,300 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-11 23:59:14,086 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-11 23:59:17,553 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-11 23:59:17,553 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-11 23:59:17,556 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 00:05:40,903 (trainer:756) INFO: 32epoch:train:7501-7600batch: iter_time=2.943, forward_time=0.182, loss_ctc=72.948, loss_interctc_layer6=76.531, loss_interctc_layer12=63.040, loss_interctc_layer15=57.537, loss_interctc_layer21=75.690, loss=69.149, backward_time=0.219, grad_norm=61.532, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=7.127e-05, train_time=4.256 +[gpua006:0/64] 2024-02-12 00:07:17,141 (trainer:756) INFO: 32epoch:train:7601-7700batch: iter_time=8.776e-05, forward_time=0.144, loss_ctc=77.157, loss_interctc_layer6=77.027, loss_interctc_layer12=63.664, loss_interctc_layer15=58.196, loss_interctc_layer21=80.020, loss=71.213, backward_time=0.210, grad_norm=73.111, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=7.127e-05, train_time=0.962 +[gpua006:0/64] 2024-02-12 00:09:15,834 (trainer:756) INFO: 32epoch:train:7701-7800batch: iter_time=9.406e-05, forward_time=0.144, loss_ctc=76.267, loss_interctc_layer6=76.436, loss_interctc_layer12=63.798, loss_interctc_layer15=58.703, loss_interctc_layer21=78.686, loss=70.778, backward_time=0.209, grad_norm=78.144, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.126e-05, train_time=1.186 +[gpua006:0/64] 2024-02-12 00:11:17,674 (trainer:756) INFO: 32epoch:train:7801-7900batch: iter_time=9.686e-05, forward_time=0.144, loss_ctc=69.761, loss_interctc_layer6=74.458, loss_interctc_layer12=61.377, loss_interctc_layer15=56.036, loss_interctc_layer21=72.169, loss=66.760, backward_time=0.209, grad_norm=63.069, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.125e-05, train_time=1.219 +[gpua006:0/64] 2024-02-12 00:13:41,390 (trainer:756) INFO: 32epoch:train:7901-8000batch: iter_time=8.580e-05, forward_time=0.144, loss_ctc=80.562, loss_interctc_layer6=85.691, loss_interctc_layer12=71.314, loss_interctc_layer15=65.589, loss_interctc_layer21=83.474, loss=77.326, backward_time=0.208, grad_norm=100.656, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.124e-05, train_time=1.437 +[gpua006:0/64] 2024-02-12 00:16:08,757 (trainer:756) INFO: 32epoch:train:8001-8100batch: iter_time=8.538e-05, forward_time=0.148, loss_ctc=80.994, loss_interctc_layer6=82.391, loss_interctc_layer12=68.323, loss_interctc_layer15=62.561, loss_interctc_layer21=84.037, loss=75.661, backward_time=0.212, grad_norm=111.683, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.124e-05, train_time=1.473 +[gpua006:0/64] 2024-02-12 00:17:52,132 (trainer:756) INFO: 32epoch:train:8101-8200batch: iter_time=8.063e-05, forward_time=0.145, loss_ctc=86.751, loss_interctc_layer6=88.386, loss_interctc_layer12=73.178, loss_interctc_layer15=67.017, loss_interctc_layer21=89.680, loss=81.002, backward_time=0.210, grad_norm=72.385, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.123e-05, train_time=1.034 +[gpua006:0/64] 2024-02-12 00:20:01,464 (trainer:756) INFO: 32epoch:train:8201-8300batch: iter_time=8.807e-05, forward_time=0.143, loss_ctc=72.606, loss_interctc_layer6=80.041, loss_interctc_layer12=65.875, loss_interctc_layer15=60.051, loss_interctc_layer21=75.075, loss=70.729, backward_time=0.209, grad_norm=63.713, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.122e-05, train_time=1.293 +[gpua006:0/64] 2024-02-12 00:21:07,542 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 00:22:12,905 (trainer:756) INFO: 32epoch:train:8301-8400batch: iter_time=8.305e-05, forward_time=0.230, loss_ctc=87.871, loss_interctc_layer6=84.243, loss_interctc_layer12=70.178, loss_interctc_layer15=64.415, loss_interctc_layer21=91.029, loss=79.547, backward_time=0.238, grad_norm=80.574, clip=100.000, loss_scale=2.991e+31, optim_step_time=0.139, optim0_lr0=7.121e-05, train_time=1.314 +[gpua006:0/64] 2024-02-12 00:24:32,646 (trainer:756) INFO: 32epoch:train:8401-8500batch: iter_time=8.196e-05, forward_time=0.146, loss_ctc=85.988, loss_interctc_layer6=93.376, loss_interctc_layer12=77.347, loss_interctc_layer15=70.907, loss_interctc_layer21=88.982, loss=83.320, backward_time=0.208, grad_norm=100.848, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.121e-05, train_time=1.398 +[gpua006:0/64] 2024-02-12 00:26:16,768 (trainer:756) INFO: 32epoch:train:8501-8600batch: iter_time=8.552e-05, forward_time=0.143, loss_ctc=79.252, loss_interctc_layer6=87.589, loss_interctc_layer12=72.118, loss_interctc_layer15=65.905, loss_interctc_layer21=82.072, loss=77.387, backward_time=0.209, grad_norm=69.696, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.120e-05, train_time=1.041 +[gpua006:0/64] 2024-02-12 00:27:20,717 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 00:28:40,836 (trainer:756) INFO: 32epoch:train:8601-8700batch: iter_time=8.286e-05, forward_time=0.146, loss_ctc=93.397, loss_interctc_layer6=98.793, loss_interctc_layer12=82.996, loss_interctc_layer15=76.736, loss_interctc_layer21=96.531, loss=89.691, backward_time=0.207, grad_norm=86.508, clip=100.000, loss_scale=1.578e+31, optim_step_time=0.136, optim0_lr0=7.119e-05, train_time=1.440 +[gpua006:0/64] 2024-02-12 00:30:02,065 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-12 00:30:21,284 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 00:30:24,795 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 00:30:24,795 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-12 00:30:24,799 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 00:36:01,817 (trainer:756) INFO: 32epoch:train:8701-8800batch: iter_time=3.203, forward_time=0.213, loss_ctc=66.923, loss_interctc_layer6=76.626, loss_interctc_layer12=62.827, loss_interctc_layer15=57.282, loss_interctc_layer21=69.364, loss=66.604, backward_time=0.224, grad_norm=61.422, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.118e-05, train_time=4.410 +[gpua006:0/64] 2024-02-12 00:37:38,852 (trainer:756) INFO: 32epoch:train:8801-8900batch: iter_time=8.181e-05, forward_time=0.145, loss_ctc=70.511, loss_interctc_layer6=81.385, loss_interctc_layer12=67.673, loss_interctc_layer15=62.209, loss_interctc_layer21=73.257, loss=71.007, backward_time=0.210, grad_norm=66.979, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.118e-05, train_time=0.970 +[gpua006:0/64] 2024-02-12 00:39:44,564 (trainer:756) INFO: 32epoch:train:8901-9000batch: iter_time=8.235e-05, forward_time=0.144, loss_ctc=81.349, loss_interctc_layer6=79.837, loss_interctc_layer12=66.016, loss_interctc_layer15=60.460, loss_interctc_layer21=84.387, loss=74.410, backward_time=0.209, grad_norm=67.083, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.117e-05, train_time=1.257 +[gpua006:0/64] 2024-02-12 00:41:37,010 (trainer:756) INFO: 32epoch:train:9001-9100batch: iter_time=8.728e-05, forward_time=0.145, loss_ctc=63.430, loss_interctc_layer6=65.172, loss_interctc_layer12=53.408, loss_interctc_layer15=48.620, loss_interctc_layer21=65.907, loss=59.308, backward_time=0.209, grad_norm=117.543, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.116e-05, train_time=1.124 +[gpua006:0/64] 2024-02-12 00:43:40,040 (trainer:756) INFO: 32epoch:train:9101-9200batch: iter_time=9.123e-05, forward_time=0.145, loss_ctc=65.956, loss_interctc_layer6=82.772, loss_interctc_layer12=68.614, loss_interctc_layer15=62.829, loss_interctc_layer21=68.185, loss=69.671, backward_time=0.209, grad_norm=72.603, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.115e-05, train_time=1.230 +[gpua006:0/64] 2024-02-12 00:46:04,042 (trainer:756) INFO: 32epoch:train:9201-9300batch: iter_time=8.899e-05, forward_time=0.245, loss_ctc=81.202, loss_interctc_layer6=84.585, loss_interctc_layer12=70.166, loss_interctc_layer15=64.391, loss_interctc_layer21=84.229, loss=76.915, backward_time=0.271, grad_norm=102.193, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=7.115e-05, train_time=1.439 +[gpua006:0/64] 2024-02-12 00:48:21,685 (trainer:756) INFO: 32epoch:train:9301-9400batch: iter_time=8.657e-05, forward_time=0.145, loss_ctc=77.658, loss_interctc_layer6=83.825, loss_interctc_layer12=69.680, loss_interctc_layer15=63.945, loss_interctc_layer21=80.492, loss=75.120, backward_time=0.209, grad_norm=96.160, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.114e-05, train_time=1.376 +[gpua006:0/64] 2024-02-12 00:50:08,320 (trainer:756) INFO: 32epoch:train:9401-9500batch: iter_time=8.932e-05, forward_time=0.145, loss_ctc=81.010, loss_interctc_layer6=86.178, loss_interctc_layer12=70.905, loss_interctc_layer15=64.734, loss_interctc_layer21=84.104, loss=77.386, backward_time=0.209, grad_norm=70.146, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.113e-05, train_time=1.067 +[gpua006:0/64] 2024-02-12 00:52:21,399 (trainer:756) INFO: 32epoch:train:9501-9600batch: iter_time=9.323e-05, forward_time=0.146, loss_ctc=73.537, loss_interctc_layer6=82.269, loss_interctc_layer12=68.073, loss_interctc_layer15=62.197, loss_interctc_layer21=76.401, loss=72.495, backward_time=0.209, grad_norm=101.167, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.112e-05, train_time=1.331 +[gpua006:0/64] 2024-02-12 00:54:12,852 (trainer:756) INFO: 32epoch:train:9601-9700batch: iter_time=9.539e-05, forward_time=0.144, loss_ctc=82.847, loss_interctc_layer6=85.858, loss_interctc_layer12=71.255, loss_interctc_layer15=65.219, loss_interctc_layer21=85.702, loss=78.176, backward_time=0.209, grad_norm=88.237, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.112e-05, train_time=1.114 +[gpua006:0/64] 2024-02-12 00:56:32,550 (trainer:756) INFO: 32epoch:train:9701-9800batch: iter_time=9.282e-05, forward_time=0.145, loss_ctc=84.321, loss_interctc_layer6=95.297, loss_interctc_layer12=78.994, loss_interctc_layer15=72.429, loss_interctc_layer21=87.247, loss=83.658, backward_time=0.207, grad_norm=81.969, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.111e-05, train_time=1.397 +[gpua006:0/64] 2024-02-12 00:58:32,507 (trainer:756) INFO: 32epoch:train:9801-9900batch: iter_time=8.454e-05, forward_time=0.145, loss_ctc=70.088, loss_interctc_layer6=82.612, loss_interctc_layer12=68.001, loss_interctc_layer15=62.246, loss_interctc_layer21=72.645, loss=71.118, backward_time=0.210, grad_norm=132.568, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.110e-05, train_time=1.199 +[gpua006:0/64] 2024-02-12 01:00:16,690 (trainer:756) INFO: 32epoch:train:9901-10000batch: iter_time=7.964e-05, forward_time=0.144, loss_ctc=88.881, loss_interctc_layer6=100.666, loss_interctc_layer12=84.619, loss_interctc_layer15=78.298, loss_interctc_layer21=91.854, loss=88.864, backward_time=0.209, grad_norm=127.825, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.109e-05, train_time=1.042 +[gpua006:0/64] 2024-02-12 01:00:36,720 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-12 01:00:55,656 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 01:00:59,155 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 01:00:59,155 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-12 01:00:59,158 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 01:09:56,838 (trainer:756) INFO: 32epoch:train:10001-10100batch: iter_time=3.081, forward_time=0.185, loss_ctc=73.309, loss_interctc_layer6=76.903, loss_interctc_layer12=63.364, loss_interctc_layer15=57.802, loss_interctc_layer21=76.156, loss=69.507, backward_time=0.218, grad_norm=166.054, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.109e-05, train_time=5.801 +[gpua006:0/64] 2024-02-12 01:11:39,279 (trainer:756) INFO: 32epoch:train:10101-10200batch: iter_time=8.233e-05, forward_time=0.146, loss_ctc=76.600, loss_interctc_layer6=76.218, loss_interctc_layer12=62.957, loss_interctc_layer15=57.522, loss_interctc_layer21=79.387, loss=70.537, backward_time=0.210, grad_norm=70.350, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.108e-05, train_time=1.025 +[gpua006:0/64] 2024-02-12 01:13:25,043 (trainer:756) INFO: 32epoch:train:10201-10300batch: iter_time=8.299e-05, forward_time=0.144, loss_ctc=73.293, loss_interctc_layer6=75.120, loss_interctc_layer12=62.195, loss_interctc_layer15=56.874, loss_interctc_layer21=76.242, loss=68.745, backward_time=0.209, grad_norm=72.758, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.107e-05, train_time=1.057 +[gpua006:0/64] 2024-02-12 01:15:35,865 (trainer:756) INFO: 32epoch:train:10301-10400batch: iter_time=8.649e-05, forward_time=0.144, loss_ctc=70.531, loss_interctc_layer6=74.668, loss_interctc_layer12=61.505, loss_interctc_layer15=56.171, loss_interctc_layer21=73.086, loss=67.192, backward_time=0.209, grad_norm=56.988, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.106e-05, train_time=1.308 +[gpua006:0/64] 2024-02-12 01:17:44,166 (trainer:756) INFO: 32epoch:train:10401-10500batch: iter_time=8.932e-05, forward_time=0.144, loss_ctc=80.948, loss_interctc_layer6=85.314, loss_interctc_layer12=70.897, loss_interctc_layer15=65.123, loss_interctc_layer21=83.731, loss=77.203, backward_time=0.209, grad_norm=72.642, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.106e-05, train_time=1.283 +[gpua006:0/64] 2024-02-12 01:20:10,817 (trainer:756) INFO: 32epoch:train:10501-10600batch: iter_time=8.881e-05, forward_time=0.144, loss_ctc=81.249, loss_interctc_layer6=81.517, loss_interctc_layer12=67.491, loss_interctc_layer15=61.985, loss_interctc_layer21=84.260, loss=75.301, backward_time=0.208, grad_norm=83.713, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.105e-05, train_time=1.466 +[gpua006:0/64] 2024-02-12 01:22:09,950 (trainer:756) INFO: 32epoch:train:10601-10700batch: iter_time=8.638e-05, forward_time=0.194, loss_ctc=87.169, loss_interctc_layer6=88.216, loss_interctc_layer12=72.979, loss_interctc_layer15=66.958, loss_interctc_layer21=90.284, loss=81.121, backward_time=0.219, grad_norm=85.204, clip=100.000, loss_scale=1.460e+31, optim_step_time=0.140, optim0_lr0=7.104e-05, train_time=1.191 +[gpua006:0/64] 2024-02-12 01:24:35,731 (trainer:756) INFO: 32epoch:train:10701-10800batch: iter_time=9.613e-05, forward_time=0.218, loss_ctc=71.531, loss_interctc_layer6=79.519, loss_interctc_layer12=65.370, loss_interctc_layer15=59.485, loss_interctc_layer21=74.076, loss=69.996, backward_time=0.218, grad_norm=58.667, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=7.103e-05, train_time=1.457 +[gpua006:0/64] 2024-02-12 01:26:07,080 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 01:26:28,874 (trainer:756) INFO: 32epoch:train:10801-10900batch: iter_time=9.410e-05, forward_time=0.145, loss_ctc=89.324, loss_interctc_layer6=84.673, loss_interctc_layer12=70.688, loss_interctc_layer15=64.885, loss_interctc_layer21=92.510, loss=80.416, backward_time=0.210, grad_norm=69.269, clip=100.000, loss_scale=1.793e+31, optim_step_time=0.138, optim0_lr0=7.103e-05, train_time=1.131 +[gpua006:0/64] 2024-02-12 01:28:43,019 (trainer:756) INFO: 32epoch:train:10901-11000batch: iter_time=9.340e-05, forward_time=0.145, loss_ctc=86.409, loss_interctc_layer6=93.527, loss_interctc_layer12=77.600, loss_interctc_layer15=71.151, loss_interctc_layer21=89.297, loss=83.597, backward_time=0.209, grad_norm=77.465, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.102e-05, train_time=1.342 +[gpua006:0/64] 2024-02-12 01:30:36,487 (trainer:756) INFO: 32epoch:train:11001-11100batch: iter_time=9.075e-05, forward_time=0.145, loss_ctc=79.571, loss_interctc_layer6=87.364, loss_interctc_layer12=71.862, loss_interctc_layer15=65.737, loss_interctc_layer21=82.380, loss=77.383, backward_time=0.209, grad_norm=95.951, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.101e-05, train_time=1.134 +[gpua006:0/64] 2024-02-12 01:32:41,110 (trainer:756) INFO: 32epoch:train:11101-11200batch: iter_time=8.969e-05, forward_time=0.145, loss_ctc=93.208, loss_interctc_layer6=99.053, loss_interctc_layer12=83.202, loss_interctc_layer15=77.093, loss_interctc_layer21=96.337, loss=89.779, backward_time=0.209, grad_norm=134.883, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.100e-05, train_time=1.246 +[gpua006:0/64] 2024-02-12 01:33:55,546 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-12 01:34:14,916 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 01:34:18,371 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 01:34:18,371 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-12 01:34:18,374 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 01:44:39,594 (trainer:756) INFO: 32epoch:train:11201-11300batch: iter_time=3.091, forward_time=0.199, loss_ctc=67.960, loss_interctc_layer6=76.649, loss_interctc_layer12=63.068, loss_interctc_layer15=57.471, loss_interctc_layer21=70.371, loss=67.104, backward_time=0.218, grad_norm=80.127, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.100e-05, train_time=7.184 +[gpua006:0/64] 2024-02-12 01:48:03,662 (trainer:756) INFO: 32epoch:train:11301-11400batch: iter_time=8.678e-05, forward_time=0.144, loss_ctc=69.430, loss_interctc_layer6=80.109, loss_interctc_layer12=66.801, loss_interctc_layer15=61.458, loss_interctc_layer21=72.034, loss=69.966, backward_time=0.209, grad_norm=79.009, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.099e-05, train_time=2.041 +[gpua006:0/64] 2024-02-12 01:51:03,404 (trainer:756) INFO: 32epoch:train:11401-11500batch: iter_time=8.828e-05, forward_time=0.143, loss_ctc=79.917, loss_interctc_layer6=79.206, loss_interctc_layer12=65.570, loss_interctc_layer15=60.057, loss_interctc_layer21=82.930, loss=73.536, backward_time=0.208, grad_norm=74.244, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.098e-05, train_time=1.797 +[gpua006:0/64] 2024-02-12 01:53:42,618 (trainer:756) INFO: 32epoch:train:11501-11600batch: iter_time=9.022e-05, forward_time=0.143, loss_ctc=62.766, loss_interctc_layer6=64.727, loss_interctc_layer12=53.027, loss_interctc_layer15=48.304, loss_interctc_layer21=65.148, loss=58.794, backward_time=0.208, grad_norm=56.003, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.097e-05, train_time=1.592 +[gpua006:0/64] 2024-02-12 01:55:49,772 (trainer:756) INFO: 32epoch:train:11601-11700batch: iter_time=8.802e-05, forward_time=0.144, loss_ctc=65.855, loss_interctc_layer6=83.083, loss_interctc_layer12=68.846, loss_interctc_layer15=63.045, loss_interctc_layer21=68.019, loss=69.770, backward_time=0.210, grad_norm=79.520, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.097e-05, train_time=1.271 +[gpua006:0/64] 2024-02-12 01:58:01,517 (trainer:756) INFO: 32epoch:train:11701-11800batch: iter_time=9.033e-05, forward_time=0.144, loss_ctc=80.231, loss_interctc_layer6=83.360, loss_interctc_layer12=69.176, loss_interctc_layer15=63.565, loss_interctc_layer21=83.202, loss=75.907, backward_time=0.209, grad_norm=74.699, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.096e-05, train_time=1.317 +[gpua006:0/64] 2024-02-12 02:00:35,797 (trainer:756) INFO: 32epoch:train:11801-11900batch: iter_time=9.011e-05, forward_time=0.170, loss_ctc=76.220, loss_interctc_layer6=82.965, loss_interctc_layer12=68.717, loss_interctc_layer15=63.046, loss_interctc_layer21=79.121, loss=74.014, backward_time=0.282, grad_norm=79.123, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.143, optim0_lr0=7.095e-05, train_time=1.542 +[gpua006:0/64] 2024-02-12 02:02:55,574 (trainer:756) INFO: 32epoch:train:11901-12000batch: iter_time=8.916e-05, forward_time=0.157, loss_ctc=81.142, loss_interctc_layer6=86.186, loss_interctc_layer12=70.890, loss_interctc_layer15=64.664, loss_interctc_layer21=84.109, loss=77.398, backward_time=0.208, grad_norm=88.701, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.094e-05, train_time=1.397 +[gpua006:0/64] 2024-02-12 02:05:10,409 (trainer:756) INFO: 32epoch:train:12001-12100batch: iter_time=9.124e-05, forward_time=0.144, loss_ctc=73.790, loss_interctc_layer6=82.932, loss_interctc_layer12=68.620, loss_interctc_layer15=62.660, loss_interctc_layer21=76.571, loss=72.914, backward_time=0.210, grad_norm=78.852, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.094e-05, train_time=1.349 +[gpua006:0/64] 2024-02-12 02:06:20,896 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 02:06:59,715 (trainer:756) INFO: 32epoch:train:12101-12200batch: iter_time=8.986e-05, forward_time=0.144, loss_ctc=81.709, loss_interctc_layer6=85.773, loss_interctc_layer12=71.018, loss_interctc_layer15=65.096, loss_interctc_layer21=84.653, loss=77.650, backward_time=0.209, grad_norm=76.417, clip=100.000, loss_scale=8.707e+30, optim_step_time=0.136, optim0_lr0=7.093e-05, train_time=1.091 +[gpua006:0/64] 2024-02-12 02:09:45,522 (trainer:756) INFO: 32epoch:train:12201-12300batch: iter_time=8.373e-05, forward_time=0.144, loss_ctc=83.967, loss_interctc_layer6=95.028, loss_interctc_layer12=78.672, loss_interctc_layer15=72.142, loss_interctc_layer21=86.652, loss=83.292, backward_time=0.208, grad_norm=71.942, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.092e-05, train_time=1.660 +[gpua006:0/64] 2024-02-12 02:11:41,506 (trainer:756) INFO: 32epoch:train:12301-12400batch: iter_time=8.490e-05, forward_time=0.144, loss_ctc=69.740, loss_interctc_layer6=82.172, loss_interctc_layer12=67.559, loss_interctc_layer15=61.795, loss_interctc_layer21=72.358, loss=70.725, backward_time=0.210, grad_norm=67.995, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.091e-05, train_time=1.160 +[gpua006:0/64] 2024-02-12 02:13:42,738 (trainer:756) INFO: 32epoch:train:12401-12500batch: iter_time=8.566e-05, forward_time=0.144, loss_ctc=88.242, loss_interctc_layer6=100.382, loss_interctc_layer12=84.232, loss_interctc_layer15=77.779, loss_interctc_layer21=90.920, loss=88.311, backward_time=0.209, grad_norm=80.011, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.091e-05, train_time=1.212 +[gpua006:0/64] 2024-02-12 02:14:02,767 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-12 02:14:21,734 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 02:14:25,209 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 02:14:25,209 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-12 02:14:25,232 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 02:23:27,826 (trainer:756) INFO: 32epoch:train:12501-12600batch: iter_time=3.000, forward_time=0.180, loss_ctc=72.479, loss_interctc_layer6=76.261, loss_interctc_layer12=62.720, loss_interctc_layer15=57.213, loss_interctc_layer21=75.368, loss=68.808, backward_time=0.221, grad_norm=77.449, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.090e-05, train_time=5.850 +[gpua006:0/64] 2024-02-12 02:25:29,958 (trainer:756) INFO: 32epoch:train:12601-12700batch: iter_time=8.408e-05, forward_time=0.143, loss_ctc=76.882, loss_interctc_layer6=76.861, loss_interctc_layer12=63.420, loss_interctc_layer15=57.980, loss_interctc_layer21=79.655, loss=70.960, backward_time=0.209, grad_norm=69.225, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.089e-05, train_time=1.221 +[gpua006:0/64] 2024-02-12 02:27:33,406 (trainer:756) INFO: 32epoch:train:12701-12800batch: iter_time=8.491e-05, forward_time=0.143, loss_ctc=73.123, loss_interctc_layer6=74.512, loss_interctc_layer12=61.541, loss_interctc_layer15=56.424, loss_interctc_layer21=75.601, loss=68.240, backward_time=0.208, grad_norm=76.055, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.089e-05, train_time=1.234 +[gpua006:0/64] 2024-02-12 02:29:43,566 (trainer:756) INFO: 32epoch:train:12801-12900batch: iter_time=9.051e-05, forward_time=0.144, loss_ctc=70.373, loss_interctc_layer6=74.651, loss_interctc_layer12=61.640, loss_interctc_layer15=56.291, loss_interctc_layer21=72.903, loss=67.172, backward_time=0.208, grad_norm=83.137, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.088e-05, train_time=1.301 +[gpua006:0/64] 2024-02-12 02:31:48,831 (trainer:756) INFO: 32epoch:train:12901-13000batch: iter_time=9.665e-05, forward_time=0.145, loss_ctc=79.754, loss_interctc_layer6=84.285, loss_interctc_layer12=70.054, loss_interctc_layer15=64.388, loss_interctc_layer21=82.405, loss=76.177, backward_time=0.208, grad_norm=75.468, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.087e-05, train_time=1.252 +[gpua006:0/64] 2024-02-12 02:34:43,733 (trainer:756) INFO: 32epoch:train:13001-13100batch: iter_time=8.163e-05, forward_time=0.195, loss_ctc=81.843, loss_interctc_layer6=82.198, loss_interctc_layer12=67.977, loss_interctc_layer15=62.238, loss_interctc_layer21=85.020, loss=75.855, backward_time=0.233, grad_norm=91.514, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.140, optim0_lr0=7.086e-05, train_time=1.749 +[gpua006:0/64] 2024-02-12 02:36:58,669 (trainer:756) INFO: 32epoch:train:13101-13200batch: iter_time=8.139e-05, forward_time=0.192, loss_ctc=86.202, loss_interctc_layer6=87.696, loss_interctc_layer12=72.541, loss_interctc_layer15=66.598, loss_interctc_layer21=89.468, loss=80.501, backward_time=0.213, grad_norm=75.806, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.086e-05, train_time=1.349 +[gpua006:0/64] 2024-02-12 02:38:54,614 (trainer:756) INFO: 32epoch:train:13201-13300batch: iter_time=8.256e-05, forward_time=0.144, loss_ctc=72.089, loss_interctc_layer6=79.868, loss_interctc_layer12=65.620, loss_interctc_layer15=59.754, loss_interctc_layer21=74.621, loss=70.391, backward_time=0.209, grad_norm=79.477, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.085e-05, train_time=1.159 +[gpua006:0/64] 2024-02-12 02:40:57,889 (trainer:756) INFO: 32epoch:train:13301-13400batch: iter_time=8.424e-05, forward_time=0.144, loss_ctc=88.428, loss_interctc_layer6=84.027, loss_interctc_layer12=70.021, loss_interctc_layer15=64.344, loss_interctc_layer21=91.682, loss=79.700, backward_time=0.208, grad_norm=76.761, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.084e-05, train_time=1.233 +[gpua006:0/64] 2024-02-12 02:43:22,248 (trainer:756) INFO: 32epoch:train:13401-13500batch: iter_time=8.679e-05, forward_time=0.144, loss_ctc=86.266, loss_interctc_layer6=93.074, loss_interctc_layer12=77.159, loss_interctc_layer15=70.668, loss_interctc_layer21=89.284, loss=83.290, backward_time=0.208, grad_norm=107.537, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.083e-05, train_time=1.443 +[gpua006:0/64] 2024-02-12 02:45:29,452 (trainer:756) INFO: 32epoch:train:13501-13600batch: iter_time=8.952e-05, forward_time=0.144, loss_ctc=79.938, loss_interctc_layer6=87.932, loss_interctc_layer12=72.366, loss_interctc_layer15=66.123, loss_interctc_layer21=82.788, loss=77.830, backward_time=0.209, grad_norm=149.400, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.083e-05, train_time=1.272 +[gpua006:0/64] 2024-02-12 02:47:23,660 (trainer:756) INFO: 32epoch:train:13601-13700batch: iter_time=8.194e-05, forward_time=0.194, loss_ctc=93.028, loss_interctc_layer6=98.532, loss_interctc_layer12=82.544, loss_interctc_layer15=76.300, loss_interctc_layer21=96.113, loss=89.303, backward_time=0.235, grad_norm=95.524, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=7.082e-05, train_time=1.142 +[gpua006:0/64] 2024-02-12 02:48:43,623 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-12 02:49:02,786 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 02:49:06,261 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 02:49:06,261 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-12 02:49:06,265 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 02:54:55,653 (trainer:756) INFO: 32epoch:train:13701-13800batch: iter_time=3.037, forward_time=0.184, loss_ctc=67.548, loss_interctc_layer6=76.305, loss_interctc_layer12=62.661, loss_interctc_layer15=57.174, loss_interctc_layer21=69.905, loss=66.719, backward_time=0.219, grad_norm=61.741, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=7.081e-05, train_time=4.520 +[gpua006:0/64] 2024-02-12 02:56:31,891 (trainer:756) INFO: 32epoch:train:13801-13900batch: iter_time=7.985e-05, forward_time=0.143, loss_ctc=69.504, loss_interctc_layer6=80.876, loss_interctc_layer12=67.130, loss_interctc_layer15=61.732, loss_interctc_layer21=72.150, loss=70.279, backward_time=0.209, grad_norm=126.350, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.080e-05, train_time=0.962 +[gpua006:0/64] 2024-02-12 02:58:34,751 (trainer:756) INFO: 32epoch:train:13901-14000batch: iter_time=8.059e-05, forward_time=0.143, loss_ctc=79.818, loss_interctc_layer6=79.202, loss_interctc_layer12=65.306, loss_interctc_layer15=59.781, loss_interctc_layer21=83.038, loss=73.429, backward_time=0.209, grad_norm=65.982, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.080e-05, train_time=1.228 +[gpua006:0/64] 2024-02-12 03:00:41,645 (trainer:756) INFO: 32epoch:train:14001-14100batch: iter_time=8.162e-05, forward_time=0.142, loss_ctc=62.643, loss_interctc_layer6=64.484, loss_interctc_layer12=52.715, loss_interctc_layer15=48.022, loss_interctc_layer21=65.080, loss=58.589, backward_time=0.208, grad_norm=58.111, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.079e-05, train_time=1.269 +[gpua006:0/64] 2024-02-12 03:02:27,228 (trainer:756) INFO: 32epoch:train:14101-14200batch: iter_time=8.207e-05, forward_time=0.144, loss_ctc=65.509, loss_interctc_layer6=82.641, loss_interctc_layer12=68.514, loss_interctc_layer15=62.780, loss_interctc_layer21=67.592, loss=69.407, backward_time=0.209, grad_norm=102.028, clip=100.000, loss_scale=6.490e+30, optim_step_time=0.137, optim0_lr0=7.078e-05, train_time=1.056 +[gpua006:0/64] 2024-02-12 03:04:42,372 (trainer:756) INFO: 32epoch:train:14201-14300batch: iter_time=8.035e-05, forward_time=0.143, loss_ctc=80.708, loss_interctc_layer6=84.112, loss_interctc_layer12=69.937, loss_interctc_layer15=64.085, loss_interctc_layer21=83.671, loss=76.503, backward_time=0.208, grad_norm=80.609, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.077e-05, train_time=1.351 +[gpua006:0/64] 2024-02-12 03:06:37,406 (trainer:756) INFO: 32epoch:train:14301-14400batch: iter_time=8.101e-05, forward_time=0.155, loss_ctc=76.563, loss_interctc_layer6=83.093, loss_interctc_layer12=69.003, loss_interctc_layer15=63.244, loss_interctc_layer21=79.504, loss=74.281, backward_time=0.209, grad_norm=119.477, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.077e-05, train_time=1.150 +[gpua006:0/64] 2024-02-12 03:09:26,236 (trainer:756) INFO: 32epoch:train:14401-14500batch: iter_time=8.241e-05, forward_time=0.221, loss_ctc=81.192, loss_interctc_layer6=85.866, loss_interctc_layer12=70.754, loss_interctc_layer15=64.481, loss_interctc_layer21=84.341, loss=77.327, backward_time=0.223, grad_norm=69.524, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=7.076e-05, train_time=1.688 +[gpua006:0/64] 2024-02-12 03:11:56,120 (trainer:756) INFO: 32epoch:train:14501-14600batch: iter_time=7.798e-05, forward_time=0.144, loss_ctc=73.934, loss_interctc_layer6=82.637, loss_interctc_layer12=68.404, loss_interctc_layer15=62.503, loss_interctc_layer21=76.659, loss=72.827, backward_time=0.209, grad_norm=69.653, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.075e-05, train_time=1.498 +[gpua006:0/64] 2024-02-12 03:14:32,588 (trainer:756) INFO: 32epoch:train:14601-14700batch: iter_time=8.210e-05, forward_time=0.143, loss_ctc=82.068, loss_interctc_layer6=85.522, loss_interctc_layer12=70.814, loss_interctc_layer15=64.846, loss_interctc_layer21=84.850, loss=77.620, backward_time=0.208, grad_norm=100.298, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.074e-05, train_time=1.565 +[gpua006:0/64] 2024-02-12 03:16:57,360 (trainer:756) INFO: 32epoch:train:14701-14800batch: iter_time=8.305e-05, forward_time=0.144, loss_ctc=83.934, loss_interctc_layer6=95.157, loss_interctc_layer12=78.933, loss_interctc_layer15=72.402, loss_interctc_layer21=86.732, loss=83.432, backward_time=0.207, grad_norm=112.687, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.074e-05, train_time=1.445 +[gpua006:0/64] 2024-02-12 03:18:54,370 (trainer:756) INFO: 32epoch:train:14801-14900batch: iter_time=8.185e-05, forward_time=0.143, loss_ctc=69.808, loss_interctc_layer6=81.950, loss_interctc_layer12=67.517, loss_interctc_layer15=61.816, loss_interctc_layer21=72.443, loss=70.707, backward_time=0.209, grad_norm=85.164, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.073e-05, train_time=1.173 +[gpua006:0/64] 2024-02-12 03:20:47,466 (trainer:756) INFO: 32epoch:train:14901-15000batch: iter_time=8.091e-05, forward_time=0.143, loss_ctc=86.917, loss_interctc_layer6=99.435, loss_interctc_layer12=83.457, loss_interctc_layer15=77.240, loss_interctc_layer21=89.593, loss=87.328, backward_time=0.209, grad_norm=119.162, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.072e-05, train_time=1.131 +[gpua006:0/64] 2024-02-12 03:51:16,699 (trainer:355) INFO: 32epoch results: [train] iter_time=0.254, forward_time=0.157, loss_ctc=78.314, loss_interctc_layer6=83.926, loss_interctc_layer12=69.607, loss_interctc_layer15=63.830, loss_interctc_layer21=81.108, loss=75.357, backward_time=0.213, grad_norm=82.799, clip=100.000, loss_scale=1.991e+31, optim_step_time=0.137, optim0_lr0=7.128e-05, train_time=1.688, time=7 hours, 2 minutes and 28.85 seconds, total_count=480000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=41.895, cer_ctc=0.193, loss_interctc_layer6=46.993, cer_interctc_layer6=0.210, loss_interctc_layer12=34.486, cer_interctc_layer12=0.144, loss_interctc_layer15=30.378, cer_interctc_layer15=0.121, loss_interctc_layer21=44.262, cer_interctc_layer21=0.206, loss=39.603, time=30 minutes and 3.29 seconds, total_count=149472, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-12 03:51:44,148 (trainer:410) INFO: The best model has been updated: valid.cer_ctc, valid.loss_ctc, valid.total_count +[gpua006:0/64] 2024-02-12 03:51:44,205 (trainer:289) INFO: 33/45epoch started. Estimated time to finish: 4 days, 2 hours and 8 minutes +[gpua006:0/64] 2024-02-12 03:51:44,223 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-12 03:52:02,345 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 03:52:05,779 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 03:52:05,779 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-12 03:52:05,782 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 03:58:16,308 (trainer:756) INFO: 33epoch:train:1-100batch: iter_time=2.591, forward_time=0.164, loss_ctc=80.560, loss_interctc_layer6=82.840, loss_interctc_layer12=68.835, loss_interctc_layer15=63.181, loss_interctc_layer21=83.638, loss=75.811, backward_time=0.216, grad_norm=78.547, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.071e-05, train_time=3.921 +[gpua006:0/64] 2024-02-12 04:00:00,153 (trainer:756) INFO: 33epoch:train:101-200batch: iter_time=9.189e-05, forward_time=0.143, loss_ctc=65.704, loss_interctc_layer6=74.835, loss_interctc_layer12=61.595, loss_interctc_layer15=56.284, loss_interctc_layer21=68.124, loss=65.309, backward_time=0.209, grad_norm=64.584, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.071e-05, train_time=1.038 +[gpua006:0/64] 2024-02-12 04:01:47,308 (trainer:756) INFO: 33epoch:train:201-300batch: iter_time=8.682e-05, forward_time=0.142, loss_ctc=74.076, loss_interctc_layer6=77.753, loss_interctc_layer12=64.802, loss_interctc_layer15=59.617, loss_interctc_layer21=76.722, loss=70.594, backward_time=0.208, grad_norm=112.022, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.070e-05, train_time=1.071 +[gpua006:0/64] 2024-02-12 04:03:35,759 (trainer:756) INFO: 33epoch:train:301-400batch: iter_time=8.810e-05, forward_time=0.154, loss_ctc=86.549, loss_interctc_layer6=91.212, loss_interctc_layer12=76.804, loss_interctc_layer15=71.179, loss_interctc_layer21=89.638, loss=83.076, backward_time=0.209, grad_norm=82.918, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.069e-05, train_time=1.083 +[gpua006:0/64] 2024-02-12 04:05:44,554 (trainer:756) INFO: 33epoch:train:401-500batch: iter_time=8.948e-05, forward_time=0.194, loss_ctc=88.442, loss_interctc_layer6=86.654, loss_interctc_layer12=72.137, loss_interctc_layer15=66.250, loss_interctc_layer21=91.624, loss=81.022, backward_time=0.224, grad_norm=100.525, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.069e-05, train_time=1.288 +[gpua006:0/64] 2024-02-12 04:06:38,399 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 04:07:24,275 (trainer:756) INFO: 33epoch:train:501-600batch: iter_time=8.949e-05, forward_time=0.147, loss_ctc=82.367, loss_interctc_layer6=85.172, loss_interctc_layer12=70.799, loss_interctc_layer15=64.973, loss_interctc_layer21=85.402, loss=77.743, backward_time=0.213, grad_norm=88.585, clip=100.000, loss_scale=7.836e+30, optim_step_time=0.137, optim0_lr0=7.068e-05, train_time=0.997 +[gpua006:0/64] 2024-02-12 04:09:26,821 (trainer:756) INFO: 33epoch:train:601-700batch: iter_time=9.185e-05, forward_time=0.162, loss_ctc=79.034, loss_interctc_layer6=78.724, loss_interctc_layer12=65.085, loss_interctc_layer15=59.416, loss_interctc_layer21=82.032, loss=72.858, backward_time=0.221, grad_norm=85.922, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.067e-05, train_time=1.225 +[gpua006:0/64] 2024-02-12 04:11:22,082 (trainer:756) INFO: 33epoch:train:701-800batch: iter_time=8.913e-05, forward_time=0.168, loss_ctc=87.031, loss_interctc_layer6=86.200, loss_interctc_layer12=71.474, loss_interctc_layer15=65.462, loss_interctc_layer21=90.260, loss=80.085, backward_time=0.214, grad_norm=75.522, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.066e-05, train_time=1.152 +[gpua006:0/64] 2024-02-12 04:13:25,473 (trainer:756) INFO: 33epoch:train:801-900batch: iter_time=8.708e-05, forward_time=0.144, loss_ctc=58.313, loss_interctc_layer6=76.027, loss_interctc_layer12=63.160, loss_interctc_layer15=58.090, loss_interctc_layer21=59.966, loss=63.111, backward_time=0.208, grad_norm=61.259, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.066e-05, train_time=1.234 +[gpua006:0/64] 2024-02-12 04:15:40,777 (trainer:756) INFO: 33epoch:train:901-1000batch: iter_time=9.023e-05, forward_time=0.150, loss_ctc=75.051, loss_interctc_layer6=78.419, loss_interctc_layer12=65.229, loss_interctc_layer15=59.850, loss_interctc_layer21=77.657, loss=71.241, backward_time=0.211, grad_norm=67.403, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.065e-05, train_time=1.351 +[gpua006:0/64] 2024-02-12 04:17:55,745 (trainer:756) INFO: 33epoch:train:1001-1100batch: iter_time=9.981e-05, forward_time=0.175, loss_ctc=73.028, loss_interctc_layer6=79.289, loss_interctc_layer12=65.189, loss_interctc_layer15=59.645, loss_interctc_layer21=75.548, loss=70.540, backward_time=0.216, grad_norm=76.846, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=7.064e-05, train_time=1.351 +[gpua006:0/64] 2024-02-12 04:20:04,277 (trainer:756) INFO: 33epoch:train:1101-1200batch: iter_time=1.051e-04, forward_time=0.165, loss_ctc=78.752, loss_interctc_layer6=83.847, loss_interctc_layer12=69.876, loss_interctc_layer15=64.217, loss_interctc_layer21=81.734, loss=75.685, backward_time=0.214, grad_norm=73.933, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.063e-05, train_time=1.285 +[gpua006:0/64] 2024-02-12 04:21:23,023 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-12 04:21:42,036 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 04:21:45,731 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 04:21:45,731 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-12 04:21:45,735 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 04:26:59,664 (trainer:756) INFO: 33epoch:train:1201-1300batch: iter_time=2.960, forward_time=0.225, loss_ctc=80.327, loss_interctc_layer6=86.684, loss_interctc_layer12=72.188, loss_interctc_layer15=66.370, loss_interctc_layer21=83.165, loss=77.747, backward_time=0.228, grad_norm=92.186, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=7.063e-05, train_time=4.154 +[gpua006:0/64] 2024-02-12 04:28:37,684 (trainer:756) INFO: 33epoch:train:1301-1400batch: iter_time=8.564e-05, forward_time=0.145, loss_ctc=77.565, loss_interctc_layer6=87.097, loss_interctc_layer12=72.594, loss_interctc_layer15=66.812, loss_interctc_layer21=80.418, loss=76.897, backward_time=0.210, grad_norm=61.514, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.062e-05, train_time=0.980 +[gpua006:0/64] 2024-02-12 04:30:16,085 (trainer:756) INFO: 33epoch:train:1401-1500batch: iter_time=8.306e-05, forward_time=0.143, loss_ctc=60.798, loss_interctc_layer6=67.646, loss_interctc_layer12=55.642, loss_interctc_layer15=50.704, loss_interctc_layer21=63.282, loss=59.615, backward_time=0.209, grad_norm=54.863, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.061e-05, train_time=0.984 +[gpua006:0/64] 2024-02-12 04:32:17,535 (trainer:756) INFO: 33epoch:train:1501-1600batch: iter_time=9.118e-05, forward_time=0.279, loss_ctc=80.985, loss_interctc_layer6=92.656, loss_interctc_layer12=77.538, loss_interctc_layer15=71.547, loss_interctc_layer21=83.573, loss=81.260, backward_time=0.233, grad_norm=85.712, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.140, optim0_lr0=7.060e-05, train_time=1.214 +[gpua006:0/64] 2024-02-12 04:34:58,378 (trainer:756) INFO: 33epoch:train:1601-1700batch: iter_time=8.301e-05, forward_time=0.145, loss_ctc=71.308, loss_interctc_layer6=83.587, loss_interctc_layer12=70.249, loss_interctc_layer15=64.399, loss_interctc_layer21=73.707, loss=72.650, backward_time=0.207, grad_norm=75.520, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.060e-05, train_time=1.609 +[gpua006:0/64] 2024-02-12 04:37:12,164 (trainer:756) INFO: 33epoch:train:1701-1800batch: iter_time=8.455e-05, forward_time=0.146, loss_ctc=82.715, loss_interctc_layer6=86.467, loss_interctc_layer12=71.803, loss_interctc_layer15=65.960, loss_interctc_layer21=85.867, loss=78.562, backward_time=0.207, grad_norm=67.399, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.059e-05, train_time=1.338 +[gpua006:0/64] 2024-02-12 04:39:11,709 (trainer:756) INFO: 33epoch:train:1801-1900batch: iter_time=8.793e-05, forward_time=0.281, loss_ctc=80.916, loss_interctc_layer6=81.706, loss_interctc_layer12=67.623, loss_interctc_layer15=61.950, loss_interctc_layer21=84.168, loss=75.273, backward_time=0.238, grad_norm=76.448, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.140, optim0_lr0=7.058e-05, train_time=1.195 +[gpua006:0/64] 2024-02-12 04:41:09,914 (trainer:756) INFO: 33epoch:train:1901-2000batch: iter_time=8.450e-05, forward_time=0.145, loss_ctc=81.914, loss_interctc_layer6=84.382, loss_interctc_layer12=69.218, loss_interctc_layer15=63.023, loss_interctc_layer21=85.069, loss=76.721, backward_time=0.207, grad_norm=99.074, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.058e-05, train_time=1.182 +[gpua006:0/64] 2024-02-12 04:43:25,301 (trainer:756) INFO: 33epoch:train:2001-2100batch: iter_time=8.216e-05, forward_time=0.144, loss_ctc=65.095, loss_interctc_layer6=80.086, loss_interctc_layer12=66.404, loss_interctc_layer15=60.868, loss_interctc_layer21=67.287, loss=67.948, backward_time=0.207, grad_norm=72.186, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=7.057e-05, train_time=1.354 +[gpua006:0/64] 2024-02-12 04:45:53,627 (trainer:756) INFO: 33epoch:train:2101-2200batch: iter_time=9.492e-05, forward_time=0.221, loss_ctc=65.169, loss_interctc_layer6=74.295, loss_interctc_layer12=61.690, loss_interctc_layer15=56.612, loss_interctc_layer21=67.336, loss=65.020, backward_time=0.245, grad_norm=61.289, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.140, optim0_lr0=7.056e-05, train_time=1.482 +[gpua006:0/64] 2024-02-12 04:47:57,481 (trainer:756) INFO: 33epoch:train:2201-2300batch: iter_time=7.752e-05, forward_time=0.143, loss_ctc=78.404, loss_interctc_layer6=87.644, loss_interctc_layer12=73.015, loss_interctc_layer15=67.199, loss_interctc_layer21=81.003, loss=77.453, backward_time=0.209, grad_norm=74.516, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.055e-05, train_time=1.238 +[gpua006:0/64] 2024-02-12 04:49:40,382 (trainer:756) INFO: 33epoch:train:2301-2400batch: iter_time=7.452e-05, forward_time=0.142, loss_ctc=65.892, loss_interctc_layer6=73.530, loss_interctc_layer12=60.508, loss_interctc_layer15=55.250, loss_interctc_layer21=68.310, loss=64.698, backward_time=0.209, grad_norm=68.506, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.137, optim0_lr0=7.055e-05, train_time=1.030 +[gpua006:0/64] 2024-02-12 04:51:24,383 (trainer:756) INFO: 33epoch:train:2401-2500batch: iter_time=7.757e-05, forward_time=0.143, loss_ctc=78.670, loss_interctc_layer6=91.749, loss_interctc_layer12=76.759, loss_interctc_layer15=70.704, loss_interctc_layer21=81.358, loss=79.848, backward_time=0.209, grad_norm=82.706, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.136, optim0_lr0=7.054e-05, train_time=1.040 +[gpua006:0/64] 2024-02-12 04:51:42,648 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-12 04:52:01,579 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 04:52:05,051 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 04:52:05,051 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-12 04:52:05,054 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 04:57:28,521 (trainer:756) INFO: 33epoch:train:2501-2600batch: iter_time=2.505, forward_time=0.145, loss_ctc=79.588, loss_interctc_layer6=81.700, loss_interctc_layer12=67.607, loss_interctc_layer15=61.916, loss_interctc_layer21=82.591, loss=74.680, backward_time=0.213, grad_norm=83.401, clip=100.000, loss_scale=7.352e+30, optim_step_time=0.138, optim0_lr0=7.053e-05, train_time=3.641 +[gpua006:0/64] 2024-02-12 04:59:37,384 (trainer:756) INFO: 33epoch:train:2601-2700batch: iter_time=8.297e-05, forward_time=0.243, loss_ctc=65.965, loss_interctc_layer6=74.858, loss_interctc_layer12=61.623, loss_interctc_layer15=56.267, loss_interctc_layer21=68.343, loss=65.411, backward_time=0.245, grad_norm=57.182, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.052e-05, train_time=1.289 +[gpua006:0/64] 2024-02-12 05:01:18,096 (trainer:756) INFO: 33epoch:train:2701-2800batch: iter_time=8.494e-05, forward_time=0.144, loss_ctc=72.693, loss_interctc_layer6=77.016, loss_interctc_layer12=64.168, loss_interctc_layer15=59.087, loss_interctc_layer21=75.144, loss=69.622, backward_time=0.209, grad_norm=84.234, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=7.052e-05, train_time=1.006 +[gpua006:0/64] 2024-02-12 05:03:15,339 (trainer:756) INFO: 33epoch:train:2801-2900batch: iter_time=8.575e-05, forward_time=0.144, loss_ctc=84.818, loss_interctc_layer6=90.631, loss_interctc_layer12=75.747, loss_interctc_layer15=69.537, loss_interctc_layer21=87.695, loss=81.685, backward_time=0.208, grad_norm=76.659, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.051e-05, train_time=1.172 +[gpua006:0/64] 2024-02-12 05:05:53,028 (trainer:756) INFO: 33epoch:train:2901-3000batch: iter_time=8.700e-05, forward_time=0.146, loss_ctc=87.233, loss_interctc_layer6=86.627, loss_interctc_layer12=71.995, loss_interctc_layer15=65.987, loss_interctc_layer21=90.546, loss=80.478, backward_time=0.208, grad_norm=89.502, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.050e-05, train_time=1.578 +[gpua006:0/64] 2024-02-12 05:07:50,635 (trainer:756) INFO: 33epoch:train:3001-3100batch: iter_time=8.630e-05, forward_time=0.144, loss_ctc=80.305, loss_interctc_layer6=84.070, loss_interctc_layer12=69.601, loss_interctc_layer15=63.838, loss_interctc_layer21=83.230, loss=76.209, backward_time=0.209, grad_norm=76.504, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.050e-05, train_time=1.176 +[gpua006:0/64] 2024-02-12 05:09:26,680 (trainer:756) INFO: 33epoch:train:3101-3200batch: iter_time=7.901e-05, forward_time=0.144, loss_ctc=79.931, loss_interctc_layer6=78.798, loss_interctc_layer12=65.011, loss_interctc_layer15=59.392, loss_interctc_layer21=82.873, loss=73.201, backward_time=0.211, grad_norm=64.371, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.049e-05, train_time=0.960 +[gpua006:0/64] 2024-02-12 05:11:18,256 (trainer:756) INFO: 33epoch:train:3201-3300batch: iter_time=8.344e-05, forward_time=0.219, loss_ctc=85.104, loss_interctc_layer6=84.781, loss_interctc_layer12=69.718, loss_interctc_layer15=63.604, loss_interctc_layer21=88.464, loss=78.334, backward_time=0.238, grad_norm=81.715, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.048e-05, train_time=1.115 +[gpua006:0/64] 2024-02-12 05:13:22,595 (trainer:756) INFO: 33epoch:train:3301-3400batch: iter_time=8.264e-05, forward_time=0.150, loss_ctc=57.822, loss_interctc_layer6=75.968, loss_interctc_layer12=63.317, loss_interctc_layer15=58.211, loss_interctc_layer21=59.441, loss=62.952, backward_time=0.209, grad_norm=126.396, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.047e-05, train_time=1.243 +[gpua006:0/64] 2024-02-12 05:15:51,965 (trainer:756) INFO: 33epoch:train:3401-3500batch: iter_time=8.469e-05, forward_time=0.144, loss_ctc=75.012, loss_interctc_layer6=78.239, loss_interctc_layer12=64.963, loss_interctc_layer15=59.518, loss_interctc_layer21=77.569, loss=71.060, backward_time=0.208, grad_norm=70.440, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.047e-05, train_time=1.493 +[gpua006:0/64] 2024-02-12 05:17:38,009 (trainer:756) INFO: 33epoch:train:3501-3600batch: iter_time=8.602e-05, forward_time=0.143, loss_ctc=72.878, loss_interctc_layer6=79.028, loss_interctc_layer12=65.034, loss_interctc_layer15=59.352, loss_interctc_layer21=75.435, loss=70.345, backward_time=0.209, grad_norm=114.818, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.046e-05, train_time=1.061 +[gpua006:0/64] 2024-02-12 05:19:44,931 (trainer:756) INFO: 33epoch:train:3601-3700batch: iter_time=8.426e-05, forward_time=0.143, loss_ctc=77.500, loss_interctc_layer6=82.783, loss_interctc_layer12=68.826, loss_interctc_layer15=63.153, loss_interctc_layer21=80.323, loss=74.517, backward_time=0.208, grad_norm=78.827, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=7.045e-05, train_time=1.268 +[gpua006:0/64] 2024-02-12 05:20:53,878 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-12 05:21:12,707 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 05:21:16,123 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 05:21:16,123 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-12 05:21:16,126 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 05:26:33,823 (trainer:756) INFO: 33epoch:train:3701-3800batch: iter_time=2.974, forward_time=0.166, loss_ctc=80.473, loss_interctc_layer6=86.415, loss_interctc_layer12=71.950, loss_interctc_layer15=66.244, loss_interctc_layer21=83.546, loss=77.726, backward_time=0.218, grad_norm=78.721, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=7.044e-05, train_time=4.089 +[gpua006:0/64] 2024-02-12 05:28:16,170 (trainer:756) INFO: 33epoch:train:3801-3900batch: iter_time=8.352e-05, forward_time=0.150, loss_ctc=77.107, loss_interctc_layer6=86.501, loss_interctc_layer12=71.927, loss_interctc_layer15=66.122, loss_interctc_layer21=79.951, loss=76.322, backward_time=0.215, grad_norm=64.907, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.044e-05, train_time=1.023 +[gpua006:0/64] 2024-02-12 05:29:53,100 (trainer:756) INFO: 33epoch:train:3901-4000batch: iter_time=8.499e-05, forward_time=0.152, loss_ctc=60.461, loss_interctc_layer6=67.314, loss_interctc_layer12=55.287, loss_interctc_layer15=50.372, loss_interctc_layer21=62.907, loss=59.268, backward_time=0.211, grad_norm=78.273, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.043e-05, train_time=0.969 +[gpua006:0/64] 2024-02-12 05:32:00,445 (trainer:756) INFO: 33epoch:train:4001-4100batch: iter_time=8.713e-05, forward_time=0.145, loss_ctc=80.267, loss_interctc_layer6=92.208, loss_interctc_layer12=76.756, loss_interctc_layer15=70.676, loss_interctc_layer21=83.116, loss=80.605, backward_time=0.210, grad_norm=112.821, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.042e-05, train_time=1.273 +[gpua006:0/64] 2024-02-12 05:33:45,076 (trainer:756) INFO: 33epoch:train:4101-4200batch: iter_time=8.320e-05, forward_time=0.145, loss_ctc=70.371, loss_interctc_layer6=82.204, loss_interctc_layer12=68.611, loss_interctc_layer15=62.998, loss_interctc_layer21=72.753, loss=71.387, backward_time=0.211, grad_norm=64.312, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.041e-05, train_time=1.046 +[gpua006:0/64] 2024-02-12 05:35:54,614 (trainer:756) INFO: 33epoch:train:4201-4300batch: iter_time=8.519e-05, forward_time=0.145, loss_ctc=81.720, loss_interctc_layer6=86.095, loss_interctc_layer12=71.153, loss_interctc_layer15=65.111, loss_interctc_layer21=84.750, loss=77.766, backward_time=0.210, grad_norm=72.952, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.041e-05, train_time=1.295 +[gpua006:0/64] 2024-02-12 05:38:29,085 (trainer:756) INFO: 33epoch:train:4301-4400batch: iter_time=8.390e-05, forward_time=0.144, loss_ctc=80.184, loss_interctc_layer6=80.942, loss_interctc_layer12=66.781, loss_interctc_layer15=61.120, loss_interctc_layer21=83.393, loss=74.484, backward_time=0.209, grad_norm=75.633, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.040e-05, train_time=1.545 +[gpua006:0/64] 2024-02-12 05:40:26,585 (trainer:756) INFO: 33epoch:train:4401-4500batch: iter_time=8.655e-05, forward_time=0.144, loss_ctc=82.146, loss_interctc_layer6=84.366, loss_interctc_layer12=69.294, loss_interctc_layer15=63.115, loss_interctc_layer21=85.385, loss=76.861, backward_time=0.211, grad_norm=68.695, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=7.039e-05, train_time=1.173 +[gpua006:0/64] 2024-02-12 05:42:31,785 (trainer:756) INFO: 33epoch:train:4501-4600batch: iter_time=8.447e-05, forward_time=0.180, loss_ctc=63.635, loss_interctc_layer6=78.827, loss_interctc_layer12=65.147, loss_interctc_layer15=59.667, loss_interctc_layer21=65.847, loss=66.625, backward_time=0.269, grad_norm=68.678, clip=100.000, loss_scale=1.470e+31, optim_step_time=0.142, optim0_lr0=7.039e-05, train_time=1.253 +[gpua006:0/64] 2024-02-12 05:44:24,500 (trainer:756) INFO: 33epoch:train:4601-4700batch: iter_time=8.324e-05, forward_time=0.152, loss_ctc=65.096, loss_interctc_layer6=73.796, loss_interctc_layer12=61.192, loss_interctc_layer15=56.128, loss_interctc_layer21=67.238, loss=64.690, backward_time=0.215, grad_norm=64.034, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.038e-05, train_time=1.127 +[gpua006:0/64] 2024-02-12 05:47:00,312 (trainer:756) INFO: 33epoch:train:4701-4800batch: iter_time=8.217e-05, forward_time=0.148, loss_ctc=78.164, loss_interctc_layer6=87.187, loss_interctc_layer12=72.467, loss_interctc_layer15=66.569, loss_interctc_layer21=80.730, loss=77.023, backward_time=0.208, grad_norm=139.000, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.037e-05, train_time=1.558 +[gpua006:0/64] 2024-02-12 05:48:49,990 (trainer:756) INFO: 33epoch:train:4801-4900batch: iter_time=8.411e-05, forward_time=0.143, loss_ctc=65.093, loss_interctc_layer6=72.915, loss_interctc_layer12=59.934, loss_interctc_layer15=54.630, loss_interctc_layer21=67.644, loss=64.043, backward_time=0.209, grad_norm=65.596, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.036e-05, train_time=1.097 +[gpua006:0/64] 2024-02-12 05:50:45,425 (trainer:756) INFO: 33epoch:train:4901-5000batch: iter_time=8.246e-05, forward_time=0.144, loss_ctc=77.299, loss_interctc_layer6=89.670, loss_interctc_layer12=74.656, loss_interctc_layer15=68.883, loss_interctc_layer21=79.725, loss=78.047, backward_time=0.209, grad_norm=84.874, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.036e-05, train_time=1.154 +[gpua006:0/64] 2024-02-12 05:51:02,799 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-12 05:51:21,535 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 05:51:24,997 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 05:51:24,997 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-12 05:51:25,001 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 05:57:43,293 (trainer:756) INFO: 33epoch:train:5001-5100batch: iter_time=2.647, forward_time=0.180, loss_ctc=78.914, loss_interctc_layer6=81.073, loss_interctc_layer12=67.147, loss_interctc_layer15=61.535, loss_interctc_layer21=81.954, loss=74.124, backward_time=0.231, grad_norm=72.548, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.035e-05, train_time=4.178 +[gpua006:0/64] 2024-02-12 05:59:35,364 (trainer:756) INFO: 33epoch:train:5101-5200batch: iter_time=8.549e-05, forward_time=0.144, loss_ctc=65.614, loss_interctc_layer6=74.791, loss_interctc_layer12=61.455, loss_interctc_layer15=56.069, loss_interctc_layer21=68.041, loss=65.194, backward_time=0.212, grad_norm=51.603, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.034e-05, train_time=1.121 +[gpua006:0/64] 2024-02-12 06:01:56,360 (trainer:756) INFO: 33epoch:train:5201-5300batch: iter_time=8.072e-05, forward_time=0.210, loss_ctc=72.719, loss_interctc_layer6=76.524, loss_interctc_layer12=63.531, loss_interctc_layer15=58.370, loss_interctc_layer21=75.398, loss=69.309, backward_time=0.256, grad_norm=77.809, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.146, optim0_lr0=7.034e-05, train_time=1.410 +[gpua006:0/64] 2024-02-12 06:03:46,937 (trainer:756) INFO: 33epoch:train:5301-5400batch: iter_time=9.249e-05, forward_time=0.148, loss_ctc=84.322, loss_interctc_layer6=90.616, loss_interctc_layer12=75.188, loss_interctc_layer15=69.225, loss_interctc_layer21=87.169, loss=81.304, backward_time=0.208, grad_norm=84.551, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.033e-05, train_time=1.106 +[gpua006:0/64] 2024-02-12 06:05:45,616 (trainer:756) INFO: 33epoch:train:5401-5500batch: iter_time=8.518e-05, forward_time=0.144, loss_ctc=86.543, loss_interctc_layer6=85.371, loss_interctc_layer12=70.677, loss_interctc_layer15=64.667, loss_interctc_layer21=89.839, loss=79.419, backward_time=0.209, grad_norm=79.028, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.032e-05, train_time=1.186 +[gpua006:0/64] 2024-02-12 06:07:52,290 (trainer:756) INFO: 33epoch:train:5501-5600batch: iter_time=8.673e-05, forward_time=0.148, loss_ctc=80.535, loss_interctc_layer6=83.147, loss_interctc_layer12=68.696, loss_interctc_layer15=62.922, loss_interctc_layer21=83.466, loss=75.753, backward_time=0.209, grad_norm=72.121, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.031e-05, train_time=1.267 +[gpua006:0/64] 2024-02-12 06:09:59,256 (trainer:756) INFO: 33epoch:train:5601-5700batch: iter_time=8.960e-05, forward_time=0.259, loss_ctc=78.421, loss_interctc_layer6=78.054, loss_interctc_layer12=64.336, loss_interctc_layer15=58.756, loss_interctc_layer21=81.368, loss=72.187, backward_time=0.255, grad_norm=62.027, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.031e-05, train_time=1.269 +[gpua006:0/64] 2024-02-12 06:12:22,685 (trainer:756) INFO: 33epoch:train:5701-5800batch: iter_time=8.398e-05, forward_time=0.168, loss_ctc=82.962, loss_interctc_layer6=83.268, loss_interctc_layer12=68.107, loss_interctc_layer15=62.289, loss_interctc_layer21=86.149, loss=76.555, backward_time=0.208, grad_norm=70.293, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.030e-05, train_time=1.434 +[gpua006:0/64] 2024-02-12 06:14:35,197 (trainer:756) INFO: 33epoch:train:5801-5900batch: iter_time=8.521e-05, forward_time=0.144, loss_ctc=58.337, loss_interctc_layer6=75.944, loss_interctc_layer12=63.186, loss_interctc_layer15=58.081, loss_interctc_layer21=59.970, loss=63.104, backward_time=0.208, grad_norm=98.427, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.029e-05, train_time=1.325 +[gpua006:0/64] 2024-02-12 06:16:46,777 (trainer:756) INFO: 33epoch:train:5901-6000batch: iter_time=9.495e-05, forward_time=0.146, loss_ctc=73.512, loss_interctc_layer6=77.332, loss_interctc_layer12=63.982, loss_interctc_layer15=58.618, loss_interctc_layer21=75.937, loss=69.876, backward_time=0.209, grad_norm=71.035, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.028e-05, train_time=1.316 +[gpua006:0/64] 2024-02-12 06:19:10,686 (trainer:756) INFO: 33epoch:train:6001-6100batch: iter_time=2.639e-04, forward_time=0.203, loss_ctc=73.028, loss_interctc_layer6=79.028, loss_interctc_layer12=64.908, loss_interctc_layer15=59.207, loss_interctc_layer21=75.341, loss=70.302, backward_time=0.286, grad_norm=74.376, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.028e-05, train_time=1.438 +[gpua006:0/64] 2024-02-12 06:21:01,935 (trainer:756) INFO: 33epoch:train:6101-6200batch: iter_time=8.502e-05, forward_time=0.143, loss_ctc=77.521, loss_interctc_layer6=82.606, loss_interctc_layer12=68.425, loss_interctc_layer15=62.758, loss_interctc_layer21=80.529, loss=74.368, backward_time=0.209, grad_norm=86.482, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.027e-05, train_time=1.113 +[gpua006:0/64] 2024-02-12 06:22:21,341 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-12 06:22:40,463 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 06:22:43,980 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 06:22:43,980 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-12 06:22:43,983 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 06:27:36,181 (trainer:756) INFO: 33epoch:train:6201-6300batch: iter_time=2.714, forward_time=0.145, loss_ctc=80.072, loss_interctc_layer6=86.206, loss_interctc_layer12=71.614, loss_interctc_layer15=65.830, loss_interctc_layer21=82.759, loss=77.296, backward_time=0.208, grad_norm=86.070, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.026e-05, train_time=3.942 +[gpua006:0/64] 2024-02-12 06:29:30,001 (trainer:756) INFO: 33epoch:train:6301-6400batch: iter_time=8.894e-05, forward_time=0.144, loss_ctc=76.432, loss_interctc_layer6=86.182, loss_interctc_layer12=71.518, loss_interctc_layer15=65.623, loss_interctc_layer21=79.227, loss=75.797, backward_time=0.209, grad_norm=80.896, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.026e-05, train_time=1.138 +[gpua006:0/64] 2024-02-12 06:31:15,585 (trainer:756) INFO: 33epoch:train:6401-6500batch: iter_time=8.351e-05, forward_time=0.142, loss_ctc=60.371, loss_interctc_layer6=67.445, loss_interctc_layer12=55.511, loss_interctc_layer15=50.548, loss_interctc_layer21=62.744, loss=59.324, backward_time=0.208, grad_norm=65.916, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.025e-05, train_time=1.056 +[gpua006:0/64] 2024-02-12 06:33:04,931 (trainer:756) INFO: 33epoch:train:6501-6600batch: iter_time=8.542e-05, forward_time=0.144, loss_ctc=80.059, loss_interctc_layer6=91.371, loss_interctc_layer12=76.293, loss_interctc_layer15=70.260, loss_interctc_layer21=82.663, loss=80.129, backward_time=0.209, grad_norm=89.733, clip=100.000, loss_scale=2.941e+31, optim_step_time=0.138, optim0_lr0=7.024e-05, train_time=1.093 +[gpua006:0/64] 2024-02-12 06:34:58,296 (trainer:756) INFO: 33epoch:train:6601-6700batch: iter_time=8.884e-05, forward_time=0.153, loss_ctc=69.903, loss_interctc_layer6=82.603, loss_interctc_layer12=68.738, loss_interctc_layer15=63.030, loss_interctc_layer21=72.022, loss=71.259, backward_time=0.210, grad_norm=81.069, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=7.023e-05, train_time=1.133 +[gpua006:0/64] 2024-02-12 06:37:28,703 (trainer:756) INFO: 33epoch:train:6701-6800batch: iter_time=8.673e-05, forward_time=0.230, loss_ctc=80.467, loss_interctc_layer6=85.157, loss_interctc_layer12=70.307, loss_interctc_layer15=64.444, loss_interctc_layer21=83.342, loss=76.743, backward_time=0.238, grad_norm=82.608, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=7.023e-05, train_time=1.503 +[gpua006:0/64] 2024-02-12 06:39:21,169 (trainer:756) INFO: 33epoch:train:6801-6900batch: iter_time=8.598e-05, forward_time=0.151, loss_ctc=80.035, loss_interctc_layer6=81.018, loss_interctc_layer12=66.920, loss_interctc_layer15=61.297, loss_interctc_layer21=83.175, loss=74.489, backward_time=0.209, grad_norm=65.666, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.022e-05, train_time=1.124 +[gpua006:0/64] 2024-02-12 06:41:43,833 (trainer:756) INFO: 33epoch:train:6901-7000batch: iter_time=8.848e-05, forward_time=0.145, loss_ctc=81.461, loss_interctc_layer6=83.764, loss_interctc_layer12=68.532, loss_interctc_layer15=62.205, loss_interctc_layer21=84.630, loss=76.118, backward_time=0.208, grad_norm=86.750, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.021e-05, train_time=1.427 +[gpua006:0/64] 2024-02-12 06:43:43,021 (trainer:756) INFO: 33epoch:train:7001-7100batch: iter_time=9.195e-05, forward_time=0.143, loss_ctc=62.432, loss_interctc_layer6=77.459, loss_interctc_layer12=63.690, loss_interctc_layer15=58.372, loss_interctc_layer21=64.686, loss=65.328, backward_time=0.209, grad_norm=83.593, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.020e-05, train_time=1.192 +[gpua006:0/64] 2024-02-12 06:45:47,034 (trainer:756) INFO: 33epoch:train:7101-7200batch: iter_time=8.521e-05, forward_time=0.143, loss_ctc=64.137, loss_interctc_layer6=73.680, loss_interctc_layer12=61.125, loss_interctc_layer15=56.157, loss_interctc_layer21=66.301, loss=64.280, backward_time=0.208, grad_norm=65.927, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.020e-05, train_time=1.240 +[gpua006:0/64] 2024-02-12 06:47:57,031 (trainer:756) INFO: 33epoch:train:7201-7300batch: iter_time=8.764e-05, forward_time=0.144, loss_ctc=77.892, loss_interctc_layer6=87.399, loss_interctc_layer12=72.562, loss_interctc_layer15=66.631, loss_interctc_layer21=80.554, loss=77.007, backward_time=0.208, grad_norm=87.257, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.019e-05, train_time=1.300 +[gpua006:0/64] 2024-02-12 06:49:57,976 (trainer:756) INFO: 33epoch:train:7301-7400batch: iter_time=8.660e-05, forward_time=0.143, loss_ctc=65.054, loss_interctc_layer6=72.679, loss_interctc_layer12=59.714, loss_interctc_layer15=54.452, loss_interctc_layer21=67.563, loss=63.892, backward_time=0.209, grad_norm=85.213, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.018e-05, train_time=1.209 +[gpua006:0/64] 2024-02-12 06:52:15,599 (trainer:756) INFO: 33epoch:train:7401-7500batch: iter_time=8.769e-05, forward_time=0.145, loss_ctc=77.221, loss_interctc_layer6=89.604, loss_interctc_layer12=74.831, loss_interctc_layer15=69.087, loss_interctc_layer21=79.544, loss=78.058, backward_time=0.208, grad_norm=85.200, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.018e-05, train_time=1.376 +[gpua006:0/64] 2024-02-12 06:52:35,628 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-12 06:52:55,073 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 06:52:58,522 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 06:52:58,523 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-12 06:52:58,526 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 06:58:53,195 (trainer:756) INFO: 33epoch:train:7501-7600batch: iter_time=2.859, forward_time=0.182, loss_ctc=72.687, loss_interctc_layer6=81.388, loss_interctc_layer12=67.299, loss_interctc_layer15=61.597, loss_interctc_layer21=75.414, loss=71.677, backward_time=0.219, grad_norm=70.907, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=7.017e-05, train_time=3.976 +[gpua006:0/64] 2024-02-12 07:00:28,798 (trainer:756) INFO: 33epoch:train:7601-7700batch: iter_time=8.862e-05, forward_time=0.144, loss_ctc=63.049, loss_interctc_layer6=74.205, loss_interctc_layer12=61.096, loss_interctc_layer15=55.705, loss_interctc_layer21=65.412, loss=63.893, backward_time=0.218, grad_norm=70.545, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=7.016e-05, train_time=0.954 +[gpua006:0/64] 2024-02-12 07:02:16,111 (trainer:756) INFO: 33epoch:train:7701-7800batch: iter_time=8.987e-05, forward_time=0.144, loss_ctc=68.414, loss_interctc_layer6=76.616, loss_interctc_layer12=63.593, loss_interctc_layer15=58.403, loss_interctc_layer21=71.013, loss=67.608, backward_time=0.210, grad_norm=83.843, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.015e-05, train_time=1.074 +[gpua006:0/64] 2024-02-12 07:04:58,291 (trainer:756) INFO: 33epoch:train:7801-7900batch: iter_time=8.777e-05, forward_time=0.146, loss_ctc=77.925, loss_interctc_layer6=89.553, loss_interctc_layer12=74.382, loss_interctc_layer15=68.378, loss_interctc_layer21=80.531, loss=78.154, backward_time=0.208, grad_norm=76.386, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.015e-05, train_time=1.622 +[gpua006:0/64] 2024-02-12 07:07:08,389 (trainer:756) INFO: 33epoch:train:7901-8000batch: iter_time=8.300e-05, forward_time=0.144, loss_ctc=81.707, loss_interctc_layer6=85.165, loss_interctc_layer12=70.555, loss_interctc_layer15=64.502, loss_interctc_layer21=84.688, loss=77.324, backward_time=0.209, grad_norm=64.993, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.014e-05, train_time=1.301 +[gpua006:0/64] 2024-02-12 07:09:09,974 (trainer:756) INFO: 33epoch:train:8001-8100batch: iter_time=9.197e-05, forward_time=0.144, loss_ctc=76.771, loss_interctc_layer6=83.307, loss_interctc_layer12=68.929, loss_interctc_layer15=63.132, loss_interctc_layer21=79.698, loss=74.368, backward_time=0.209, grad_norm=62.559, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.013e-05, train_time=1.216 +[gpua006:0/64] 2024-02-12 07:11:04,711 (trainer:756) INFO: 33epoch:train:8101-8200batch: iter_time=8.835e-05, forward_time=0.143, loss_ctc=73.431, loss_interctc_layer6=77.953, loss_interctc_layer12=64.203, loss_interctc_layer15=58.593, loss_interctc_layer21=76.240, loss=70.084, backward_time=0.209, grad_norm=60.170, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.013e-05, train_time=1.147 +[gpua006:0/64] 2024-02-12 07:13:12,910 (trainer:756) INFO: 33epoch:train:8201-8300batch: iter_time=8.763e-05, forward_time=0.143, loss_ctc=77.303, loss_interctc_layer6=82.715, loss_interctc_layer12=67.644, loss_interctc_layer15=61.710, loss_interctc_layer21=80.508, loss=73.976, backward_time=0.209, grad_norm=79.548, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.012e-05, train_time=1.282 +[gpua006:0/64] 2024-02-12 07:15:43,183 (trainer:756) INFO: 33epoch:train:8301-8400batch: iter_time=9.114e-05, forward_time=0.143, loss_ctc=56.792, loss_interctc_layer6=75.361, loss_interctc_layer12=62.682, loss_interctc_layer15=57.589, loss_interctc_layer21=58.392, loss=62.163, backward_time=0.207, grad_norm=93.516, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=7.011e-05, train_time=1.502 +[gpua006:0/64] 2024-02-12 07:17:59,807 (trainer:756) INFO: 33epoch:train:8401-8500batch: iter_time=8.946e-05, forward_time=0.172, loss_ctc=71.577, loss_interctc_layer6=77.587, loss_interctc_layer12=64.358, loss_interctc_layer15=59.005, loss_interctc_layer21=74.016, loss=69.309, backward_time=0.215, grad_norm=69.007, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=7.010e-05, train_time=1.366 +[gpua006:0/64] 2024-02-12 07:20:16,681 (trainer:756) INFO: 33epoch:train:8501-8600batch: iter_time=8.927e-05, forward_time=0.192, loss_ctc=68.692, loss_interctc_layer6=78.410, loss_interctc_layer12=64.338, loss_interctc_layer15=58.646, loss_interctc_layer21=71.076, loss=68.232, backward_time=0.217, grad_norm=78.890, clip=100.000, loss_scale=5.882e+31, optim_step_time=0.138, optim0_lr0=7.010e-05, train_time=1.369 +[gpua006:0/64] 2024-02-12 07:21:55,180 (trainer:756) INFO: 33epoch:train:8601-8700batch: iter_time=7.843e-05, forward_time=0.144, loss_ctc=72.580, loss_interctc_layer6=82.142, loss_interctc_layer12=68.067, loss_interctc_layer15=62.387, loss_interctc_layer21=75.277, loss=72.091, backward_time=0.209, grad_norm=63.337, clip=100.000, loss_scale=8.113e+31, optim_step_time=0.137, optim0_lr0=7.009e-05, train_time=0.984 +[gpua006:0/64] 2024-02-12 07:23:18,097 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-12 07:23:37,209 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 07:23:40,690 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 07:23:40,690 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-12 07:23:40,694 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 07:28:42,501 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 07:29:06,390 (trainer:756) INFO: 33epoch:train:8701-8800batch: iter_time=3.066, forward_time=0.174, loss_ctc=75.441, loss_interctc_layer6=85.524, loss_interctc_layer12=71.014, loss_interctc_layer15=65.189, loss_interctc_layer21=78.109, loss=75.055, backward_time=0.231, grad_norm=78.313, clip=100.000, loss_scale=7.212e+31, optim_step_time=0.138, optim0_lr0=7.008e-05, train_time=4.312 +[gpua006:0/64] 2024-02-12 07:30:18,791 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 07:30:55,881 (trainer:756) INFO: 33epoch:train:8801-8900batch: iter_time=7.921e-05, forward_time=0.156, loss_ctc=76.124, loss_interctc_layer6=85.703, loss_interctc_layer12=71.148, loss_interctc_layer15=65.239, loss_interctc_layer21=78.920, loss=75.427, backward_time=0.210, grad_norm=70.875, clip=100.000, loss_scale=3.503e+31, optim_step_time=0.137, optim0_lr0=7.008e-05, train_time=1.095 +[gpua006:0/64] 2024-02-12 07:32:38,098 (trainer:756) INFO: 33epoch:train:8901-9000batch: iter_time=7.952e-05, forward_time=0.159, loss_ctc=59.877, loss_interctc_layer6=66.915, loss_interctc_layer12=54.860, loss_interctc_layer15=49.918, loss_interctc_layer21=62.301, loss=58.774, backward_time=0.218, grad_norm=62.134, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.007e-05, train_time=1.022 +[gpua006:0/64] 2024-02-12 07:34:42,869 (trainer:756) INFO: 33epoch:train:9001-9100batch: iter_time=8.628e-05, forward_time=0.148, loss_ctc=79.891, loss_interctc_layer6=92.206, loss_interctc_layer12=77.028, loss_interctc_layer15=70.737, loss_interctc_layer21=82.685, loss=80.509, backward_time=0.208, grad_norm=83.978, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=7.006e-05, train_time=1.245 +[gpua006:0/64] 2024-02-12 07:36:48,593 (trainer:756) INFO: 33epoch:train:9101-9200batch: iter_time=8.615e-05, forward_time=0.223, loss_ctc=69.337, loss_interctc_layer6=81.756, loss_interctc_layer12=68.238, loss_interctc_layer15=62.774, loss_interctc_layer21=71.561, loss=70.733, backward_time=0.224, grad_norm=83.904, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=7.005e-05, train_time=1.258 +[gpua006:0/64] 2024-02-12 07:39:06,839 (trainer:756) INFO: 33epoch:train:9201-9300batch: iter_time=8.633e-05, forward_time=0.151, loss_ctc=81.344, loss_interctc_layer6=85.411, loss_interctc_layer12=70.504, loss_interctc_layer15=64.536, loss_interctc_layer21=84.236, loss=77.206, backward_time=0.216, grad_norm=80.902, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.005e-05, train_time=1.384 +[gpua006:0/64] 2024-02-12 07:41:08,912 (trainer:756) INFO: 33epoch:train:9301-9400batch: iter_time=8.475e-05, forward_time=0.144, loss_ctc=79.839, loss_interctc_layer6=80.743, loss_interctc_layer12=66.533, loss_interctc_layer15=60.846, loss_interctc_layer21=82.999, loss=74.192, backward_time=0.209, grad_norm=73.997, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=7.004e-05, train_time=1.221 +[gpua006:0/64] 2024-02-12 07:43:08,708 (trainer:756) INFO: 33epoch:train:9401-9500batch: iter_time=8.715e-05, forward_time=0.185, loss_ctc=81.611, loss_interctc_layer6=84.224, loss_interctc_layer12=68.915, loss_interctc_layer15=62.670, loss_interctc_layer21=84.736, loss=76.431, backward_time=0.216, grad_norm=145.671, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.003e-05, train_time=1.197 +[gpua006:0/64] 2024-02-12 07:45:24,323 (trainer:756) INFO: 33epoch:train:9501-9600batch: iter_time=8.486e-05, forward_time=0.170, loss_ctc=62.303, loss_interctc_layer6=77.151, loss_interctc_layer12=63.514, loss_interctc_layer15=58.089, loss_interctc_layer21=64.318, loss=65.075, backward_time=0.216, grad_norm=62.751, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.003e-05, train_time=1.356 +[gpua006:0/64] 2024-02-12 07:47:32,050 (trainer:756) INFO: 33epoch:train:9601-9700batch: iter_time=8.297e-05, forward_time=0.154, loss_ctc=64.044, loss_interctc_layer6=73.504, loss_interctc_layer12=60.948, loss_interctc_layer15=55.855, loss_interctc_layer21=66.212, loss=64.113, backward_time=0.223, grad_norm=163.966, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=7.002e-05, train_time=1.277 +[gpua006:0/64] 2024-02-12 07:49:38,200 (trainer:756) INFO: 33epoch:train:9701-9800batch: iter_time=4.441e-04, forward_time=0.153, loss_ctc=77.281, loss_interctc_layer6=86.606, loss_interctc_layer12=71.742, loss_interctc_layer15=65.756, loss_interctc_layer21=79.788, loss=76.235, backward_time=0.219, grad_norm=75.877, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=7.001e-05, train_time=1.261 +[gpua006:0/64] 2024-02-12 07:51:57,287 (trainer:756) INFO: 33epoch:train:9801-9900batch: iter_time=2.391e-04, forward_time=0.166, loss_ctc=65.476, loss_interctc_layer6=72.662, loss_interctc_layer12=59.648, loss_interctc_layer15=54.371, loss_interctc_layer21=67.988, loss=64.029, backward_time=0.252, grad_norm=59.427, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.000e-05, train_time=1.391 +[gpua006:0/64] 2024-02-12 07:53:58,928 (trainer:756) INFO: 33epoch:train:9901-10000batch: iter_time=8.064e-05, forward_time=0.160, loss_ctc=77.013, loss_interctc_layer6=89.308, loss_interctc_layer12=74.521, loss_interctc_layer15=68.892, loss_interctc_layer21=79.706, loss=77.888, backward_time=0.214, grad_norm=89.507, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=7.000e-05, train_time=1.216 +[gpua006:0/64] 2024-02-12 07:54:18,969 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-12 07:54:37,970 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 07:54:41,448 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 07:54:41,448 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-12 07:54:41,453 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 08:00:54,069 (trainer:756) INFO: 33epoch:train:10001-10100batch: iter_time=3.074, forward_time=0.144, loss_ctc=79.544, loss_interctc_layer6=81.578, loss_interctc_layer12=67.369, loss_interctc_layer15=61.668, loss_interctc_layer21=82.632, loss=74.558, backward_time=0.210, grad_norm=82.021, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.999e-05, train_time=4.152 +[gpua006:0/64] 2024-02-12 08:02:35,295 (trainer:756) INFO: 33epoch:train:10101-10200batch: iter_time=8.156e-05, forward_time=0.143, loss_ctc=65.820, loss_interctc_layer6=74.211, loss_interctc_layer12=60.984, loss_interctc_layer15=55.648, loss_interctc_layer21=68.297, loss=64.992, backward_time=0.209, grad_norm=68.672, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.998e-05, train_time=1.012 +[gpua006:0/64] 2024-02-12 08:04:35,159 (trainer:756) INFO: 33epoch:train:10201-10300batch: iter_time=8.198e-05, forward_time=0.225, loss_ctc=71.804, loss_interctc_layer6=76.797, loss_interctc_layer12=63.708, loss_interctc_layer15=58.466, loss_interctc_layer21=74.420, loss=69.039, backward_time=0.225, grad_norm=76.800, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.998e-05, train_time=1.198 +[gpua006:0/64] 2024-02-12 08:06:28,912 (trainer:756) INFO: 33epoch:train:10301-10400batch: iter_time=8.461e-05, forward_time=0.155, loss_ctc=84.171, loss_interctc_layer6=90.117, loss_interctc_layer12=74.881, loss_interctc_layer15=69.023, loss_interctc_layer21=86.928, loss=81.024, backward_time=0.223, grad_norm=103.576, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.997e-05, train_time=1.137 +[gpua006:0/64] 2024-02-12 08:08:31,718 (trainer:756) INFO: 33epoch:train:10401-10500batch: iter_time=8.962e-05, forward_time=0.145, loss_ctc=86.545, loss_interctc_layer6=85.277, loss_interctc_layer12=70.640, loss_interctc_layer15=64.633, loss_interctc_layer21=89.924, loss=79.404, backward_time=0.210, grad_norm=70.500, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.996e-05, train_time=1.229 +[gpua006:0/64] 2024-02-12 08:10:53,081 (trainer:756) INFO: 33epoch:train:10501-10600batch: iter_time=9.049e-05, forward_time=0.144, loss_ctc=80.246, loss_interctc_layer6=84.003, loss_interctc_layer12=69.374, loss_interctc_layer15=63.511, loss_interctc_layer21=83.102, loss=76.047, backward_time=0.208, grad_norm=69.859, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.995e-05, train_time=1.413 +[gpua006:0/64] 2024-02-12 08:13:12,390 (trainer:756) INFO: 33epoch:train:10601-10700batch: iter_time=8.617e-05, forward_time=0.143, loss_ctc=78.995, loss_interctc_layer6=77.742, loss_interctc_layer12=64.049, loss_interctc_layer15=58.434, loss_interctc_layer21=82.012, loss=72.246, backward_time=0.207, grad_norm=60.364, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.995e-05, train_time=1.393 +[gpua006:0/64] 2024-02-12 08:15:31,567 (trainer:756) INFO: 33epoch:train:10701-10800batch: iter_time=8.651e-05, forward_time=0.145, loss_ctc=82.950, loss_interctc_layer6=82.322, loss_interctc_layer12=67.310, loss_interctc_layer15=61.366, loss_interctc_layer21=86.404, loss=76.070, backward_time=0.208, grad_norm=89.701, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.994e-05, train_time=1.392 +[gpua006:0/64] 2024-02-12 08:17:50,805 (trainer:756) INFO: 33epoch:train:10801-10900batch: iter_time=9.186e-05, forward_time=0.191, loss_ctc=57.376, loss_interctc_layer6=75.439, loss_interctc_layer12=62.740, loss_interctc_layer15=57.703, loss_interctc_layer21=58.901, loss=62.432, backward_time=0.272, grad_norm=67.124, clip=100.000, loss_scale=2.576e+31, optim_step_time=0.141, optim0_lr0=6.993e-05, train_time=1.392 +[gpua006:0/64] 2024-02-12 08:19:30,230 (trainer:756) INFO: 33epoch:train:10901-11000batch: iter_time=9.367e-05, forward_time=0.158, loss_ctc=73.629, loss_interctc_layer6=76.861, loss_interctc_layer12=63.629, loss_interctc_layer15=58.189, loss_interctc_layer21=76.229, loss=69.707, backward_time=0.222, grad_norm=84.696, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.993e-05, train_time=0.993 +[gpua006:0/64] 2024-02-12 08:21:13,456 (trainer:756) INFO: 33epoch:train:11001-11100batch: iter_time=8.636e-05, forward_time=0.143, loss_ctc=71.668, loss_interctc_layer6=78.052, loss_interctc_layer12=63.978, loss_interctc_layer15=58.352, loss_interctc_layer21=74.268, loss=69.264, backward_time=0.209, grad_norm=85.460, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.992e-05, train_time=1.033 +[gpua006:0/64] 2024-02-12 08:23:32,921 (trainer:756) INFO: 33epoch:train:11101-11200batch: iter_time=8.804e-05, forward_time=0.143, loss_ctc=77.091, loss_interctc_layer6=82.108, loss_interctc_layer12=67.944, loss_interctc_layer15=62.309, loss_interctc_layer21=79.979, loss=73.886, backward_time=0.208, grad_norm=88.520, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.991e-05, train_time=1.394 +[gpua006:0/64] 2024-02-12 08:25:34,050 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-12 08:25:53,102 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 08:25:56,568 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 08:25:56,568 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-12 08:25:56,572 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 08:31:08,814 (trainer:756) INFO: 33epoch:train:11201-11300batch: iter_time=2.968, forward_time=0.169, loss_ctc=79.005, loss_interctc_layer6=85.449, loss_interctc_layer12=70.957, loss_interctc_layer15=65.198, loss_interctc_layer21=81.834, loss=76.489, backward_time=0.213, grad_norm=77.230, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.990e-05, train_time=4.559 +[gpua006:0/64] 2024-02-12 08:32:44,072 (trainer:756) INFO: 33epoch:train:11301-11400batch: iter_time=8.383e-05, forward_time=0.145, loss_ctc=76.255, loss_interctc_layer6=85.552, loss_interctc_layer12=71.019, loss_interctc_layer15=65.242, loss_interctc_layer21=78.993, loss=75.412, backward_time=0.211, grad_norm=74.744, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=6.990e-05, train_time=0.952 +[gpua006:0/64] 2024-02-12 08:33:48,212 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 08:34:20,300 (trainer:756) INFO: 33epoch:train:11401-11500batch: iter_time=8.755e-05, forward_time=0.154, loss_ctc=60.174, loss_interctc_layer6=67.471, loss_interctc_layer12=55.389, loss_interctc_layer15=50.503, loss_interctc_layer21=62.582, loss=59.224, backward_time=0.212, grad_norm=87.004, clip=100.000, loss_scale=3.401e+31, optim_step_time=0.138, optim0_lr0=6.989e-05, train_time=0.962 +[gpua006:0/64] 2024-02-12 08:36:57,246 (trainer:756) INFO: 33epoch:train:11501-11600batch: iter_time=8.778e-05, forward_time=0.189, loss_ctc=79.862, loss_interctc_layer6=92.012, loss_interctc_layer12=76.421, loss_interctc_layer15=70.284, loss_interctc_layer21=82.590, loss=80.234, backward_time=0.224, grad_norm=90.055, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.988e-05, train_time=1.569 +[gpua006:0/64] 2024-02-12 08:38:59,466 (trainer:756) INFO: 33epoch:train:11601-11700batch: iter_time=9.405e-05, forward_time=0.144, loss_ctc=69.378, loss_interctc_layer6=82.526, loss_interctc_layer12=68.561, loss_interctc_layer15=62.737, loss_interctc_layer21=71.672, loss=70.975, backward_time=0.210, grad_norm=76.730, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.988e-05, train_time=1.221 +[gpua006:0/64] 2024-02-12 08:41:19,060 (trainer:756) INFO: 33epoch:train:11701-11800batch: iter_time=8.384e-05, forward_time=0.143, loss_ctc=81.127, loss_interctc_layer6=85.843, loss_interctc_layer12=70.851, loss_interctc_layer15=64.872, loss_interctc_layer21=83.991, loss=77.337, backward_time=0.207, grad_norm=100.147, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.987e-05, train_time=1.397 +[gpua006:0/64] 2024-02-12 08:43:08,920 (trainer:756) INFO: 33epoch:train:11801-11900batch: iter_time=8.462e-05, forward_time=0.143, loss_ctc=79.496, loss_interctc_layer6=80.323, loss_interctc_layer12=66.151, loss_interctc_layer15=60.397, loss_interctc_layer21=82.718, loss=73.817, backward_time=0.209, grad_norm=63.427, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.986e-05, train_time=1.098 +[gpua006:0/64] 2024-02-12 08:45:27,522 (trainer:756) INFO: 33epoch:train:11901-12000batch: iter_time=8.410e-05, forward_time=0.143, loss_ctc=82.008, loss_interctc_layer6=84.240, loss_interctc_layer12=68.994, loss_interctc_layer15=62.737, loss_interctc_layer21=85.203, loss=76.636, backward_time=0.208, grad_norm=69.180, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.985e-05, train_time=1.386 +[gpua006:0/64] 2024-02-12 08:47:24,180 (trainer:756) INFO: 33epoch:train:12001-12100batch: iter_time=8.251e-05, forward_time=0.161, loss_ctc=61.756, loss_interctc_layer6=76.684, loss_interctc_layer12=62.995, loss_interctc_layer15=57.599, loss_interctc_layer21=63.920, loss=64.591, backward_time=0.226, grad_norm=61.569, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.985e-05, train_time=1.166 +[gpua006:0/64] 2024-02-12 08:49:30,022 (trainer:756) INFO: 33epoch:train:12101-12200batch: iter_time=8.360e-05, forward_time=0.143, loss_ctc=64.266, loss_interctc_layer6=73.952, loss_interctc_layer12=61.230, loss_interctc_layer15=56.113, loss_interctc_layer21=66.416, loss=64.396, backward_time=0.208, grad_norm=70.031, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.984e-05, train_time=1.258 +[gpua006:0/64] 2024-02-12 08:52:06,110 (trainer:756) INFO: 33epoch:train:12201-12300batch: iter_time=8.661e-05, forward_time=0.165, loss_ctc=77.571, loss_interctc_layer6=87.093, loss_interctc_layer12=72.294, loss_interctc_layer15=66.412, loss_interctc_layer21=80.089, loss=76.692, backward_time=0.224, grad_norm=80.695, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.983e-05, train_time=1.561 +[gpua006:0/64] 2024-02-12 08:53:56,652 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 08:54:09,698 (trainer:756) INFO: 33epoch:train:12301-12400batch: iter_time=8.775e-05, forward_time=0.168, loss_ctc=64.782, loss_interctc_layer6=72.471, loss_interctc_layer12=59.456, loss_interctc_layer15=54.183, loss_interctc_layer21=67.475, loss=63.673, backward_time=0.215, grad_norm=75.018, clip=100.000, loss_scale=1.936e+31, optim_step_time=0.137, optim0_lr0=6.983e-05, train_time=1.236 +[gpua006:0/64] 2024-02-12 08:55:57,185 (trainer:756) INFO: 33epoch:train:12401-12500batch: iter_time=8.206e-05, forward_time=0.142, loss_ctc=76.206, loss_interctc_layer6=89.057, loss_interctc_layer12=74.055, loss_interctc_layer15=68.251, loss_interctc_layer21=78.655, loss=77.245, backward_time=0.209, grad_norm=82.342, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.982e-05, train_time=1.074 +[gpua006:0/64] 2024-02-12 08:56:17,279 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-12 08:56:36,448 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 08:56:39,932 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 08:56:39,932 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-12 08:56:39,936 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 09:02:45,617 (trainer:756) INFO: 33epoch:train:12501-12600batch: iter_time=3.035, forward_time=0.198, loss_ctc=78.718, loss_interctc_layer6=80.600, loss_interctc_layer12=66.545, loss_interctc_layer15=60.887, loss_interctc_layer21=81.797, loss=73.710, backward_time=0.220, grad_norm=86.309, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.981e-05, train_time=4.085 +[gpua006:0/64] 2024-02-12 09:04:33,177 (trainer:756) INFO: 33epoch:train:12601-12700batch: iter_time=8.197e-05, forward_time=0.143, loss_ctc=65.201, loss_interctc_layer6=74.368, loss_interctc_layer12=61.134, loss_interctc_layer15=55.715, loss_interctc_layer21=67.697, loss=64.823, backward_time=0.210, grad_norm=70.928, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.980e-05, train_time=1.075 +[gpua006:0/64] 2024-02-12 09:06:32,248 (trainer:756) INFO: 33epoch:train:12701-12800batch: iter_time=8.446e-05, forward_time=0.146, loss_ctc=72.724, loss_interctc_layer6=76.361, loss_interctc_layer12=63.343, loss_interctc_layer15=58.070, loss_interctc_layer21=75.484, loss=69.197, backward_time=0.209, grad_norm=66.045, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.980e-05, train_time=1.189 +[gpua006:0/64] 2024-02-12 09:08:44,137 (trainer:756) INFO: 33epoch:train:12801-12900batch: iter_time=8.236e-05, forward_time=0.143, loss_ctc=83.916, loss_interctc_layer6=90.056, loss_interctc_layer12=74.595, loss_interctc_layer15=68.409, loss_interctc_layer21=86.845, loss=80.764, backward_time=0.208, grad_norm=99.497, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.979e-05, train_time=1.320 +[gpua006:0/64] 2024-02-12 09:11:21,756 (trainer:756) INFO: 33epoch:train:12901-13000batch: iter_time=8.147e-04, forward_time=0.279, loss_ctc=86.845, loss_interctc_layer6=85.390, loss_interctc_layer12=70.610, loss_interctc_layer15=64.456, loss_interctc_layer21=90.121, loss=79.484, backward_time=0.227, grad_norm=78.565, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.144, optim0_lr0=6.978e-05, train_time=1.576 +[gpua006:0/64] 2024-02-12 09:13:14,908 (trainer:756) INFO: 33epoch:train:13001-13100batch: iter_time=8.461e-05, forward_time=0.144, loss_ctc=80.488, loss_interctc_layer6=83.373, loss_interctc_layer12=68.778, loss_interctc_layer15=62.902, loss_interctc_layer21=83.520, loss=75.812, backward_time=0.208, grad_norm=85.848, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.978e-05, train_time=1.131 +[gpua006:0/64] 2024-02-12 09:15:20,276 (trainer:756) INFO: 33epoch:train:13101-13200batch: iter_time=8.644e-05, forward_time=0.143, loss_ctc=79.088, loss_interctc_layer6=77.947, loss_interctc_layer12=64.080, loss_interctc_layer15=58.449, loss_interctc_layer21=82.057, loss=72.324, backward_time=0.208, grad_norm=118.069, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.977e-05, train_time=1.253 +[gpua006:0/64] 2024-02-12 09:17:25,748 (trainer:756) INFO: 33epoch:train:13201-13300batch: iter_time=8.898e-05, forward_time=0.143, loss_ctc=83.022, loss_interctc_layer6=82.093, loss_interctc_layer12=67.267, loss_interctc_layer15=61.201, loss_interctc_layer21=86.289, loss=75.975, backward_time=0.209, grad_norm=164.358, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.976e-05, train_time=1.254 +[gpua006:0/64] 2024-02-12 09:19:58,831 (trainer:756) INFO: 33epoch:train:13301-13400batch: iter_time=9.263e-05, forward_time=0.143, loss_ctc=58.300, loss_interctc_layer6=75.925, loss_interctc_layer12=63.144, loss_interctc_layer15=58.096, loss_interctc_layer21=59.879, loss=63.069, backward_time=0.208, grad_norm=81.846, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.976e-05, train_time=1.529 +[gpua006:0/64] 2024-02-12 09:22:46,326 (trainer:756) INFO: 33epoch:train:13401-13500batch: iter_time=8.617e-05, forward_time=0.199, loss_ctc=74.102, loss_interctc_layer6=77.367, loss_interctc_layer12=64.076, loss_interctc_layer15=58.627, loss_interctc_layer21=76.597, loss=70.154, backward_time=0.234, grad_norm=81.024, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.975e-05, train_time=1.676 +[gpua006:0/64] 2024-02-12 09:24:44,926 (trainer:756) INFO: 33epoch:train:13501-13600batch: iter_time=8.870e-05, forward_time=0.185, loss_ctc=72.104, loss_interctc_layer6=78.136, loss_interctc_layer12=64.130, loss_interctc_layer15=58.465, loss_interctc_layer21=74.717, loss=69.510, backward_time=0.218, grad_norm=70.115, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.974e-05, train_time=1.185 +[gpua006:0/64] 2024-02-12 09:26:24,915 (trainer:756) INFO: 33epoch:train:13601-13700batch: iter_time=8.481e-05, forward_time=0.143, loss_ctc=76.971, loss_interctc_layer6=81.893, loss_interctc_layer12=67.842, loss_interctc_layer15=62.161, loss_interctc_layer21=79.935, loss=73.760, backward_time=0.209, grad_norm=66.139, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.973e-05, train_time=1.001 +[gpua006:0/64] 2024-02-12 09:27:36,362 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-12 09:27:55,319 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 09:27:58,793 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 09:27:58,793 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-12 09:27:58,796 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 09:32:36,503 (trainer:756) INFO: 33epoch:train:13701-13800batch: iter_time=2.546, forward_time=0.197, loss_ctc=80.342, loss_interctc_layer6=84.900, loss_interctc_layer12=70.317, loss_interctc_layer15=64.555, loss_interctc_layer21=83.177, loss=76.658, backward_time=0.219, grad_norm=84.003, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.973e-05, train_time=3.715 +[gpua006:0/64] 2024-02-12 09:34:11,433 (trainer:756) INFO: 33epoch:train:13801-13900batch: iter_time=8.279e-05, forward_time=0.145, loss_ctc=82.242, loss_interctc_layer6=85.843, loss_interctc_layer12=71.135, loss_interctc_layer15=65.235, loss_interctc_layer21=85.302, loss=77.952, backward_time=0.210, grad_norm=92.002, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.972e-05, train_time=0.950 +[gpua006:0/64] 2024-02-12 09:35:59,860 (trainer:756) INFO: 33epoch:train:13901-14000batch: iter_time=8.416e-05, forward_time=0.143, loss_ctc=62.179, loss_interctc_layer6=66.809, loss_interctc_layer12=54.867, loss_interctc_layer15=49.980, loss_interctc_layer21=64.611, loss=59.689, backward_time=0.210, grad_norm=52.943, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.971e-05, train_time=1.084 +[gpua006:0/64] 2024-02-12 09:38:41,185 (trainer:756) INFO: 33epoch:train:14001-14100batch: iter_time=8.657e-05, forward_time=0.144, loss_ctc=84.584, loss_interctc_layer6=90.778, loss_interctc_layer12=75.345, loss_interctc_layer15=69.186, loss_interctc_layer21=87.542, loss=81.487, backward_time=0.208, grad_norm=79.813, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.971e-05, train_time=1.612 +[gpua006:0/64] 2024-02-12 09:40:44,920 (trainer:756) INFO: 33epoch:train:14101-14200batch: iter_time=8.746e-05, forward_time=0.143, loss_ctc=70.973, loss_interctc_layer6=81.549, loss_interctc_layer12=67.838, loss_interctc_layer15=62.151, loss_interctc_layer21=73.471, loss=71.197, backward_time=0.208, grad_norm=76.601, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.970e-05, train_time=1.238 +[gpua006:0/64] 2024-02-12 09:43:10,533 (trainer:756) INFO: 33epoch:train:14201-14300batch: iter_time=1.731e-04, forward_time=0.224, loss_ctc=85.211, loss_interctc_layer6=85.647, loss_interctc_layer12=70.685, loss_interctc_layer15=64.643, loss_interctc_layer21=88.335, loss=78.904, backward_time=0.240, grad_norm=78.604, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.150, optim0_lr0=6.969e-05, train_time=1.456 +[gpua006:0/64] 2024-02-12 09:45:50,717 (trainer:756) INFO: 33epoch:train:14301-14400batch: iter_time=8.916e-05, forward_time=0.145, loss_ctc=84.988, loss_interctc_layer6=80.125, loss_interctc_layer12=65.911, loss_interctc_layer15=60.167, loss_interctc_layer21=88.431, loss=75.924, backward_time=0.208, grad_norm=70.318, clip=100.000, loss_scale=1.105e+31, optim_step_time=0.136, optim0_lr0=6.968e-05, train_time=1.602 +[gpua006:0/64] 2024-02-12 09:48:19,821 (trainer:756) INFO: 33epoch:train:14401-14500batch: iter_time=8.576e-05, forward_time=0.145, loss_ctc=87.622, loss_interctc_layer6=83.666, loss_interctc_layer12=68.397, loss_interctc_layer15=62.112, loss_interctc_layer21=91.004, loss=78.560, backward_time=0.208, grad_norm=60.267, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.968e-05, train_time=1.491 +[gpua006:0/64] 2024-02-12 09:50:26,889 (trainer:756) INFO: 33epoch:train:14501-14600batch: iter_time=8.629e-05, forward_time=0.143, loss_ctc=63.198, loss_interctc_layer6=75.889, loss_interctc_layer12=62.393, loss_interctc_layer15=57.016, loss_interctc_layer21=65.294, loss=64.758, backward_time=0.208, grad_norm=78.468, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.967e-05, train_time=1.271 +[gpua006:0/64] 2024-02-12 09:51:48,993 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 09:52:53,085 (trainer:756) INFO: 33epoch:train:14601-14700batch: iter_time=8.325e-05, forward_time=0.143, loss_ctc=65.670, loss_interctc_layer6=73.375, loss_interctc_layer12=60.696, loss_interctc_layer15=55.594, loss_interctc_layer21=67.780, loss=64.623, backward_time=0.209, grad_norm=74.429, clip=100.000, loss_scale=1.649e+31, optim_step_time=0.136, optim0_lr0=6.966e-05, train_time=1.462 +[gpua006:0/64] 2024-02-12 09:55:20,929 (trainer:756) INFO: 33epoch:train:14701-14800batch: iter_time=5.973e-04, forward_time=0.248, loss_ctc=82.728, loss_interctc_layer6=86.972, loss_interctc_layer12=72.125, loss_interctc_layer15=66.171, loss_interctc_layer21=85.480, loss=78.695, backward_time=0.241, grad_norm=78.877, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.151, optim0_lr0=6.966e-05, train_time=1.478 +[gpua006:0/64] 2024-02-12 09:57:07,203 (trainer:756) INFO: 33epoch:train:14801-14900batch: iter_time=8.344e-05, forward_time=0.143, loss_ctc=68.478, loss_interctc_layer6=72.779, loss_interctc_layer12=59.718, loss_interctc_layer15=54.348, loss_interctc_layer21=71.215, loss=65.308, backward_time=0.208, grad_norm=306.165, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.965e-05, train_time=1.063 +[gpua006:0/64] 2024-02-12 09:59:34,730 (trainer:756) INFO: 33epoch:train:14901-15000batch: iter_time=7.928e-05, forward_time=0.145, loss_ctc=81.324, loss_interctc_layer6=88.748, loss_interctc_layer12=73.977, loss_interctc_layer15=68.170, loss_interctc_layer21=84.206, loss=79.285, backward_time=0.208, grad_norm=85.300, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.964e-05, train_time=1.475 +[gpua006:0/64] 2024-02-12 10:30:55,051 (trainer:355) INFO: 33epoch results: [train] iter_time=0.226, forward_time=0.161, loss_ctc=74.872, loss_interctc_layer6=81.384, loss_interctc_layer12=67.358, loss_interctc_layer15=61.711, loss_interctc_layer21=77.572, loss=72.579, backward_time=0.216, grad_norm=81.172, clip=100.000, loss_scale=1.948e+31, optim_step_time=0.138, optim0_lr0=7.017e-05, train_time=1.471, time=6 hours, 8 minutes and 14.59 seconds, total_count=495000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=41.516, cer_ctc=0.195, loss_interctc_layer6=46.810, cer_interctc_layer6=0.211, loss_interctc_layer12=34.206, cer_interctc_layer12=0.145, loss_interctc_layer15=30.108, cer_interctc_layer15=0.121, loss_interctc_layer21=43.911, cer_interctc_layer21=0.206, loss=39.310, time=30 minutes and 56.2 seconds, total_count=154143, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-12 10:31:16,072 (trainer:410) INFO: The best model has been updated: valid.loss_ctc, valid.total_count +[gpua006:0/64] 2024-02-12 10:31:16,225 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/26epoch.pth, exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/27epoch.pth, exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/28epoch.pth +[gpua006:0/64] 2024-02-12 10:31:16,272 (trainer:289) INFO: 34/45epoch started. Estimated time to finish: 3 days, 13 hours and 15 minutes +[gpua006:0/64] 2024-02-12 10:31:17,267 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-12 10:31:35,463 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 10:31:38,957 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 10:31:38,957 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-12 10:31:38,960 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 10:38:04,127 (trainer:756) INFO: 34epoch:train:1-100batch: iter_time=2.755, forward_time=0.180, loss_ctc=86.492, loss_interctc_layer6=85.070, loss_interctc_layer12=70.797, loss_interctc_layer15=64.952, loss_interctc_layer21=89.673, loss=79.397, backward_time=0.218, grad_norm=102.466, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.964e-05, train_time=4.072 +[gpua006:0/64] 2024-02-12 10:39:40,309 (trainer:756) INFO: 34epoch:train:101-200batch: iter_time=8.734e-05, forward_time=0.142, loss_ctc=71.666, loss_interctc_layer6=80.631, loss_interctc_layer12=66.900, loss_interctc_layer15=61.325, loss_interctc_layer21=74.149, loss=70.934, backward_time=0.209, grad_norm=71.891, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.963e-05, train_time=0.962 +[gpua006:0/64] 2024-02-12 10:41:23,971 (trainer:756) INFO: 34epoch:train:201-300batch: iter_time=2.332e-04, forward_time=0.196, loss_ctc=77.988, loss_interctc_layer6=89.894, loss_interctc_layer12=74.979, loss_interctc_layer15=69.016, loss_interctc_layer21=80.451, loss=78.466, backward_time=0.232, grad_norm=76.765, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=6.962e-05, train_time=1.036 +[gpua006:0/64] 2024-02-12 10:43:07,289 (trainer:756) INFO: 34epoch:train:301-400batch: iter_time=9.013e-05, forward_time=0.186, loss_ctc=73.760, loss_interctc_layer6=81.467, loss_interctc_layer12=68.152, loss_interctc_layer15=62.724, loss_interctc_layer21=76.257, loss=72.472, backward_time=0.233, grad_norm=65.604, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.961e-05, train_time=1.032 +[gpua006:0/64] 2024-02-12 10:47:09,527 (trainer:756) INFO: 34epoch:train:401-500batch: iter_time=9.444e-05, forward_time=0.158, loss_ctc=81.973, loss_interctc_layer6=82.798, loss_interctc_layer12=68.921, loss_interctc_layer15=63.369, loss_interctc_layer21=84.944, loss=76.401, backward_time=0.215, grad_norm=79.726, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.961e-05, train_time=2.423 +[gpua006:0/64] 2024-02-12 10:49:11,118 (trainer:756) INFO: 34epoch:train:501-600batch: iter_time=8.892e-05, forward_time=0.174, loss_ctc=79.955, loss_interctc_layer6=80.411, loss_interctc_layer12=66.688, loss_interctc_layer15=61.026, loss_interctc_layer21=83.056, loss=74.227, backward_time=0.221, grad_norm=92.423, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.960e-05, train_time=1.215 +[gpua006:0/64] 2024-02-12 10:52:29,445 (trainer:756) INFO: 34epoch:train:601-700batch: iter_time=8.573e-05, forward_time=0.178, loss_ctc=75.345, loss_interctc_layer6=82.407, loss_interctc_layer12=68.200, loss_interctc_layer15=62.377, loss_interctc_layer21=77.993, loss=73.264, backward_time=0.251, grad_norm=74.929, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.959e-05, train_time=1.983 +[gpua006:0/64] 2024-02-12 10:54:28,490 (trainer:756) INFO: 34epoch:train:701-800batch: iter_time=8.721e-05, forward_time=0.179, loss_ctc=93.388, loss_interctc_layer6=94.060, loss_interctc_layer12=78.548, loss_interctc_layer15=72.335, loss_interctc_layer21=96.588, loss=86.984, backward_time=0.213, grad_norm=93.447, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.959e-05, train_time=1.191 +[gpua006:0/64] 2024-02-12 10:56:47,051 (trainer:756) INFO: 34epoch:train:801-900batch: iter_time=9.487e-05, forward_time=0.161, loss_ctc=84.830, loss_interctc_layer6=83.862, loss_interctc_layer12=69.604, loss_interctc_layer15=63.841, loss_interctc_layer21=87.970, loss=78.021, backward_time=0.218, grad_norm=80.060, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.958e-05, train_time=1.385 +[gpua006:0/64] 2024-02-12 10:58:58,026 (trainer:756) INFO: 34epoch:train:901-1000batch: iter_time=1.036e-04, forward_time=0.168, loss_ctc=73.100, loss_interctc_layer6=84.396, loss_interctc_layer12=70.128, loss_interctc_layer15=64.230, loss_interctc_layer21=75.747, loss=73.520, backward_time=0.216, grad_norm=89.000, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.957e-05, train_time=1.309 +[gpua006:0/64] 2024-02-12 11:01:09,835 (trainer:756) INFO: 34epoch:train:1001-1100batch: iter_time=9.172e-05, forward_time=0.159, loss_ctc=70.440, loss_interctc_layer6=77.775, loss_interctc_layer12=64.545, loss_interctc_layer15=59.239, loss_interctc_layer21=72.843, loss=68.968, backward_time=0.223, grad_norm=72.045, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.957e-05, train_time=1.317 +[gpua006:0/64] 2024-02-12 11:03:18,722 (trainer:756) INFO: 34epoch:train:1101-1200batch: iter_time=9.111e-05, forward_time=0.168, loss_ctc=75.169, loss_interctc_layer6=77.319, loss_interctc_layer12=64.200, loss_interctc_layer15=59.012, loss_interctc_layer21=77.637, loss=70.667, backward_time=0.220, grad_norm=227.308, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.956e-05, train_time=1.290 +[gpua006:0/64] 2024-02-12 11:04:29,206 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-12 11:04:48,141 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 11:04:51,609 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 11:04:51,609 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-12 11:04:51,613 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 11:10:07,361 (trainer:756) INFO: 34epoch:train:1201-1300batch: iter_time=2.607, forward_time=0.201, loss_ctc=76.979, loss_interctc_layer6=79.206, loss_interctc_layer12=65.552, loss_interctc_layer15=59.840, loss_interctc_layer21=79.846, loss=72.285, backward_time=0.220, grad_norm=67.947, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.955e-05, train_time=4.086 +[gpua006:0/64] 2024-02-12 11:11:47,916 (trainer:756) INFO: 34epoch:train:1301-1400batch: iter_time=8.863e-05, forward_time=0.147, loss_ctc=82.729, loss_interctc_layer6=86.592, loss_interctc_layer12=72.227, loss_interctc_layer15=66.288, loss_interctc_layer21=85.957, loss=78.759, backward_time=0.211, grad_norm=72.725, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.954e-05, train_time=1.006 +[gpua006:0/64] 2024-02-12 11:13:32,726 (trainer:756) INFO: 34epoch:train:1401-1500batch: iter_time=8.297e-05, forward_time=0.189, loss_ctc=64.456, loss_interctc_layer6=79.128, loss_interctc_layer12=65.662, loss_interctc_layer15=60.263, loss_interctc_layer21=66.552, loss=67.212, backward_time=0.224, grad_norm=78.764, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.954e-05, train_time=1.047 +[gpua006:0/64] 2024-02-12 11:15:49,132 (trainer:756) INFO: 34epoch:train:1501-1600batch: iter_time=8.609e-05, forward_time=0.231, loss_ctc=82.807, loss_interctc_layer6=95.488, loss_interctc_layer12=79.576, loss_interctc_layer15=73.175, loss_interctc_layer21=85.720, loss=83.353, backward_time=0.224, grad_norm=73.152, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.953e-05, train_time=1.364 +[gpua006:0/64] 2024-02-12 11:17:54,927 (trainer:756) INFO: 34epoch:train:1601-1700batch: iter_time=8.296e-05, forward_time=0.144, loss_ctc=65.159, loss_interctc_layer6=75.173, loss_interctc_layer12=62.512, loss_interctc_layer15=57.477, loss_interctc_layer21=67.372, loss=65.538, backward_time=0.208, grad_norm=69.649, clip=100.000, loss_scale=1.389e+31, optim_step_time=0.137, optim0_lr0=6.952e-05, train_time=1.258 +[gpua006:0/64] 2024-02-12 11:20:07,859 (trainer:756) INFO: 34epoch:train:1701-1800batch: iter_time=8.514e-05, forward_time=0.144, loss_ctc=79.371, loss_interctc_layer6=82.405, loss_interctc_layer12=67.913, loss_interctc_layer15=62.070, loss_interctc_layer21=82.579, loss=74.868, backward_time=0.209, grad_norm=94.642, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.952e-05, train_time=1.329 +[gpua006:0/64] 2024-02-12 11:22:30,432 (trainer:756) INFO: 34epoch:train:1801-1900batch: iter_time=8.281e-05, forward_time=0.205, loss_ctc=66.591, loss_interctc_layer6=75.857, loss_interctc_layer12=62.473, loss_interctc_layer15=56.934, loss_interctc_layer21=69.119, loss=66.195, backward_time=0.243, grad_norm=74.021, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.951e-05, train_time=1.425 +[gpua006:0/64] 2024-02-12 11:24:41,470 (trainer:756) INFO: 34epoch:train:1901-2000batch: iter_time=8.390e-05, forward_time=0.173, loss_ctc=74.556, loss_interctc_layer6=89.100, loss_interctc_layer12=74.442, loss_interctc_layer15=68.480, loss_interctc_layer21=76.869, loss=76.689, backward_time=0.226, grad_norm=70.125, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.950e-05, train_time=1.311 +[gpua006:0/64] 2024-02-12 11:26:39,073 (trainer:756) INFO: 34epoch:train:2001-2100batch: iter_time=8.118e-05, forward_time=0.143, loss_ctc=88.654, loss_interctc_layer6=86.148, loss_interctc_layer12=71.493, loss_interctc_layer15=65.474, loss_interctc_layer21=92.467, loss=80.847, backward_time=0.207, grad_norm=71.021, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.950e-05, train_time=1.176 +[gpua006:0/64] 2024-02-12 11:28:45,986 (trainer:756) INFO: 34epoch:train:2101-2200batch: iter_time=8.204e-05, forward_time=0.143, loss_ctc=79.622, loss_interctc_layer6=90.889, loss_interctc_layer12=75.393, loss_interctc_layer15=69.085, loss_interctc_layer21=82.946, loss=79.587, backward_time=0.207, grad_norm=79.258, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.949e-05, train_time=1.269 +[gpua006:0/64] 2024-02-12 11:30:37,941 (trainer:756) INFO: 34epoch:train:2201-2300batch: iter_time=8.500e-05, forward_time=0.149, loss_ctc=67.318, loss_interctc_layer6=76.897, loss_interctc_layer12=63.421, loss_interctc_layer15=58.037, loss_interctc_layer21=69.738, loss=67.082, backward_time=0.209, grad_norm=61.685, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.948e-05, train_time=1.119 +[gpua006:0/64] 2024-02-12 11:32:39,365 (trainer:756) INFO: 34epoch:train:2301-2400batch: iter_time=8.536e-05, forward_time=0.213, loss_ctc=73.178, loss_interctc_layer6=73.495, loss_interctc_layer12=60.690, loss_interctc_layer15=55.431, loss_interctc_layer21=76.078, loss=67.774, backward_time=0.234, grad_norm=67.000, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=6.947e-05, train_time=1.213 +[gpua006:0/64] 2024-02-12 11:34:24,947 (trainer:756) INFO: 34epoch:train:2401-2500batch: iter_time=7.817e-05, forward_time=0.172, loss_ctc=67.700, loss_interctc_layer6=79.291, loss_interctc_layer12=65.882, loss_interctc_layer15=60.393, loss_interctc_layer21=70.092, loss=68.671, backward_time=0.216, grad_norm=69.813, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.947e-05, train_time=1.057 +[gpua006:0/64] 2024-02-12 11:34:43,285 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-12 11:35:02,128 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 11:35:05,563 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 11:35:05,563 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-12 11:35:05,566 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 11:41:01,858 (trainer:756) INFO: 34epoch:train:2501-2600batch: iter_time=2.880, forward_time=0.144, loss_ctc=87.116, loss_interctc_layer6=84.369, loss_interctc_layer12=69.960, loss_interctc_layer15=64.090, loss_interctc_layer21=90.281, loss=79.163, backward_time=0.209, grad_norm=68.851, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.946e-05, train_time=3.969 +[gpua006:0/64] 2024-02-12 11:42:39,011 (trainer:756) INFO: 34epoch:train:2601-2700batch: iter_time=7.722e-05, forward_time=0.143, loss_ctc=72.061, loss_interctc_layer6=80.845, loss_interctc_layer12=67.084, loss_interctc_layer15=61.537, loss_interctc_layer21=74.600, loss=71.225, backward_time=0.209, grad_norm=76.372, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.945e-05, train_time=0.970 +[gpua006:0/64] 2024-02-12 11:44:19,884 (trainer:756) INFO: 34epoch:train:2701-2800batch: iter_time=1.048e-04, forward_time=0.144, loss_ctc=77.528, loss_interctc_layer6=89.607, loss_interctc_layer12=74.610, loss_interctc_layer15=68.539, loss_interctc_layer21=79.916, loss=78.040, backward_time=0.209, grad_norm=93.656, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.945e-05, train_time=1.010 +[gpua006:0/64] 2024-02-12 11:46:38,867 (trainer:756) INFO: 34epoch:train:2801-2900batch: iter_time=8.210e-05, forward_time=0.143, loss_ctc=72.057, loss_interctc_layer6=79.971, loss_interctc_layer12=66.485, loss_interctc_layer15=60.851, loss_interctc_layer21=74.600, loss=70.793, backward_time=0.208, grad_norm=75.765, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.944e-05, train_time=1.390 +[gpua006:0/64] 2024-02-12 11:48:35,532 (trainer:756) INFO: 34epoch:train:2901-3000batch: iter_time=1.471e-04, forward_time=0.184, loss_ctc=80.898, loss_interctc_layer6=81.712, loss_interctc_layer12=67.664, loss_interctc_layer15=62.029, loss_interctc_layer21=83.841, loss=75.229, backward_time=0.231, grad_norm=113.344, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.943e-05, train_time=1.166 +[gpua006:0/64] 2024-02-12 11:50:52,893 (trainer:756) INFO: 34epoch:train:3001-3100batch: iter_time=8.560e-05, forward_time=0.176, loss_ctc=79.455, loss_interctc_layer6=79.893, loss_interctc_layer12=66.243, loss_interctc_layer15=60.663, loss_interctc_layer21=82.564, loss=73.764, backward_time=0.227, grad_norm=73.232, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.943e-05, train_time=1.372 +[gpua006:0/64] 2024-02-12 11:52:46,167 (trainer:756) INFO: 34epoch:train:3101-3200batch: iter_time=8.730e-05, forward_time=0.144, loss_ctc=75.502, loss_interctc_layer6=82.380, loss_interctc_layer12=68.102, loss_interctc_layer15=62.231, loss_interctc_layer21=78.087, loss=73.260, backward_time=0.208, grad_norm=76.179, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.942e-05, train_time=1.134 +[gpua006:0/64] 2024-02-12 11:54:41,645 (trainer:756) INFO: 34epoch:train:3201-3300batch: iter_time=8.455e-05, forward_time=0.145, loss_ctc=91.399, loss_interctc_layer6=92.440, loss_interctc_layer12=76.858, loss_interctc_layer15=70.585, loss_interctc_layer21=94.707, loss=85.198, backward_time=0.208, grad_norm=69.108, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.941e-05, train_time=1.155 +[gpua006:0/64] 2024-02-12 11:56:50,272 (trainer:756) INFO: 34epoch:train:3301-3400batch: iter_time=8.579e-05, forward_time=0.145, loss_ctc=83.669, loss_interctc_layer6=83.083, loss_interctc_layer12=68.695, loss_interctc_layer15=62.800, loss_interctc_layer21=87.010, loss=77.051, backward_time=0.208, grad_norm=69.064, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.940e-05, train_time=1.286 +[gpua006:0/64] 2024-02-12 11:59:47,198 (trainer:756) INFO: 34epoch:train:3401-3500batch: iter_time=8.522e-05, forward_time=0.144, loss_ctc=72.039, loss_interctc_layer6=83.694, loss_interctc_layer12=69.008, loss_interctc_layer15=63.165, loss_interctc_layer21=74.697, loss=72.520, backward_time=0.207, grad_norm=101.531, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.940e-05, train_time=1.770 +[gpua006:0/64] 2024-02-12 12:01:54,840 (trainer:756) INFO: 34epoch:train:3501-3600batch: iter_time=8.272e-05, forward_time=0.153, loss_ctc=69.304, loss_interctc_layer6=76.817, loss_interctc_layer12=63.401, loss_interctc_layer15=58.095, loss_interctc_layer21=71.833, loss=67.890, backward_time=0.210, grad_norm=72.731, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.939e-05, train_time=1.276 +[gpua006:0/64] 2024-02-12 12:03:47,341 (trainer:756) INFO: 34epoch:train:3601-3700batch: iter_time=8.100e-05, forward_time=0.227, loss_ctc=74.971, loss_interctc_layer6=76.715, loss_interctc_layer12=63.594, loss_interctc_layer15=58.379, loss_interctc_layer21=77.542, loss=70.240, backward_time=0.239, grad_norm=69.925, clip=100.000, loss_scale=2.779e+31, optim_step_time=0.141, optim0_lr0=6.938e-05, train_time=1.125 +[gpua006:0/64] 2024-02-12 12:05:02,536 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-12 12:05:21,128 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 12:05:24,562 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 12:05:24,562 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-12 12:05:24,566 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 12:10:30,735 (trainer:756) INFO: 34epoch:train:3701-3800batch: iter_time=2.869, forward_time=0.241, loss_ctc=77.586, loss_interctc_layer6=78.724, loss_interctc_layer12=65.000, loss_interctc_layer15=59.387, loss_interctc_layer21=80.633, loss=72.266, backward_time=0.231, grad_norm=77.456, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=6.938e-05, train_time=4.034 +[gpua006:0/64] 2024-02-12 12:12:35,248 (trainer:756) INFO: 34epoch:train:3801-3900batch: iter_time=7.956e-05, forward_time=0.310, loss_ctc=87.913, loss_interctc_layer6=86.043, loss_interctc_layer12=71.575, loss_interctc_layer15=65.740, loss_interctc_layer21=91.170, loss=80.488, backward_time=0.247, grad_norm=91.359, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=6.937e-05, train_time=1.242 +[gpua006:0/64] 2024-02-12 12:14:17,657 (trainer:756) INFO: 34epoch:train:3901-4000batch: iter_time=2.537e-04, forward_time=0.158, loss_ctc=64.927, loss_interctc_layer6=78.353, loss_interctc_layer12=65.076, loss_interctc_layer15=59.679, loss_interctc_layer21=67.032, loss=67.013, backward_time=0.210, grad_norm=65.362, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=6.936e-05, train_time=1.026 +[gpua006:0/64] 2024-02-12 12:15:17,461 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 12:16:30,524 (trainer:756) INFO: 34epoch:train:4001-4100batch: iter_time=8.290e-05, forward_time=0.282, loss_ctc=86.143, loss_interctc_layer6=94.822, loss_interctc_layer12=78.911, loss_interctc_layer15=72.476, loss_interctc_layer21=88.976, loss=84.266, backward_time=0.238, grad_norm=82.402, clip=100.000, loss_scale=2.991e+31, optim_step_time=0.141, optim0_lr0=6.936e-05, train_time=1.328 +[gpua006:0/64] 2024-02-12 12:18:25,376 (trainer:756) INFO: 34epoch:train:4101-4200batch: iter_time=1.989e-04, forward_time=0.161, loss_ctc=67.209, loss_interctc_layer6=74.597, loss_interctc_layer12=61.945, loss_interctc_layer15=56.828, loss_interctc_layer21=69.611, loss=66.038, backward_time=0.212, grad_norm=64.283, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.935e-05, train_time=1.150 +[gpua006:0/64] 2024-02-12 12:21:36,772 (trainer:756) INFO: 34epoch:train:4201-4300batch: iter_time=8.511e-05, forward_time=0.283, loss_ctc=84.917, loss_interctc_layer6=82.296, loss_interctc_layer12=67.842, loss_interctc_layer15=61.987, loss_interctc_layer21=88.349, loss=77.078, backward_time=0.240, grad_norm=81.501, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.934e-05, train_time=1.914 +[gpua006:0/64] 2024-02-12 12:23:58,756 (trainer:756) INFO: 34epoch:train:4301-4400batch: iter_time=4.407e-04, forward_time=0.148, loss_ctc=69.172, loss_interctc_layer6=75.953, loss_interctc_layer12=62.522, loss_interctc_layer15=56.989, loss_interctc_layer21=71.714, loss=67.270, backward_time=0.214, grad_norm=99.397, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.934e-05, train_time=1.417 +[gpua006:0/64] 2024-02-12 12:26:27,321 (trainer:756) INFO: 34epoch:train:4401-4500batch: iter_time=8.125e-05, forward_time=0.201, loss_ctc=77.086, loss_interctc_layer6=88.774, loss_interctc_layer12=73.909, loss_interctc_layer15=67.984, loss_interctc_layer21=79.443, loss=77.439, backward_time=0.297, grad_norm=81.754, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.933e-05, train_time=1.488 +[gpua006:0/64] 2024-02-12 12:28:39,575 (trainer:756) INFO: 34epoch:train:4501-4600batch: iter_time=8.266e-05, forward_time=0.148, loss_ctc=93.809, loss_interctc_layer6=85.314, loss_interctc_layer12=70.618, loss_interctc_layer15=64.585, loss_interctc_layer21=97.370, loss=82.339, backward_time=0.214, grad_norm=134.410, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.932e-05, train_time=1.320 +[gpua006:0/64] 2024-02-12 12:31:13,963 (trainer:756) INFO: 34epoch:train:4601-4700batch: iter_time=8.188e-05, forward_time=0.294, loss_ctc=84.216, loss_interctc_layer6=89.969, loss_interctc_layer12=74.370, loss_interctc_layer15=68.079, loss_interctc_layer21=87.462, loss=80.819, backward_time=0.236, grad_norm=74.930, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.931e-05, train_time=1.546 +[gpua006:0/64] 2024-02-12 12:33:05,982 (trainer:756) INFO: 34epoch:train:4701-4800batch: iter_time=2.090e-04, forward_time=0.145, loss_ctc=68.875, loss_interctc_layer6=76.058, loss_interctc_layer12=62.629, loss_interctc_layer15=57.180, loss_interctc_layer21=71.344, loss=67.217, backward_time=0.212, grad_norm=73.274, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.931e-05, train_time=1.120 +[gpua006:0/64] 2024-02-12 12:34:55,195 (trainer:756) INFO: 34epoch:train:4801-4900batch: iter_time=8.039e-05, forward_time=0.142, loss_ctc=72.223, loss_interctc_layer6=72.715, loss_interctc_layer12=59.868, loss_interctc_layer15=54.615, loss_interctc_layer21=74.872, loss=66.859, backward_time=0.208, grad_norm=67.089, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.930e-05, train_time=1.091 +[gpua006:0/64] 2024-02-12 12:37:45,325 (trainer:756) INFO: 34epoch:train:4901-5000batch: iter_time=2.598e-04, forward_time=0.220, loss_ctc=71.620, loss_interctc_layer6=78.964, loss_interctc_layer12=65.349, loss_interctc_layer15=59.926, loss_interctc_layer21=74.107, loss=69.993, backward_time=0.284, grad_norm=85.425, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.144, optim0_lr0=6.929e-05, train_time=1.702 +[gpua006:0/64] 2024-02-12 12:38:05,376 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-12 12:38:24,565 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 12:38:28,037 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 12:38:28,038 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-12 12:38:28,041 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 12:44:50,844 (trainer:756) INFO: 34epoch:train:5001-5100batch: iter_time=3.060, forward_time=0.144, loss_ctc=82.263, loss_interctc_layer6=83.760, loss_interctc_layer12=69.265, loss_interctc_layer15=63.359, loss_interctc_layer21=85.474, loss=76.824, backward_time=0.210, grad_norm=119.925, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.929e-05, train_time=4.255 +[gpua006:0/64] 2024-02-12 12:46:28,459 (trainer:756) INFO: 34epoch:train:5101-5200batch: iter_time=7.806e-05, forward_time=0.143, loss_ctc=67.004, loss_interctc_layer6=80.830, loss_interctc_layer12=66.960, loss_interctc_layer15=61.340, loss_interctc_layer21=69.482, loss=69.123, backward_time=0.210, grad_norm=75.821, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.928e-05, train_time=0.976 +[gpua006:0/64] 2024-02-12 12:48:25,796 (trainer:756) INFO: 34epoch:train:5201-5300batch: iter_time=8.076e-05, forward_time=0.146, loss_ctc=76.381, loss_interctc_layer6=89.007, loss_interctc_layer12=73.980, loss_interctc_layer15=68.074, loss_interctc_layer21=78.875, loss=77.263, backward_time=0.209, grad_norm=192.591, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.927e-05, train_time=1.173 +[gpua006:0/64] 2024-02-12 12:50:41,773 (trainer:756) INFO: 34epoch:train:5301-5400batch: iter_time=8.371e-05, forward_time=0.254, loss_ctc=68.647, loss_interctc_layer6=80.025, loss_interctc_layer12=66.499, loss_interctc_layer15=60.955, loss_interctc_layer21=70.757, loss=69.377, backward_time=0.238, grad_norm=86.284, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=6.927e-05, train_time=1.358 +[gpua006:0/64] 2024-02-12 12:52:38,278 (trainer:756) INFO: 34epoch:train:5401-5500batch: iter_time=8.671e-05, forward_time=0.144, loss_ctc=77.515, loss_interctc_layer6=82.169, loss_interctc_layer12=67.951, loss_interctc_layer15=62.248, loss_interctc_layer21=80.352, loss=74.047, backward_time=0.209, grad_norm=79.383, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.926e-05, train_time=1.166 +[gpua006:0/64] 2024-02-12 12:55:09,378 (trainer:756) INFO: 34epoch:train:5501-5600batch: iter_time=8.401e-05, forward_time=0.143, loss_ctc=73.903, loss_interctc_layer6=79.732, loss_interctc_layer12=65.923, loss_interctc_layer15=60.334, loss_interctc_layer21=76.951, loss=71.369, backward_time=0.208, grad_norm=80.502, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.925e-05, train_time=1.511 +[gpua006:0/64] 2024-02-12 12:56:53,764 (trainer:756) INFO: 34epoch:train:5601-5700batch: iter_time=8.677e-05, forward_time=0.143, loss_ctc=72.905, loss_interctc_layer6=81.690, loss_interctc_layer12=67.498, loss_interctc_layer15=61.754, loss_interctc_layer21=75.568, loss=71.883, backward_time=0.209, grad_norm=77.276, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.925e-05, train_time=1.044 +[gpua006:0/64] 2024-02-12 12:58:43,314 (trainer:756) INFO: 34epoch:train:5701-5800batch: iter_time=8.589e-05, forward_time=0.144, loss_ctc=83.625, loss_interctc_layer6=92.310, loss_interctc_layer12=76.485, loss_interctc_layer15=70.167, loss_interctc_layer21=86.690, loss=81.855, backward_time=0.209, grad_norm=105.784, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.924e-05, train_time=1.095 +[gpua006:0/64] 2024-02-12 13:01:54,232 (trainer:756) INFO: 34epoch:train:5801-5900batch: iter_time=8.338e-05, forward_time=0.143, loss_ctc=78.890, loss_interctc_layer6=82.400, loss_interctc_layer12=68.097, loss_interctc_layer15=62.181, loss_interctc_layer21=82.204, loss=74.754, backward_time=0.206, grad_norm=92.238, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.923e-05, train_time=1.909 +[gpua006:0/64] 2024-02-12 13:04:14,323 (trainer:756) INFO: 34epoch:train:5901-6000batch: iter_time=9.394e-05, forward_time=0.288, loss_ctc=67.984, loss_interctc_layer6=82.301, loss_interctc_layer12=67.920, loss_interctc_layer15=61.990, loss_interctc_layer21=70.479, loss=70.135, backward_time=0.244, grad_norm=78.256, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.147, optim0_lr0=6.922e-05, train_time=1.400 +[gpua006:0/64] 2024-02-12 13:06:11,562 (trainer:756) INFO: 34epoch:train:6001-6100batch: iter_time=8.363e-05, forward_time=0.144, loss_ctc=66.571, loss_interctc_layer6=75.868, loss_interctc_layer12=62.512, loss_interctc_layer15=57.075, loss_interctc_layer21=69.047, loss=66.215, backward_time=0.209, grad_norm=70.655, clip=100.000, loss_scale=3.083e+31, optim_step_time=0.136, optim0_lr0=6.922e-05, train_time=1.173 +[gpua006:0/64] 2024-02-12 13:07:50,662 (trainer:756) INFO: 34epoch:train:6101-6200batch: iter_time=9.255e-05, forward_time=0.143, loss_ctc=72.005, loss_interctc_layer6=76.289, loss_interctc_layer12=63.210, loss_interctc_layer15=57.950, loss_interctc_layer21=74.455, loss=68.782, backward_time=0.209, grad_norm=108.759, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.921e-05, train_time=0.991 +[gpua006:0/64] 2024-02-12 13:09:20,278 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-12 13:09:39,340 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 13:09:42,786 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 13:09:42,786 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-12 13:09:42,790 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 13:14:58,042 (trainer:756) INFO: 34epoch:train:6201-6300batch: iter_time=3.024, forward_time=0.169, loss_ctc=75.092, loss_interctc_layer6=78.621, loss_interctc_layer12=65.075, loss_interctc_layer15=59.390, loss_interctc_layer21=77.832, loss=71.202, backward_time=0.217, grad_norm=87.372, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.920e-05, train_time=4.273 +[gpua006:0/64] 2024-02-12 13:16:34,725 (trainer:756) INFO: 34epoch:train:6301-6400batch: iter_time=8.378e-05, forward_time=0.145, loss_ctc=82.928, loss_interctc_layer6=86.089, loss_interctc_layer12=71.436, loss_interctc_layer15=65.620, loss_interctc_layer21=85.933, loss=78.401, backward_time=0.210, grad_norm=78.408, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.920e-05, train_time=0.967 +[gpua006:0/64] 2024-02-12 13:18:55,407 (trainer:756) INFO: 34epoch:train:6401-6500batch: iter_time=6.647e-04, forward_time=0.237, loss_ctc=63.264, loss_interctc_layer6=78.153, loss_interctc_layer12=64.821, loss_interctc_layer15=59.411, loss_interctc_layer21=65.333, loss=66.197, backward_time=0.253, grad_norm=70.370, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.163, optim0_lr0=6.919e-05, train_time=1.406 +[gpua006:0/64] 2024-02-12 13:21:19,397 (trainer:756) INFO: 34epoch:train:6501-6600batch: iter_time=9.493e-05, forward_time=0.144, loss_ctc=82.620, loss_interctc_layer6=94.962, loss_interctc_layer12=78.778, loss_interctc_layer15=72.242, loss_interctc_layer21=85.694, loss=82.859, backward_time=0.209, grad_norm=123.934, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.918e-05, train_time=1.440 +[gpua006:0/64] 2024-02-12 13:23:51,375 (trainer:756) INFO: 34epoch:train:6601-6700batch: iter_time=9.057e-05, forward_time=0.143, loss_ctc=64.632, loss_interctc_layer6=74.511, loss_interctc_layer12=61.805, loss_interctc_layer15=56.677, loss_interctc_layer21=66.947, loss=64.914, backward_time=0.209, grad_norm=65.984, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.918e-05, train_time=1.520 +[gpua006:0/64] 2024-02-12 13:25:46,608 (trainer:756) INFO: 34epoch:train:6701-6800batch: iter_time=8.441e-05, forward_time=0.144, loss_ctc=78.617, loss_interctc_layer6=81.791, loss_interctc_layer12=67.275, loss_interctc_layer15=61.480, loss_interctc_layer21=81.805, loss=74.194, backward_time=0.209, grad_norm=88.099, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.917e-05, train_time=1.152 +[gpua006:0/64] 2024-02-12 13:27:47,039 (trainer:756) INFO: 34epoch:train:6801-6900batch: iter_time=8.801e-05, forward_time=0.200, loss_ctc=65.887, loss_interctc_layer6=74.959, loss_interctc_layer12=61.584, loss_interctc_layer15=56.080, loss_interctc_layer21=68.343, loss=65.371, backward_time=0.293, grad_norm=69.560, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=6.916e-05, train_time=1.203 +[gpua006:0/64] 2024-02-12 13:29:26,511 (trainer:756) INFO: 34epoch:train:6901-7000batch: iter_time=8.921e-05, forward_time=0.144, loss_ctc=72.793, loss_interctc_layer6=88.048, loss_interctc_layer12=73.437, loss_interctc_layer15=67.521, loss_interctc_layer21=75.024, loss=75.365, backward_time=0.210, grad_norm=76.127, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.916e-05, train_time=0.995 +[gpua006:0/64] 2024-02-12 13:31:31,586 (trainer:756) INFO: 34epoch:train:7001-7100batch: iter_time=8.906e-05, forward_time=0.146, loss_ctc=88.290, loss_interctc_layer6=85.524, loss_interctc_layer12=70.730, loss_interctc_layer15=64.796, loss_interctc_layer21=91.932, loss=80.254, backward_time=0.208, grad_norm=70.367, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.915e-05, train_time=1.251 +[gpua006:0/64] 2024-02-12 13:34:05,826 (trainer:756) INFO: 34epoch:train:7101-7200batch: iter_time=8.839e-05, forward_time=0.143, loss_ctc=77.837, loss_interctc_layer6=88.889, loss_interctc_layer12=73.555, loss_interctc_layer15=67.365, loss_interctc_layer21=80.847, loss=77.699, backward_time=0.208, grad_norm=73.803, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.914e-05, train_time=1.542 +[gpua006:0/64] 2024-02-12 13:36:10,435 (trainer:756) INFO: 34epoch:train:7201-7300batch: iter_time=8.813e-05, forward_time=0.234, loss_ctc=66.279, loss_interctc_layer6=75.674, loss_interctc_layer12=62.266, loss_interctc_layer15=56.846, loss_interctc_layer21=68.829, loss=65.979, backward_time=0.264, grad_norm=70.893, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=6.913e-05, train_time=1.246 +[gpua006:0/64] 2024-02-12 13:38:38,706 (trainer:756) INFO: 34epoch:train:7301-7400batch: iter_time=8.532e-05, forward_time=0.147, loss_ctc=70.773, loss_interctc_layer6=72.456, loss_interctc_layer12=59.609, loss_interctc_layer15=54.338, loss_interctc_layer21=73.400, loss=66.115, backward_time=0.210, grad_norm=59.994, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.913e-05, train_time=1.482 +[gpua006:0/64] 2024-02-12 13:40:53,711 (trainer:756) INFO: 34epoch:train:7401-7500batch: iter_time=8.385e-05, forward_time=0.143, loss_ctc=67.536, loss_interctc_layer6=78.835, loss_interctc_layer12=65.321, loss_interctc_layer15=59.902, loss_interctc_layer21=70.179, loss=68.354, backward_time=0.209, grad_norm=90.383, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.912e-05, train_time=1.350 +[gpua006:0/64] 2024-02-12 13:41:13,763 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-12 13:41:32,987 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 13:41:36,456 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 13:41:36,456 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-12 13:41:36,460 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 13:46:14,442 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 13:46:36,450 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 13:47:48,203 (trainer:756) INFO: 34epoch:train:7501-7600batch: iter_time=3.039, forward_time=0.145, loss_ctc=85.755, loss_interctc_layer6=83.765, loss_interctc_layer12=69.340, loss_interctc_layer15=63.429, loss_interctc_layer21=89.125, loss=78.283, backward_time=0.211, grad_norm=71.872, clip=100.000, loss_scale=1.273e+31, optim_step_time=0.137, optim0_lr0=6.911e-05, train_time=4.145 +[gpua006:0/64] 2024-02-12 13:49:26,073 (trainer:756) INFO: 34epoch:train:7601-7700batch: iter_time=8.168e-05, forward_time=0.142, loss_ctc=70.840, loss_interctc_layer6=79.768, loss_interctc_layer12=66.072, loss_interctc_layer15=60.530, loss_interctc_layer21=73.347, loss=70.111, backward_time=0.209, grad_norm=86.587, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.911e-05, train_time=0.978 +[gpua006:0/64] 2024-02-12 13:52:10,968 (trainer:756) INFO: 34epoch:train:7701-7800batch: iter_time=8.224e-05, forward_time=0.261, loss_ctc=77.002, loss_interctc_layer6=87.994, loss_interctc_layer12=72.935, loss_interctc_layer15=67.025, loss_interctc_layer21=79.548, loss=76.901, backward_time=0.262, grad_norm=86.126, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=6.910e-05, train_time=1.649 +[gpua006:0/64] 2024-02-12 13:54:34,893 (trainer:756) INFO: 34epoch:train:7801-7900batch: iter_time=8.772e-05, forward_time=0.148, loss_ctc=71.485, loss_interctc_layer6=79.613, loss_interctc_layer12=66.284, loss_interctc_layer15=60.604, loss_interctc_layer21=73.913, loss=70.380, backward_time=0.208, grad_norm=66.876, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.909e-05, train_time=1.439 +[gpua006:0/64] 2024-02-12 13:56:40,579 (trainer:756) INFO: 34epoch:train:7901-8000batch: iter_time=8.213e-05, forward_time=0.144, loss_ctc=79.325, loss_interctc_layer6=80.928, loss_interctc_layer12=66.876, loss_interctc_layer15=61.207, loss_interctc_layer21=82.209, loss=74.109, backward_time=0.209, grad_norm=84.368, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.909e-05, train_time=1.255 +[gpua006:0/64] 2024-02-12 13:58:29,951 (trainer:756) INFO: 34epoch:train:8001-8100batch: iter_time=8.872e-05, forward_time=0.143, loss_ctc=79.443, loss_interctc_layer6=79.678, loss_interctc_layer12=65.767, loss_interctc_layer15=60.086, loss_interctc_layer21=82.626, loss=73.520, backward_time=0.210, grad_norm=77.454, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.908e-05, train_time=1.095 +[gpua006:0/64] 2024-02-12 14:00:13,523 (trainer:756) INFO: 34epoch:train:8101-8200batch: iter_time=8.218e-05, forward_time=0.146, loss_ctc=74.493, loss_interctc_layer6=81.035, loss_interctc_layer12=66.887, loss_interctc_layer15=61.140, loss_interctc_layer21=77.077, loss=72.126, backward_time=0.210, grad_norm=72.858, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.907e-05, train_time=1.036 +[gpua006:0/64] 2024-02-12 14:03:02,012 (trainer:756) INFO: 34epoch:train:8201-8300batch: iter_time=8.299e-05, forward_time=0.232, loss_ctc=91.239, loss_interctc_layer6=91.899, loss_interctc_layer12=76.333, loss_interctc_layer15=70.093, loss_interctc_layer21=94.453, loss=84.803, backward_time=0.260, grad_norm=82.049, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.149, optim0_lr0=6.907e-05, train_time=1.685 +[gpua006:0/64] 2024-02-12 14:05:47,689 (trainer:756) INFO: 34epoch:train:8301-8400batch: iter_time=8.492e-05, forward_time=0.163, loss_ctc=83.442, loss_interctc_layer6=82.146, loss_interctc_layer12=67.785, loss_interctc_layer15=61.997, loss_interctc_layer21=86.976, loss=76.469, backward_time=0.213, grad_norm=95.352, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.906e-05, train_time=1.657 +[gpua006:0/64] 2024-02-12 14:07:57,308 (trainer:756) INFO: 34epoch:train:8401-8500batch: iter_time=8.040e-05, forward_time=0.143, loss_ctc=71.069, loss_interctc_layer6=82.480, loss_interctc_layer12=68.011, loss_interctc_layer15=62.119, loss_interctc_layer21=73.748, loss=71.486, backward_time=0.209, grad_norm=68.665, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.905e-05, train_time=1.296 +[gpua006:0/64] 2024-02-12 14:09:38,384 (trainer:756) INFO: 34epoch:train:8501-8600batch: iter_time=8.200e-05, forward_time=0.144, loss_ctc=67.401, loss_interctc_layer6=75.587, loss_interctc_layer12=62.313, loss_interctc_layer15=56.899, loss_interctc_layer21=69.632, loss=66.366, backward_time=0.210, grad_norm=99.715, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.905e-05, train_time=1.010 +[gpua006:0/64] 2024-02-12 14:11:48,378 (trainer:756) INFO: 34epoch:train:8601-8700batch: iter_time=8.024e-05, forward_time=0.143, loss_ctc=73.187, loss_interctc_layer6=76.374, loss_interctc_layer12=63.200, loss_interctc_layer15=57.912, loss_interctc_layer21=75.737, loss=69.282, backward_time=0.209, grad_norm=71.283, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.904e-05, train_time=1.300 +[gpua006:0/64] 2024-02-12 14:13:05,960 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-12 14:13:25,095 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 14:13:28,588 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 14:13:28,588 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-12 14:13:28,591 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 14:19:05,654 (trainer:756) INFO: 34epoch:train:8701-8800batch: iter_time=3.137, forward_time=0.173, loss_ctc=75.390, loss_interctc_layer6=77.912, loss_interctc_layer12=64.167, loss_interctc_layer15=58.520, loss_interctc_layer21=78.363, loss=70.870, backward_time=0.217, grad_norm=65.902, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.903e-05, train_time=4.373 +[gpua006:0/64] 2024-02-12 14:20:43,023 (trainer:756) INFO: 34epoch:train:8801-8900batch: iter_time=7.951e-05, forward_time=0.143, loss_ctc=82.140, loss_interctc_layer6=85.406, loss_interctc_layer12=70.983, loss_interctc_layer15=65.082, loss_interctc_layer21=85.176, loss=77.757, backward_time=0.209, grad_norm=122.534, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.902e-05, train_time=0.973 +[gpua006:0/64] 2024-02-12 14:22:50,741 (trainer:756) INFO: 34epoch:train:8901-9000batch: iter_time=8.194e-05, forward_time=0.142, loss_ctc=63.345, loss_interctc_layer6=78.318, loss_interctc_layer12=64.879, loss_interctc_layer15=59.448, loss_interctc_layer21=65.390, loss=66.276, backward_time=0.208, grad_norm=108.004, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.902e-05, train_time=1.277 +[gpua006:0/64] 2024-02-12 14:25:05,931 (trainer:756) INFO: 34epoch:train:9001-9100batch: iter_time=8.395e-05, forward_time=0.143, loss_ctc=82.400, loss_interctc_layer6=94.594, loss_interctc_layer12=78.248, loss_interctc_layer15=71.856, loss_interctc_layer21=85.255, loss=82.471, backward_time=0.209, grad_norm=105.811, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.901e-05, train_time=1.351 +[gpua006:0/64] 2024-02-12 14:27:24,653 (trainer:756) INFO: 34epoch:train:9101-9200batch: iter_time=8.653e-05, forward_time=0.143, loss_ctc=65.026, loss_interctc_layer6=74.478, loss_interctc_layer12=61.733, loss_interctc_layer15=56.659, loss_interctc_layer21=67.308, loss=65.041, backward_time=0.209, grad_norm=66.263, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.900e-05, train_time=1.388 +[gpua006:0/64] 2024-02-12 14:29:15,525 (trainer:756) INFO: 34epoch:train:9201-9300batch: iter_time=8.626e-05, forward_time=0.143, loss_ctc=78.227, loss_interctc_layer6=81.534, loss_interctc_layer12=66.989, loss_interctc_layer15=61.058, loss_interctc_layer21=81.466, loss=73.855, backward_time=0.208, grad_norm=72.865, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.900e-05, train_time=1.108 +[gpua006:0/64] 2024-02-12 14:32:28,778 (trainer:756) INFO: 34epoch:train:9301-9400batch: iter_time=8.337e-05, forward_time=0.279, loss_ctc=66.225, loss_interctc_layer6=75.497, loss_interctc_layer12=62.007, loss_interctc_layer15=56.494, loss_interctc_layer21=68.741, loss=65.793, backward_time=0.240, grad_norm=80.249, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.154, optim0_lr0=6.899e-05, train_time=1.933 +[gpua006:0/64] 2024-02-12 14:34:59,295 (trainer:756) INFO: 34epoch:train:9401-9500batch: iter_time=8.066e-05, forward_time=0.144, loss_ctc=73.072, loss_interctc_layer6=87.917, loss_interctc_layer12=73.293, loss_interctc_layer15=67.394, loss_interctc_layer21=75.295, loss=75.394, backward_time=0.208, grad_norm=66.103, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.898e-05, train_time=1.505 +[gpua006:0/64] 2024-02-12 14:37:40,452 (trainer:756) INFO: 34epoch:train:9501-9600batch: iter_time=8.180e-05, forward_time=0.143, loss_ctc=87.711, loss_interctc_layer6=84.939, loss_interctc_layer12=70.291, loss_interctc_layer15=64.345, loss_interctc_layer21=91.322, loss=79.722, backward_time=0.207, grad_norm=60.208, clip=100.000, loss_scale=1.775e+31, optim_step_time=0.136, optim0_lr0=6.898e-05, train_time=1.611 +[gpua006:0/64] 2024-02-12 14:39:36,868 (trainer:756) INFO: 34epoch:train:9601-9700batch: iter_time=8.019e-05, forward_time=0.143, loss_ctc=76.926, loss_interctc_layer6=88.351, loss_interctc_layer12=72.881, loss_interctc_layer15=66.724, loss_interctc_layer21=80.139, loss=77.004, backward_time=0.208, grad_norm=88.998, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.897e-05, train_time=1.164 +[gpua006:0/64] 2024-02-12 14:41:18,779 (trainer:756) INFO: 34epoch:train:9701-9800batch: iter_time=8.137e-05, forward_time=0.142, loss_ctc=65.781, loss_interctc_layer6=75.127, loss_interctc_layer12=61.717, loss_interctc_layer15=56.299, loss_interctc_layer21=68.225, loss=65.430, backward_time=0.210, grad_norm=113.172, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.896e-05, train_time=1.018 +[gpua006:0/64] 2024-02-12 14:43:07,605 (trainer:756) INFO: 34epoch:train:9801-9900batch: iter_time=8.439e-05, forward_time=0.142, loss_ctc=71.643, loss_interctc_layer6=72.950, loss_interctc_layer12=59.971, loss_interctc_layer15=54.722, loss_interctc_layer21=74.261, loss=66.709, backward_time=0.208, grad_norm=68.693, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.896e-05, train_time=1.089 +[gpua006:0/64] 2024-02-12 14:45:06,757 (trainer:756) INFO: 34epoch:train:9901-10000batch: iter_time=7.993e-05, forward_time=0.142, loss_ctc=67.349, loss_interctc_layer6=78.729, loss_interctc_layer12=65.244, loss_interctc_layer15=59.841, loss_interctc_layer21=69.803, loss=68.193, backward_time=0.209, grad_norm=66.758, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.895e-05, train_time=1.191 +[gpua006:0/64] 2024-02-12 14:45:26,787 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-12 14:45:45,582 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 14:45:49,300 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 14:45:49,300 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-12 14:45:49,303 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 14:52:20,778 (trainer:756) INFO: 34epoch:train:10001-10100batch: iter_time=3.277, forward_time=0.144, loss_ctc=85.205, loss_interctc_layer6=83.462, loss_interctc_layer12=68.960, loss_interctc_layer15=62.988, loss_interctc_layer21=88.352, loss=77.793, backward_time=0.210, grad_norm=85.209, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.894e-05, train_time=4.339 +[gpua006:0/64] 2024-02-12 14:54:08,786 (trainer:756) INFO: 34epoch:train:10101-10200batch: iter_time=8.074e-05, forward_time=0.144, loss_ctc=70.801, loss_interctc_layer6=80.117, loss_interctc_layer12=66.421, loss_interctc_layer15=60.782, loss_interctc_layer21=73.310, loss=70.286, backward_time=0.209, grad_norm=74.416, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.894e-05, train_time=1.080 +[gpua006:0/64] 2024-02-12 14:56:33,021 (trainer:756) INFO: 34epoch:train:10201-10300batch: iter_time=8.862e-05, forward_time=0.221, loss_ctc=77.556, loss_interctc_layer6=88.530, loss_interctc_layer12=73.556, loss_interctc_layer15=67.543, loss_interctc_layer21=80.172, loss=77.471, backward_time=0.282, grad_norm=83.502, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=6.893e-05, train_time=1.442 +[gpua006:0/64] 2024-02-12 14:58:14,053 (trainer:756) INFO: 34epoch:train:10301-10400batch: iter_time=8.791e-05, forward_time=0.142, loss_ctc=71.087, loss_interctc_layer6=79.442, loss_interctc_layer12=65.851, loss_interctc_layer15=60.478, loss_interctc_layer21=73.550, loss=70.081, backward_time=0.209, grad_norm=80.516, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.892e-05, train_time=1.010 +[gpua006:0/64] 2024-02-12 15:00:01,872 (trainer:756) INFO: 34epoch:train:10401-10500batch: iter_time=8.373e-05, forward_time=0.142, loss_ctc=79.218, loss_interctc_layer6=80.552, loss_interctc_layer12=66.470, loss_interctc_layer15=60.760, loss_interctc_layer21=82.205, loss=73.841, backward_time=0.209, grad_norm=72.174, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.892e-05, train_time=1.078 +[gpua006:0/64] 2024-02-12 15:02:26,477 (trainer:756) INFO: 34epoch:train:10501-10600batch: iter_time=8.997e-05, forward_time=0.145, loss_ctc=78.898, loss_interctc_layer6=79.055, loss_interctc_layer12=65.220, loss_interctc_layer15=59.564, loss_interctc_layer21=82.060, loss=72.959, backward_time=0.209, grad_norm=84.678, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.891e-05, train_time=1.445 +[gpua006:0/64] 2024-02-12 15:04:56,343 (trainer:756) INFO: 34epoch:train:10601-10700batch: iter_time=8.883e-05, forward_time=0.143, loss_ctc=74.365, loss_interctc_layer6=80.793, loss_interctc_layer12=66.625, loss_interctc_layer15=60.789, loss_interctc_layer21=77.082, loss=71.931, backward_time=0.208, grad_norm=84.711, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.890e-05, train_time=1.499 +[gpua006:0/64] 2024-02-12 15:07:52,456 (trainer:756) INFO: 34epoch:train:10701-10800batch: iter_time=9.650e-05, forward_time=0.143, loss_ctc=90.734, loss_interctc_layer6=91.157, loss_interctc_layer12=75.661, loss_interctc_layer15=69.418, loss_interctc_layer21=94.123, loss=84.219, backward_time=0.208, grad_norm=91.559, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.890e-05, train_time=1.761 +[gpua006:0/64] 2024-02-12 15:10:22,515 (trainer:756) INFO: 34epoch:train:10801-10900batch: iter_time=9.115e-05, forward_time=0.142, loss_ctc=84.069, loss_interctc_layer6=82.140, loss_interctc_layer12=67.750, loss_interctc_layer15=61.945, loss_interctc_layer21=87.664, loss=76.714, backward_time=0.207, grad_norm=73.683, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.889e-05, train_time=1.500 +[gpua006:0/64] 2024-02-12 15:12:19,771 (trainer:756) INFO: 34epoch:train:10901-11000batch: iter_time=9.111e-05, forward_time=0.144, loss_ctc=70.714, loss_interctc_layer6=81.959, loss_interctc_layer12=67.489, loss_interctc_layer15=61.632, loss_interctc_layer21=73.324, loss=71.023, backward_time=0.208, grad_norm=65.803, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.888e-05, train_time=1.173 +[gpua006:0/64] 2024-02-12 15:14:18,895 (trainer:756) INFO: 34epoch:train:11001-11100batch: iter_time=8.935e-05, forward_time=0.227, loss_ctc=66.907, loss_interctc_layer6=75.306, loss_interctc_layer12=61.906, loss_interctc_layer15=56.591, loss_interctc_layer21=69.272, loss=65.997, backward_time=0.259, grad_norm=123.797, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.887e-05, train_time=1.191 +[gpua006:0/64] 2024-02-12 15:16:08,939 (trainer:756) INFO: 34epoch:train:11101-11200batch: iter_time=9.134e-05, forward_time=0.142, loss_ctc=73.672, loss_interctc_layer6=76.071, loss_interctc_layer12=62.979, loss_interctc_layer15=57.641, loss_interctc_layer21=76.074, loss=69.287, backward_time=0.209, grad_norm=64.023, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.887e-05, train_time=1.101 +[gpua006:0/64] 2024-02-12 15:17:21,641 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-12 15:17:40,421 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 15:17:44,144 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 15:17:44,145 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-12 15:17:44,148 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 15:23:27,540 (trainer:756) INFO: 34epoch:train:11201-11300batch: iter_time=3.222, forward_time=0.143, loss_ctc=75.947, loss_interctc_layer6=77.600, loss_interctc_layer12=63.892, loss_interctc_layer15=58.307, loss_interctc_layer21=78.810, loss=70.911, backward_time=0.209, grad_norm=72.527, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.886e-05, train_time=4.386 +[gpua006:0/64] 2024-02-12 15:25:18,925 (trainer:756) INFO: 34epoch:train:11301-11400batch: iter_time=9.078e-05, forward_time=0.144, loss_ctc=86.230, loss_interctc_layer6=85.573, loss_interctc_layer12=70.949, loss_interctc_layer15=65.105, loss_interctc_layer21=89.326, loss=79.437, backward_time=0.209, grad_norm=88.338, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.885e-05, train_time=1.114 +[gpua006:0/64] 2024-02-12 15:26:56,632 (trainer:756) INFO: 34epoch:train:11401-11500batch: iter_time=8.740e-05, forward_time=0.142, loss_ctc=65.647, loss_interctc_layer6=78.839, loss_interctc_layer12=65.225, loss_interctc_layer15=59.754, loss_interctc_layer21=67.845, loss=67.462, backward_time=0.209, grad_norm=76.951, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.885e-05, train_time=0.977 +[gpua006:0/64] 2024-02-12 15:29:04,956 (trainer:756) INFO: 34epoch:train:11501-11600batch: iter_time=9.801e-05, forward_time=0.225, loss_ctc=84.999, loss_interctc_layer6=93.905, loss_interctc_layer12=77.871, loss_interctc_layer15=71.506, loss_interctc_layer21=87.817, loss=83.219, backward_time=0.233, grad_norm=89.626, clip=100.000, loss_scale=3.549e+31, optim_step_time=0.146, optim0_lr0=6.884e-05, train_time=1.283 +[gpua006:0/64] 2024-02-12 15:31:00,800 (trainer:756) INFO: 34epoch:train:11601-11700batch: iter_time=8.536e-05, forward_time=0.183, loss_ctc=66.231, loss_interctc_layer6=74.289, loss_interctc_layer12=61.504, loss_interctc_layer15=56.385, loss_interctc_layer21=68.553, loss=65.392, backward_time=0.229, grad_norm=70.682, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=6.883e-05, train_time=1.158 +[gpua006:0/64] 2024-02-12 15:33:03,586 (trainer:756) INFO: 34epoch:train:11701-11800batch: iter_time=8.389e-05, forward_time=0.144, loss_ctc=84.432, loss_interctc_layer6=82.181, loss_interctc_layer12=67.561, loss_interctc_layer15=61.803, loss_interctc_layer21=87.902, loss=76.776, backward_time=0.209, grad_norm=85.987, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.883e-05, train_time=1.228 +[gpua006:0/64] 2024-02-12 15:35:29,595 (trainer:756) INFO: 34epoch:train:11801-11900batch: iter_time=8.106e-05, forward_time=0.155, loss_ctc=67.918, loss_interctc_layer6=74.732, loss_interctc_layer12=61.324, loss_interctc_layer15=55.885, loss_interctc_layer21=70.507, loss=66.073, backward_time=0.209, grad_norm=66.373, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.882e-05, train_time=1.459 +[gpua006:0/64] 2024-02-12 15:37:40,830 (trainer:756) INFO: 34epoch:train:11901-12000batch: iter_time=8.011e-05, forward_time=0.145, loss_ctc=76.743, loss_interctc_layer6=87.871, loss_interctc_layer12=73.179, loss_interctc_layer15=67.166, loss_interctc_layer21=79.210, loss=76.834, backward_time=0.209, grad_norm=73.607, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.881e-05, train_time=1.313 +[gpua006:0/64] 2024-02-12 15:40:16,886 (trainer:756) INFO: 34epoch:train:12001-12100batch: iter_time=8.498e-05, forward_time=0.161, loss_ctc=93.739, loss_interctc_layer6=84.671, loss_interctc_layer12=69.933, loss_interctc_layer15=63.956, loss_interctc_layer21=97.584, loss=81.977, backward_time=0.261, grad_norm=77.500, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=6.881e-05, train_time=1.560 +[gpua006:0/64] 2024-02-12 15:42:37,941 (trainer:756) INFO: 34epoch:train:12101-12200batch: iter_time=8.478e-05, forward_time=0.200, loss_ctc=84.083, loss_interctc_layer6=88.777, loss_interctc_layer12=73.338, loss_interctc_layer15=67.171, loss_interctc_layer21=87.473, loss=80.169, backward_time=0.217, grad_norm=77.288, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.880e-05, train_time=1.410 +[gpua006:0/64] 2024-02-12 15:45:06,944 (trainer:756) INFO: 34epoch:train:12201-12300batch: iter_time=8.290e-05, forward_time=0.143, loss_ctc=67.702, loss_interctc_layer6=75.018, loss_interctc_layer12=61.638, loss_interctc_layer15=56.238, loss_interctc_layer21=70.126, loss=66.144, backward_time=0.208, grad_norm=63.332, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.879e-05, train_time=1.490 +[gpua006:0/64] 2024-02-12 15:47:23,146 (trainer:756) INFO: 34epoch:train:12301-12400batch: iter_time=8.570e-05, forward_time=0.143, loss_ctc=70.623, loss_interctc_layer6=71.925, loss_interctc_layer12=58.981, loss_interctc_layer15=53.793, loss_interctc_layer21=73.342, loss=65.733, backward_time=0.209, grad_norm=58.093, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.879e-05, train_time=1.362 +[gpua006:0/64] 2024-02-12 15:49:51,566 (trainer:756) INFO: 34epoch:train:12401-12500batch: iter_time=8.241e-05, forward_time=0.143, loss_ctc=71.276, loss_interctc_layer6=78.972, loss_interctc_layer12=65.380, loss_interctc_layer15=59.852, loss_interctc_layer21=73.748, loss=69.846, backward_time=0.209, grad_norm=64.679, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.878e-05, train_time=1.483 +[gpua006:0/64] 2024-02-12 15:50:11,613 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-12 15:50:30,954 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 15:50:34,864 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 15:50:34,864 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-12 15:50:34,867 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 15:56:55,388 (trainer:756) INFO: 34epoch:train:12501-12600batch: iter_time=3.106, forward_time=0.184, loss_ctc=83.831, loss_interctc_layer6=83.347, loss_interctc_layer12=68.805, loss_interctc_layer15=62.898, loss_interctc_layer21=87.031, loss=77.182, backward_time=0.220, grad_norm=79.686, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=6.877e-05, train_time=4.239 +[gpua006:0/64] 2024-02-12 15:58:31,271 (trainer:756) INFO: 34epoch:train:12601-12700batch: iter_time=8.345e-05, forward_time=0.143, loss_ctc=66.107, loss_interctc_layer6=80.199, loss_interctc_layer12=66.356, loss_interctc_layer15=60.740, loss_interctc_layer21=68.564, loss=68.393, backward_time=0.210, grad_norm=67.638, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.877e-05, train_time=0.959 +[gpua006:0/64] 2024-02-12 16:00:34,091 (trainer:756) INFO: 34epoch:train:12701-12800batch: iter_time=8.276e-05, forward_time=0.143, loss_ctc=75.525, loss_interctc_layer6=88.225, loss_interctc_layer12=73.253, loss_interctc_layer15=67.370, loss_interctc_layer21=77.968, loss=76.468, backward_time=0.209, grad_norm=83.375, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.876e-05, train_time=1.228 +[gpua006:0/64] 2024-02-12 16:02:32,352 (trainer:756) INFO: 34epoch:train:12801-12900batch: iter_time=8.360e-05, forward_time=0.143, loss_ctc=68.212, loss_interctc_layer6=79.241, loss_interctc_layer12=65.805, loss_interctc_layer15=60.327, loss_interctc_layer21=70.603, loss=68.838, backward_time=0.209, grad_norm=66.018, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.875e-05, train_time=1.182 +[gpua006:0/64] 2024-02-12 16:05:05,717 (trainer:756) INFO: 34epoch:train:12901-13000batch: iter_time=8.417e-05, forward_time=0.222, loss_ctc=76.021, loss_interctc_layer6=79.767, loss_interctc_layer12=65.920, loss_interctc_layer15=60.165, loss_interctc_layer21=79.012, loss=72.177, backward_time=0.260, grad_norm=84.808, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.151, optim0_lr0=6.875e-05, train_time=1.533 +[gpua006:0/64] 2024-02-12 16:07:21,358 (trainer:756) INFO: 34epoch:train:13001-13100batch: iter_time=8.390e-05, forward_time=0.144, loss_ctc=73.692, loss_interctc_layer6=79.575, loss_interctc_layer12=65.785, loss_interctc_layer15=60.245, loss_interctc_layer21=76.700, loss=71.199, backward_time=0.209, grad_norm=68.757, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.874e-05, train_time=1.357 +[gpua006:0/64] 2024-02-12 16:10:06,198 (trainer:756) INFO: 34epoch:train:13101-13200batch: iter_time=8.333e-05, forward_time=0.145, loss_ctc=70.886, loss_interctc_layer6=80.157, loss_interctc_layer12=66.070, loss_interctc_layer15=60.225, loss_interctc_layer21=73.470, loss=70.162, backward_time=0.207, grad_norm=75.941, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.873e-05, train_time=1.648 +[gpua006:0/64] 2024-02-12 16:12:42,394 (trainer:756) INFO: 34epoch:train:13201-13300batch: iter_time=8.636e-05, forward_time=0.144, loss_ctc=82.911, loss_interctc_layer6=90.951, loss_interctc_layer12=75.471, loss_interctc_layer15=69.225, loss_interctc_layer21=85.935, loss=80.899, backward_time=0.208, grad_norm=73.214, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.873e-05, train_time=1.562 +[gpua006:0/64] 2024-02-12 16:14:25,070 (trainer:756) INFO: 34epoch:train:13301-13400batch: iter_time=8.285e-05, forward_time=0.142, loss_ctc=77.970, loss_interctc_layer6=81.812, loss_interctc_layer12=67.458, loss_interctc_layer15=61.728, loss_interctc_layer21=81.218, loss=74.037, backward_time=0.209, grad_norm=72.122, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.872e-05, train_time=1.027 +[gpua006:0/64] 2024-02-12 16:17:05,789 (trainer:756) INFO: 34epoch:train:13401-13500batch: iter_time=8.160e-05, forward_time=0.214, loss_ctc=67.834, loss_interctc_layer6=82.035, loss_interctc_layer12=67.583, loss_interctc_layer15=61.669, loss_interctc_layer21=70.384, loss=69.901, backward_time=0.282, grad_norm=60.635, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.146, optim0_lr0=6.871e-05, train_time=1.606 +[gpua006:0/64] 2024-02-12 16:17:36,939 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 16:19:06,970 (trainer:756) INFO: 34epoch:train:13501-13600batch: iter_time=8.488e-05, forward_time=0.143, loss_ctc=66.198, loss_interctc_layer6=75.408, loss_interctc_layer12=62.048, loss_interctc_layer15=56.900, loss_interctc_layer21=68.590, loss=65.829, backward_time=0.208, grad_norm=69.479, clip=100.000, loss_scale=2.479e+31, optim_step_time=0.136, optim0_lr0=6.871e-05, train_time=1.213 +[gpua006:0/64] 2024-02-12 16:21:05,386 (trainer:756) INFO: 34epoch:train:13601-13700batch: iter_time=8.346e-05, forward_time=0.142, loss_ctc=71.091, loss_interctc_layer6=75.824, loss_interctc_layer12=62.650, loss_interctc_layer15=57.340, loss_interctc_layer21=73.603, loss=68.102, backward_time=0.209, grad_norm=65.349, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.870e-05, train_time=1.184 +[gpua006:0/64] 2024-02-12 16:22:23,148 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-12 16:22:42,179 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 16:22:45,663 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 16:22:45,663 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-12 16:22:45,667 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 16:28:07,019 (trainer:756) INFO: 34epoch:train:13701-13800batch: iter_time=3.013, forward_time=0.142, loss_ctc=73.189, loss_interctc_layer6=77.844, loss_interctc_layer12=64.156, loss_interctc_layer15=58.460, loss_interctc_layer21=76.090, loss=69.948, backward_time=0.208, grad_norm=72.835, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.869e-05, train_time=4.216 +[gpua006:0/64] 2024-02-12 16:29:44,493 (trainer:756) INFO: 34epoch:train:13801-13900batch: iter_time=8.596e-05, forward_time=0.144, loss_ctc=81.184, loss_interctc_layer6=85.181, loss_interctc_layer12=70.576, loss_interctc_layer15=64.690, loss_interctc_layer21=84.189, loss=77.164, backward_time=0.211, grad_norm=75.757, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.868e-05, train_time=0.975 +[gpua006:0/64] 2024-02-12 16:31:37,356 (trainer:756) INFO: 34epoch:train:13901-14000batch: iter_time=8.095e-05, forward_time=0.143, loss_ctc=63.073, loss_interctc_layer6=77.565, loss_interctc_layer12=64.315, loss_interctc_layer15=58.821, loss_interctc_layer21=65.145, loss=65.784, backward_time=0.209, grad_norm=83.511, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.868e-05, train_time=1.128 +[gpua006:0/64] 2024-02-12 16:33:50,922 (trainer:756) INFO: 34epoch:train:14001-14100batch: iter_time=8.265e-05, forward_time=0.276, loss_ctc=81.822, loss_interctc_layer6=94.138, loss_interctc_layer12=77.954, loss_interctc_layer15=71.439, loss_interctc_layer21=84.291, loss=81.929, backward_time=0.242, grad_norm=100.986, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.153, optim0_lr0=6.867e-05, train_time=1.335 +[gpua006:0/64] 2024-02-12 16:36:21,997 (trainer:756) INFO: 34epoch:train:14101-14200batch: iter_time=8.721e-05, forward_time=0.143, loss_ctc=63.715, loss_interctc_layer6=73.562, loss_interctc_layer12=61.015, loss_interctc_layer15=55.800, loss_interctc_layer21=65.949, loss=64.008, backward_time=0.208, grad_norm=64.791, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.866e-05, train_time=1.509 +[gpua006:0/64] 2024-02-12 16:39:05,734 (trainer:756) INFO: 34epoch:train:14201-14300batch: iter_time=8.502e-05, forward_time=0.144, loss_ctc=77.552, loss_interctc_layer6=81.266, loss_interctc_layer12=66.665, loss_interctc_layer15=60.682, loss_interctc_layer21=80.914, loss=73.416, backward_time=0.208, grad_norm=89.504, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.866e-05, train_time=1.639 +[gpua006:0/64] 2024-02-12 16:40:58,417 (trainer:756) INFO: 34epoch:train:14301-14400batch: iter_time=8.415e-05, forward_time=0.142, loss_ctc=65.344, loss_interctc_layer6=74.676, loss_interctc_layer12=61.283, loss_interctc_layer15=55.877, loss_interctc_layer21=67.789, loss=64.994, backward_time=0.209, grad_norm=133.521, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.865e-05, train_time=1.127 +[gpua006:0/64] 2024-02-12 16:43:39,637 (trainer:756) INFO: 34epoch:train:14401-14500batch: iter_time=9.819e-05, forward_time=0.143, loss_ctc=72.643, loss_interctc_layer6=87.242, loss_interctc_layer12=72.544, loss_interctc_layer15=66.658, loss_interctc_layer21=74.966, loss=74.811, backward_time=0.208, grad_norm=67.599, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.864e-05, train_time=1.612 +[gpua006:0/64] 2024-02-12 16:43:47,933 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 16:45:49,054 (trainer:756) INFO: 34epoch:train:14501-14600batch: iter_time=8.866e-05, forward_time=0.145, loss_ctc=87.966, loss_interctc_layer6=84.520, loss_interctc_layer12=69.707, loss_interctc_layer15=63.694, loss_interctc_layer21=91.678, loss=79.513, backward_time=0.208, grad_norm=107.252, clip=100.000, loss_scale=1.086e+31, optim_step_time=0.136, optim0_lr0=6.864e-05, train_time=1.294 +[gpua006:0/64] 2024-02-12 16:48:07,147 (trainer:756) INFO: 34epoch:train:14601-14700batch: iter_time=8.145e-05, forward_time=0.145, loss_ctc=77.062, loss_interctc_layer6=88.539, loss_interctc_layer12=73.161, loss_interctc_layer15=66.931, loss_interctc_layer21=80.174, loss=77.173, backward_time=0.209, grad_norm=70.017, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.863e-05, train_time=1.381 +[gpua006:0/64] 2024-02-12 16:50:28,563 (trainer:756) INFO: 34epoch:train:14701-14800batch: iter_time=8.673e-05, forward_time=0.277, loss_ctc=66.094, loss_interctc_layer6=75.133, loss_interctc_layer12=61.718, loss_interctc_layer15=56.342, loss_interctc_layer21=68.458, loss=65.549, backward_time=0.253, grad_norm=64.931, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.146, optim0_lr0=6.862e-05, train_time=1.414 +[gpua006:0/64] 2024-02-12 16:52:10,929 (trainer:756) INFO: 34epoch:train:14801-14900batch: iter_time=8.970e-05, forward_time=0.142, loss_ctc=71.570, loss_interctc_layer6=72.404, loss_interctc_layer12=59.342, loss_interctc_layer15=54.044, loss_interctc_layer21=74.161, loss=66.304, backward_time=0.210, grad_norm=62.127, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.862e-05, train_time=1.023 +[gpua006:0/64] 2024-02-12 16:54:40,213 (trainer:756) INFO: 34epoch:train:14901-15000batch: iter_time=8.069e-05, forward_time=0.143, loss_ctc=66.358, loss_interctc_layer6=78.391, loss_interctc_layer12=64.735, loss_interctc_layer15=59.312, loss_interctc_layer21=68.721, loss=67.503, backward_time=0.209, grad_norm=63.319, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.861e-05, train_time=1.491 +[gpua006:0/64] 2024-02-12 17:26:29,713 (trainer:355) INFO: 34epoch results: [train] iter_time=0.240, forward_time=0.167, loss_ctc=75.505, loss_interctc_layer6=81.771, loss_interctc_layer12=67.670, loss_interctc_layer15=61.960, loss_interctc_layer21=78.273, loss=73.036, backward_time=0.219, grad_norm=81.518, clip=100.000, loss_scale=2.266e+31, optim_step_time=0.138, optim0_lr0=6.912e-05, train_time=1.533, time=6 hours, 23 minutes and 47.55 seconds, total_count=510000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=41.589, cer_ctc=0.196, loss_interctc_layer6=47.141, cer_interctc_layer6=0.210, loss_interctc_layer12=34.414, cer_interctc_layer12=0.145, loss_interctc_layer15=30.166, cer_interctc_layer15=0.120, loss_interctc_layer21=43.980, cer_interctc_layer21=0.205, loss=39.458, time=31 minutes and 25.63 seconds, total_count=158814, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-12 17:26:50,459 (trainer:410) INFO: The best model has been updated: valid.total_count +[gpua006:0/64] 2024-02-12 17:26:50,538 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/29epoch.pth +[gpua006:0/64] 2024-02-12 17:26:50,538 (trainer:289) INFO: 35/45epoch started. Estimated time to finish: 3 days, 5 hours and 29 minutes +[gpua006:0/64] 2024-02-12 17:26:50,556 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-12 17:27:08,775 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 17:27:12,267 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 17:27:12,267 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-12 17:27:12,271 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 17:33:40,528 (trainer:756) INFO: 35epoch:train:1-100batch: iter_time=2.876, forward_time=0.171, loss_ctc=56.218, loss_interctc_layer6=68.815, loss_interctc_layer12=56.705, loss_interctc_layer15=51.887, loss_interctc_layer21=58.033, loss=58.332, backward_time=0.220, grad_norm=66.361, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.860e-05, train_time=4.099 +[gpua006:0/64] 2024-02-12 17:35:16,776 (trainer:756) INFO: 35epoch:train:101-200batch: iter_time=8.587e-05, forward_time=0.146, loss_ctc=82.602, loss_interctc_layer6=91.163, loss_interctc_layer12=76.477, loss_interctc_layer15=70.720, loss_interctc_layer21=85.318, loss=81.256, backward_time=0.210, grad_norm=113.080, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.860e-05, train_time=0.962 +[gpua006:0/64] 2024-02-12 17:36:52,941 (trainer:756) INFO: 35epoch:train:201-300batch: iter_time=8.875e-05, forward_time=0.144, loss_ctc=92.237, loss_interctc_layer6=99.757, loss_interctc_layer12=83.475, loss_interctc_layer15=77.276, loss_interctc_layer21=95.286, loss=89.606, backward_time=0.209, grad_norm=103.517, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.859e-05, train_time=0.961 +[gpua006:0/64] 2024-02-12 17:38:30,833 (trainer:756) INFO: 35epoch:train:301-400batch: iter_time=9.391e-05, forward_time=0.143, loss_ctc=57.979, loss_interctc_layer6=72.039, loss_interctc_layer12=59.878, loss_interctc_layer15=55.013, loss_interctc_layer21=59.830, loss=60.948, backward_time=0.210, grad_norm=69.714, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.858e-05, train_time=0.979 +[gpua006:0/64] 2024-02-12 17:40:45,979 (trainer:756) INFO: 35epoch:train:401-500batch: iter_time=9.000e-05, forward_time=0.144, loss_ctc=75.832, loss_interctc_layer6=84.699, loss_interctc_layer12=69.909, loss_interctc_layer15=63.877, loss_interctc_layer21=78.647, loss=74.593, backward_time=0.209, grad_norm=67.801, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.858e-05, train_time=1.351 +[gpua006:0/64] 2024-02-12 17:42:37,483 (trainer:756) INFO: 35epoch:train:501-600batch: iter_time=8.659e-05, forward_time=0.146, loss_ctc=66.083, loss_interctc_layer6=72.028, loss_interctc_layer12=60.764, loss_interctc_layer15=56.091, loss_interctc_layer21=68.732, loss=64.740, backward_time=0.217, grad_norm=80.587, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.857e-05, train_time=1.111 +[gpua006:0/64] 2024-02-12 17:44:38,074 (trainer:756) INFO: 35epoch:train:601-700batch: iter_time=8.398e-05, forward_time=0.177, loss_ctc=63.966, loss_interctc_layer6=77.349, loss_interctc_layer12=64.695, loss_interctc_layer15=59.348, loss_interctc_layer21=66.238, loss=66.319, backward_time=0.230, grad_norm=64.484, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.856e-05, train_time=1.209 +[gpua006:0/64] 2024-02-12 17:46:35,189 (trainer:756) INFO: 35epoch:train:701-800batch: iter_time=8.795e-05, forward_time=0.176, loss_ctc=64.217, loss_interctc_layer6=69.958, loss_interctc_layer12=57.693, loss_interctc_layer15=52.716, loss_interctc_layer21=66.587, loss=62.234, backward_time=0.239, grad_norm=76.347, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.856e-05, train_time=1.171 +[gpua006:0/64] 2024-02-12 17:48:48,218 (trainer:756) INFO: 35epoch:train:801-900batch: iter_time=8.567e-05, forward_time=0.141, loss_ctc=65.013, loss_interctc_layer6=71.217, loss_interctc_layer12=59.205, loss_interctc_layer15=54.337, loss_interctc_layer21=67.505, loss=63.455, backward_time=0.206, grad_norm=64.967, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=6.855e-05, train_time=1.330 +[gpua006:0/64] 2024-02-12 17:50:58,027 (trainer:756) INFO: 35epoch:train:901-1000batch: iter_time=8.429e-05, forward_time=0.142, loss_ctc=79.116, loss_interctc_layer6=86.377, loss_interctc_layer12=71.384, loss_interctc_layer15=65.364, loss_interctc_layer21=82.170, loss=76.882, backward_time=0.207, grad_norm=73.064, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=6.854e-05, train_time=1.298 +[gpua006:0/64] 2024-02-12 17:53:23,287 (trainer:756) INFO: 35epoch:train:1001-1100batch: iter_time=8.890e-05, forward_time=0.142, loss_ctc=77.607, loss_interctc_layer6=78.945, loss_interctc_layer12=65.288, loss_interctc_layer15=59.686, loss_interctc_layer21=80.472, loss=72.400, backward_time=0.209, grad_norm=66.992, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=6.854e-05, train_time=1.452 +[gpua006:0/64] 2024-02-12 17:55:35,548 (trainer:756) INFO: 35epoch:train:1101-1200batch: iter_time=8.963e-05, forward_time=0.143, loss_ctc=71.376, loss_interctc_layer6=78.205, loss_interctc_layer12=64.779, loss_interctc_layer15=59.266, loss_interctc_layer21=74.024, loss=69.530, backward_time=0.207, grad_norm=69.817, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.135, optim0_lr0=6.853e-05, train_time=1.322 +[gpua006:0/64] 2024-02-12 17:56:55,827 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-12 17:57:14,990 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 17:57:18,490 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 17:57:18,490 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-12 17:57:18,494 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 18:02:47,504 (trainer:756) INFO: 35epoch:train:1201-1300batch: iter_time=2.492, forward_time=0.194, loss_ctc=63.130, loss_interctc_layer6=72.228, loss_interctc_layer12=59.354, loss_interctc_layer15=54.099, loss_interctc_layer21=65.394, loss=62.841, backward_time=0.217, grad_norm=139.043, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.852e-05, train_time=4.319 +[gpua006:0/64] 2024-02-12 18:04:23,499 (trainer:756) INFO: 35epoch:train:1301-1400batch: iter_time=8.054e-05, forward_time=0.143, loss_ctc=68.193, loss_interctc_layer6=78.315, loss_interctc_layer12=64.696, loss_interctc_layer15=59.293, loss_interctc_layer21=70.348, loss=68.169, backward_time=0.210, grad_norm=69.919, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.852e-05, train_time=0.960 +[gpua006:0/64] 2024-02-12 18:06:21,106 (trainer:756) INFO: 35epoch:train:1401-1500batch: iter_time=9.034e-04, forward_time=0.300, loss_ctc=87.537, loss_interctc_layer6=94.967, loss_interctc_layer12=79.170, loss_interctc_layer15=72.743, loss_interctc_layer21=90.789, loss=85.041, backward_time=0.244, grad_norm=113.896, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.851e-05, train_time=1.174 +[gpua006:0/64] 2024-02-12 18:08:39,122 (trainer:756) INFO: 35epoch:train:1501-1600batch: iter_time=8.344e-05, forward_time=0.143, loss_ctc=74.842, loss_interctc_layer6=85.124, loss_interctc_layer12=72.022, loss_interctc_layer15=66.104, loss_interctc_layer21=76.803, loss=74.979, backward_time=0.209, grad_norm=84.421, clip=100.000, loss_scale=1.947e+31, optim_step_time=0.136, optim0_lr0=6.850e-05, train_time=1.381 +[gpua006:0/64] 2024-02-12 18:10:33,172 (trainer:756) INFO: 35epoch:train:1601-1700batch: iter_time=8.284e-05, forward_time=0.144, loss_ctc=75.353, loss_interctc_layer6=82.110, loss_interctc_layer12=67.773, loss_interctc_layer15=61.868, loss_interctc_layer21=78.217, loss=73.064, backward_time=0.209, grad_norm=70.901, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.850e-05, train_time=1.141 +[gpua006:0/64] 2024-02-12 18:13:09,612 (trainer:756) INFO: 35epoch:train:1701-1800batch: iter_time=8.018e-05, forward_time=0.259, loss_ctc=63.348, loss_interctc_layer6=74.016, loss_interctc_layer12=61.256, loss_interctc_layer15=55.926, loss_interctc_layer21=65.196, loss=63.948, backward_time=0.245, grad_norm=64.985, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.145, optim0_lr0=6.849e-05, train_time=1.561 +[gpua006:0/64] 2024-02-12 18:15:06,666 (trainer:756) INFO: 35epoch:train:1801-1900batch: iter_time=8.206e-05, forward_time=0.142, loss_ctc=67.247, loss_interctc_layer6=70.752, loss_interctc_layer12=58.355, loss_interctc_layer15=53.349, loss_interctc_layer21=69.775, loss=63.896, backward_time=0.208, grad_norm=79.483, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.848e-05, train_time=1.173 +[gpua006:0/64] 2024-02-12 18:17:06,838 (trainer:756) INFO: 35epoch:train:1901-2000batch: iter_time=8.281e-05, forward_time=0.142, loss_ctc=61.590, loss_interctc_layer6=78.109, loss_interctc_layer12=64.982, loss_interctc_layer15=59.483, loss_interctc_layer21=63.732, loss=65.579, backward_time=0.208, grad_norm=77.004, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.848e-05, train_time=1.201 +[gpua006:0/64] 2024-02-12 18:19:24,964 (trainer:756) INFO: 35epoch:train:2001-2100batch: iter_time=8.335e-05, forward_time=0.143, loss_ctc=61.615, loss_interctc_layer6=70.400, loss_interctc_layer12=58.304, loss_interctc_layer15=53.294, loss_interctc_layer21=63.923, loss=61.507, backward_time=0.208, grad_norm=68.238, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.847e-05, train_time=1.381 +[gpua006:0/64] 2024-02-12 18:21:46,391 (trainer:756) INFO: 35epoch:train:2101-2200batch: iter_time=8.214e-05, forward_time=0.155, loss_ctc=77.172, loss_interctc_layer6=81.617, loss_interctc_layer12=67.705, loss_interctc_layer15=62.039, loss_interctc_layer21=80.096, loss=73.726, backward_time=0.208, grad_norm=75.078, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.846e-05, train_time=1.414 +[gpua006:0/64] 2024-02-12 18:23:54,138 (trainer:756) INFO: 35epoch:train:2201-2300batch: iter_time=8.463e-05, forward_time=0.157, loss_ctc=71.287, loss_interctc_layer6=79.197, loss_interctc_layer12=65.342, loss_interctc_layer15=59.573, loss_interctc_layer21=73.965, loss=69.873, backward_time=0.210, grad_norm=94.540, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.846e-05, train_time=1.277 +[gpua006:0/64] 2024-02-12 18:26:17,896 (trainer:756) INFO: 35epoch:train:2301-2400batch: iter_time=8.269e-05, forward_time=0.202, loss_ctc=85.743, loss_interctc_layer6=80.696, loss_interctc_layer12=66.629, loss_interctc_layer15=60.870, loss_interctc_layer21=89.112, loss=76.610, backward_time=0.321, grad_norm=99.652, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=6.845e-05, train_time=1.436 +[gpua006:0/64] 2024-02-12 18:28:04,538 (trainer:756) INFO: 35epoch:train:2401-2500batch: iter_time=8.297e-05, forward_time=0.143, loss_ctc=69.578, loss_interctc_layer6=77.700, loss_interctc_layer12=63.970, loss_interctc_layer15=58.453, loss_interctc_layer21=72.174, loss=68.375, backward_time=0.208, grad_norm=110.713, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.844e-05, train_time=1.068 +[gpua006:0/64] 2024-02-12 18:28:24,568 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-12 18:28:43,716 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 18:28:47,189 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 18:28:47,189 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-12 18:28:47,192 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 18:35:54,898 (trainer:756) INFO: 35epoch:train:2501-2600batch: iter_time=3.366, forward_time=0.142, loss_ctc=54.850, loss_interctc_layer6=67.681, loss_interctc_layer12=55.632, loss_interctc_layer15=50.861, loss_interctc_layer21=56.713, loss=57.147, backward_time=0.208, grad_norm=60.540, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.844e-05, train_time=4.703 +[gpua006:0/64] 2024-02-12 18:37:41,720 (trainer:756) INFO: 35epoch:train:2601-2700batch: iter_time=8.268e-05, forward_time=0.143, loss_ctc=80.493, loss_interctc_layer6=89.126, loss_interctc_layer12=74.319, loss_interctc_layer15=68.416, loss_interctc_layer21=83.239, loss=79.118, backward_time=0.209, grad_norm=105.242, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.843e-05, train_time=1.067 +[gpua006:0/64] 2024-02-12 18:39:35,699 (trainer:756) INFO: 35epoch:train:2701-2800batch: iter_time=8.735e-05, forward_time=0.144, loss_ctc=92.968, loss_interctc_layer6=99.722, loss_interctc_layer12=83.873, loss_interctc_layer15=77.371, loss_interctc_layer21=96.410, loss=90.069, backward_time=0.212, grad_norm=92.880, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.842e-05, train_time=1.140 +[gpua006:0/64] 2024-02-12 18:41:46,663 (trainer:756) INFO: 35epoch:train:2801-2900batch: iter_time=8.950e-05, forward_time=0.273, loss_ctc=57.363, loss_interctc_layer6=71.417, loss_interctc_layer12=59.132, loss_interctc_layer15=54.312, loss_interctc_layer21=59.262, loss=60.297, backward_time=0.259, grad_norm=69.057, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.842e-05, train_time=1.309 +[gpua006:0/64] 2024-02-12 18:43:54,721 (trainer:756) INFO: 35epoch:train:2901-3000batch: iter_time=9.071e-05, forward_time=0.143, loss_ctc=74.618, loss_interctc_layer6=83.985, loss_interctc_layer12=69.081, loss_interctc_layer15=62.960, loss_interctc_layer21=77.294, loss=73.588, backward_time=0.209, grad_norm=76.443, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.841e-05, train_time=1.278 +[gpua006:0/64] 2024-02-12 18:45:51,704 (trainer:756) INFO: 35epoch:train:3001-3100batch: iter_time=8.287e-05, forward_time=0.154, loss_ctc=64.357, loss_interctc_layer6=70.835, loss_interctc_layer12=58.657, loss_interctc_layer15=53.729, loss_interctc_layer21=66.727, loss=62.861, backward_time=0.208, grad_norm=76.094, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.840e-05, train_time=1.172 +[gpua006:0/64] 2024-02-12 18:47:38,161 (trainer:756) INFO: 35epoch:train:3101-3200batch: iter_time=8.883e-05, forward_time=0.142, loss_ctc=64.018, loss_interctc_layer6=77.033, loss_interctc_layer12=64.008, loss_interctc_layer15=58.674, loss_interctc_layer21=66.237, loss=65.994, backward_time=0.209, grad_norm=93.125, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.840e-05, train_time=1.064 +[gpua006:0/64] 2024-02-12 18:49:43,392 (trainer:756) INFO: 35epoch:train:3201-3300batch: iter_time=8.581e-05, forward_time=0.141, loss_ctc=64.224, loss_interctc_layer6=70.010, loss_interctc_layer12=57.622, loss_interctc_layer15=52.629, loss_interctc_layer21=66.607, loss=62.218, backward_time=0.208, grad_norm=73.585, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.839e-05, train_time=1.252 +[gpua006:0/64] 2024-02-12 18:52:21,981 (trainer:756) INFO: 35epoch:train:3301-3400batch: iter_time=8.181e-05, forward_time=0.146, loss_ctc=64.428, loss_interctc_layer6=70.448, loss_interctc_layer12=58.321, loss_interctc_layer15=53.314, loss_interctc_layer21=66.955, loss=62.693, backward_time=0.207, grad_norm=68.779, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.838e-05, train_time=1.586 +[gpua006:0/64] 2024-02-12 18:54:24,078 (trainer:756) INFO: 35epoch:train:3401-3500batch: iter_time=1.010e-04, forward_time=0.148, loss_ctc=77.755, loss_interctc_layer6=85.457, loss_interctc_layer12=70.320, loss_interctc_layer15=64.249, loss_interctc_layer21=80.972, loss=75.751, backward_time=0.208, grad_norm=76.531, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.838e-05, train_time=1.221 +[gpua006:0/64] 2024-02-12 18:56:56,599 (trainer:756) INFO: 35epoch:train:3501-3600batch: iter_time=8.860e-05, forward_time=0.260, loss_ctc=77.976, loss_interctc_layer6=78.767, loss_interctc_layer12=65.002, loss_interctc_layer15=59.334, loss_interctc_layer21=80.874, loss=72.391, backward_time=0.247, grad_norm=71.575, clip=100.000, loss_scale=3.894e+31, optim_step_time=0.139, optim0_lr0=6.837e-05, train_time=1.525 +[gpua006:0/64] 2024-02-12 18:58:44,011 (trainer:756) INFO: 35epoch:train:3601-3700batch: iter_time=8.278e-05, forward_time=0.143, loss_ctc=71.131, loss_interctc_layer6=77.899, loss_interctc_layer12=64.508, loss_interctc_layer15=59.001, loss_interctc_layer21=73.816, loss=69.271, backward_time=0.209, grad_norm=83.056, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.836e-05, train_time=1.074 +[gpua006:0/64] 2024-02-12 18:59:54,791 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-12 19:00:13,446 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 19:00:16,892 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 19:00:16,892 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-12 19:00:16,914 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 19:04:59,765 (trainer:756) INFO: 35epoch:train:3701-3800batch: iter_time=2.695, forward_time=0.169, loss_ctc=62.331, loss_interctc_layer6=72.300, loss_interctc_layer12=59.200, loss_interctc_layer15=53.933, loss_interctc_layer21=64.511, loss=62.455, backward_time=0.215, grad_norm=54.880, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.836e-05, train_time=3.757 +[gpua006:0/64] 2024-02-12 19:06:40,387 (trainer:756) INFO: 35epoch:train:3801-3900batch: iter_time=8.252e-05, forward_time=0.144, loss_ctc=68.097, loss_interctc_layer6=77.988, loss_interctc_layer12=64.211, loss_interctc_layer15=58.896, loss_interctc_layer21=70.516, loss=67.942, backward_time=0.210, grad_norm=72.651, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.835e-05, train_time=1.006 +[gpua006:0/64] 2024-02-12 19:08:17,311 (trainer:756) INFO: 35epoch:train:3901-4000batch: iter_time=8.578e-05, forward_time=0.144, loss_ctc=91.383, loss_interctc_layer6=94.346, loss_interctc_layer12=79.000, loss_interctc_layer15=72.868, loss_interctc_layer21=94.711, loss=86.461, backward_time=0.212, grad_norm=80.966, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=6.834e-05, train_time=0.969 +[gpua006:0/64] 2024-02-12 19:10:49,508 (trainer:756) INFO: 35epoch:train:4001-4100batch: iter_time=8.922e-05, forward_time=0.272, loss_ctc=78.604, loss_interctc_layer6=84.833, loss_interctc_layer12=71.669, loss_interctc_layer15=66.936, loss_interctc_layer21=80.665, loss=76.541, backward_time=0.264, grad_norm=86.801, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=6.834e-05, train_time=1.521 +[gpua006:0/64] 2024-02-12 19:12:52,430 (trainer:756) INFO: 35epoch:train:4101-4200batch: iter_time=8.932e-05, forward_time=0.144, loss_ctc=79.474, loss_interctc_layer6=81.641, loss_interctc_layer12=67.188, loss_interctc_layer15=61.296, loss_interctc_layer21=82.681, loss=74.456, backward_time=0.209, grad_norm=66.507, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.833e-05, train_time=1.229 +[gpua006:0/64] 2024-02-12 19:15:20,584 (trainer:756) INFO: 35epoch:train:4201-4300batch: iter_time=8.944e-05, forward_time=0.143, loss_ctc=63.847, loss_interctc_layer6=73.437, loss_interctc_layer12=60.820, loss_interctc_layer15=55.471, loss_interctc_layer21=66.172, loss=63.949, backward_time=0.208, grad_norm=61.155, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.832e-05, train_time=1.482 +[gpua006:0/64] 2024-02-12 19:17:41,475 (trainer:756) INFO: 35epoch:train:4301-4400batch: iter_time=0.001, forward_time=0.274, loss_ctc=68.554, loss_interctc_layer6=70.574, loss_interctc_layer12=58.186, loss_interctc_layer15=53.208, loss_interctc_layer21=71.055, loss=64.316, backward_time=0.257, grad_norm=71.613, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.174, optim0_lr0=6.832e-05, train_time=1.408 +[gpua006:0/64] 2024-02-12 19:19:44,867 (trainer:756) INFO: 35epoch:train:4401-4500batch: iter_time=9.228e-05, forward_time=0.142, loss_ctc=63.444, loss_interctc_layer6=77.794, loss_interctc_layer12=64.525, loss_interctc_layer15=59.157, loss_interctc_layer21=65.714, loss=66.127, backward_time=0.208, grad_norm=72.272, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.831e-05, train_time=1.234 +[gpua006:0/64] 2024-02-12 19:21:59,274 (trainer:756) INFO: 35epoch:train:4501-4600batch: iter_time=9.183e-05, forward_time=0.144, loss_ctc=64.912, loss_interctc_layer6=69.588, loss_interctc_layer12=57.463, loss_interctc_layer15=52.510, loss_interctc_layer21=67.324, loss=62.359, backward_time=0.208, grad_norm=90.392, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.830e-05, train_time=1.345 +[gpua006:0/64] 2024-02-12 19:23:45,083 (trainer:756) INFO: 35epoch:train:4601-4700batch: iter_time=9.320e-05, forward_time=0.143, loss_ctc=84.037, loss_interctc_layer6=81.358, loss_interctc_layer12=67.181, loss_interctc_layer15=61.493, loss_interctc_layer21=87.576, loss=76.329, backward_time=0.208, grad_norm=89.954, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.830e-05, train_time=1.058 +[gpua006:0/64] 2024-02-12 19:26:11,973 (trainer:756) INFO: 35epoch:train:4701-4800batch: iter_time=1.693e-04, forward_time=0.281, loss_ctc=77.101, loss_interctc_layer6=79.012, loss_interctc_layer12=65.115, loss_interctc_layer15=59.527, loss_interctc_layer21=79.804, loss=72.112, backward_time=0.231, grad_norm=70.104, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=6.829e-05, train_time=1.469 +[gpua006:0/64] 2024-02-12 19:28:12,133 (trainer:756) INFO: 35epoch:train:4801-4900batch: iter_time=8.798e-05, forward_time=0.151, loss_ctc=89.455, loss_interctc_layer6=80.637, loss_interctc_layer12=66.467, loss_interctc_layer15=60.745, loss_interctc_layer21=93.047, loss=78.070, backward_time=0.212, grad_norm=72.429, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.828e-05, train_time=1.200 +[gpua006:0/64] 2024-02-12 19:30:13,689 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 19:30:27,326 (trainer:756) INFO: 35epoch:train:4901-5000batch: iter_time=8.761e-05, forward_time=0.143, loss_ctc=75.183, loss_interctc_layer6=77.711, loss_interctc_layer12=63.746, loss_interctc_layer15=58.070, loss_interctc_layer21=78.032, loss=70.549, backward_time=0.208, grad_norm=74.388, clip=100.000, loss_scale=3.770e+31, optim_step_time=0.136, optim0_lr0=6.828e-05, train_time=1.352 +[gpua006:0/64] 2024-02-12 19:30:47,355 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-12 19:31:06,071 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 19:31:09,503 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 19:31:09,503 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-12 19:31:09,518 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 19:36:32,989 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 19:36:57,931 (trainer:756) INFO: 35epoch:train:5001-5100batch: iter_time=2.858, forward_time=0.142, loss_ctc=53.839, loss_interctc_layer6=67.902, loss_interctc_layer12=55.786, loss_interctc_layer15=50.904, loss_interctc_layer21=55.641, loss=56.814, backward_time=0.208, grad_norm=52.574, clip=100.000, loss_scale=1.762e+31, optim_step_time=0.136, optim0_lr0=6.827e-05, train_time=3.906 +[gpua006:0/64] 2024-02-12 19:38:53,633 (trainer:756) INFO: 35epoch:train:5101-5200batch: iter_time=8.215e-05, forward_time=0.267, loss_ctc=81.424, loss_interctc_layer6=88.233, loss_interctc_layer12=73.324, loss_interctc_layer15=67.415, loss_interctc_layer21=84.348, loss=78.949, backward_time=0.237, grad_norm=79.556, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.826e-05, train_time=1.157 +[gpua006:0/64] 2024-02-12 19:40:41,060 (trainer:756) INFO: 35epoch:train:5201-5300batch: iter_time=8.075e-05, forward_time=0.154, loss_ctc=93.621, loss_interctc_layer6=97.261, loss_interctc_layer12=81.025, loss_interctc_layer15=74.875, loss_interctc_layer21=96.734, loss=88.703, backward_time=0.217, grad_norm=108.871, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.826e-05, train_time=1.074 +[gpua006:0/64] 2024-02-12 19:43:06,255 (trainer:756) INFO: 35epoch:train:5301-5400batch: iter_time=8.724e-05, forward_time=0.143, loss_ctc=59.564, loss_interctc_layer6=71.315, loss_interctc_layer12=59.057, loss_interctc_layer15=54.189, loss_interctc_layer21=61.330, loss=61.091, backward_time=0.208, grad_norm=99.886, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.825e-05, train_time=1.450 +[gpua006:0/64] 2024-02-12 19:45:20,277 (trainer:756) INFO: 35epoch:train:5401-5500batch: iter_time=8.428e-05, forward_time=0.143, loss_ctc=78.151, loss_interctc_layer6=83.504, loss_interctc_layer12=68.573, loss_interctc_layer15=62.436, loss_interctc_layer21=81.097, loss=74.752, backward_time=0.208, grad_norm=88.110, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.824e-05, train_time=1.341 +[gpua006:0/64] 2024-02-12 19:47:33,414 (trainer:756) INFO: 35epoch:train:5501-5600batch: iter_time=8.592e-05, forward_time=0.144, loss_ctc=63.311, loss_interctc_layer6=70.218, loss_interctc_layer12=58.057, loss_interctc_layer15=53.269, loss_interctc_layer21=65.709, loss=62.113, backward_time=0.209, grad_norm=64.477, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.824e-05, train_time=1.331 +[gpua006:0/64] 2024-02-12 19:49:53,438 (trainer:756) INFO: 35epoch:train:5601-5700batch: iter_time=4.801e-04, forward_time=0.273, loss_ctc=66.049, loss_interctc_layer6=76.824, loss_interctc_layer12=63.625, loss_interctc_layer15=58.362, loss_interctc_layer21=68.397, loss=66.651, backward_time=0.238, grad_norm=64.039, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.162, optim0_lr0=6.823e-05, train_time=1.400 +[gpua006:0/64] 2024-02-12 19:52:07,947 (trainer:756) INFO: 35epoch:train:5701-5800batch: iter_time=8.096e-05, forward_time=0.148, loss_ctc=68.247, loss_interctc_layer6=69.587, loss_interctc_layer12=57.277, loss_interctc_layer15=52.286, loss_interctc_layer21=70.770, loss=63.633, backward_time=0.212, grad_norm=123.203, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.822e-05, train_time=1.345 +[gpua006:0/64] 2024-02-12 19:54:17,484 (trainer:756) INFO: 35epoch:train:5801-5900batch: iter_time=8.193e-05, forward_time=0.142, loss_ctc=68.180, loss_interctc_layer6=69.848, loss_interctc_layer12=57.735, loss_interctc_layer15=52.820, loss_interctc_layer21=70.917, loss=63.900, backward_time=0.209, grad_norm=75.090, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.822e-05, train_time=1.295 +[gpua006:0/64] 2024-02-12 19:56:11,404 (trainer:756) INFO: 35epoch:train:5901-6000batch: iter_time=8.476e-05, forward_time=0.143, loss_ctc=85.357, loss_interctc_layer6=85.925, loss_interctc_layer12=70.769, loss_interctc_layer15=64.752, loss_interctc_layer21=88.776, loss=79.116, backward_time=0.208, grad_norm=76.414, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.821e-05, train_time=1.138 +[gpua006:0/64] 2024-02-12 19:58:46,420 (trainer:756) INFO: 35epoch:train:6001-6100batch: iter_time=8.155e-05, forward_time=0.143, loss_ctc=82.955, loss_interctc_layer6=78.700, loss_interctc_layer12=64.792, loss_interctc_layer15=59.171, loss_interctc_layer21=86.144, loss=74.352, backward_time=0.207, grad_norm=68.761, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.820e-05, train_time=1.551 +[gpua006:0/64] 2024-02-12 20:00:59,722 (trainer:756) INFO: 35epoch:train:6101-6200batch: iter_time=0.005, forward_time=0.267, loss_ctc=76.281, loss_interctc_layer6=77.526, loss_interctc_layer12=64.018, loss_interctc_layer15=58.560, loss_interctc_layer21=79.212, loss=71.119, backward_time=0.236, grad_norm=76.955, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.820e-05, train_time=1.332 +[gpua006:0/64] 2024-02-12 20:02:24,949 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-12 20:02:43,679 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 20:02:47,169 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 20:02:47,169 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-12 20:02:47,172 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 20:11:36,442 (trainer:756) INFO: 35epoch:train:6201-6300batch: iter_time=3.228, forward_time=0.178, loss_ctc=63.462, loss_interctc_layer6=71.419, loss_interctc_layer12=58.385, loss_interctc_layer15=53.101, loss_interctc_layer21=65.757, loss=62.425, backward_time=0.224, grad_norm=58.440, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.819e-05, train_time=6.367 +[gpua006:0/64] 2024-02-12 20:13:17,895 (trainer:756) INFO: 35epoch:train:6301-6400batch: iter_time=8.038e-05, forward_time=0.143, loss_ctc=67.232, loss_interctc_layer6=78.520, loss_interctc_layer12=64.599, loss_interctc_layer15=59.108, loss_interctc_layer21=69.437, loss=67.779, backward_time=0.210, grad_norm=150.530, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.818e-05, train_time=1.015 +[gpua006:0/64] 2024-02-12 20:15:19,340 (trainer:756) INFO: 35epoch:train:6401-6500batch: iter_time=8.332e-05, forward_time=0.219, loss_ctc=90.675, loss_interctc_layer6=94.511, loss_interctc_layer12=79.080, loss_interctc_layer15=72.904, loss_interctc_layer21=93.450, loss=86.124, backward_time=0.270, grad_norm=81.017, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=6.818e-05, train_time=1.213 +[gpua006:0/64] 2024-02-12 20:17:04,071 (trainer:756) INFO: 35epoch:train:6501-6600batch: iter_time=8.525e-05, forward_time=0.143, loss_ctc=78.568, loss_interctc_layer6=85.013, loss_interctc_layer12=71.867, loss_interctc_layer15=66.685, loss_interctc_layer21=80.516, loss=76.530, backward_time=0.209, grad_norm=88.897, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.817e-05, train_time=1.048 +[gpua006:0/64] 2024-02-12 20:19:04,101 (trainer:756) INFO: 35epoch:train:6601-6700batch: iter_time=8.484e-05, forward_time=0.146, loss_ctc=79.072, loss_interctc_layer6=81.639, loss_interctc_layer12=67.131, loss_interctc_layer15=61.244, loss_interctc_layer21=82.073, loss=74.232, backward_time=0.209, grad_norm=61.065, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.816e-05, train_time=1.200 +[gpua006:0/64] 2024-02-12 20:21:21,205 (trainer:756) INFO: 35epoch:train:6701-6800batch: iter_time=8.674e-05, forward_time=0.143, loss_ctc=62.589, loss_interctc_layer6=72.791, loss_interctc_layer12=60.034, loss_interctc_layer15=54.766, loss_interctc_layer21=64.846, loss=63.005, backward_time=0.208, grad_norm=86.189, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.816e-05, train_time=1.371 +[gpua006:0/64] 2024-02-12 20:23:26,691 (trainer:756) INFO: 35epoch:train:6801-6900batch: iter_time=9.007e-05, forward_time=0.142, loss_ctc=67.249, loss_interctc_layer6=69.991, loss_interctc_layer12=57.596, loss_interctc_layer15=52.618, loss_interctc_layer21=69.700, loss=63.431, backward_time=0.208, grad_norm=55.715, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.815e-05, train_time=1.255 +[gpua006:0/64] 2024-02-12 20:26:10,273 (trainer:756) INFO: 35epoch:train:6901-7000batch: iter_time=8.461e-05, forward_time=0.199, loss_ctc=62.988, loss_interctc_layer6=76.863, loss_interctc_layer12=63.654, loss_interctc_layer15=58.355, loss_interctc_layer21=65.257, loss=65.424, backward_time=0.310, grad_norm=71.386, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=6.814e-05, train_time=1.635 +[gpua006:0/64] 2024-02-12 20:28:20,187 (trainer:756) INFO: 35epoch:train:7001-7100batch: iter_time=8.412e-05, forward_time=0.143, loss_ctc=64.135, loss_interctc_layer6=68.783, loss_interctc_layer12=56.725, loss_interctc_layer15=51.818, loss_interctc_layer21=66.537, loss=61.600, backward_time=0.208, grad_norm=64.429, clip=100.000, loss_scale=1.278e+31, optim_step_time=0.136, optim0_lr0=6.814e-05, train_time=1.299 +[gpua006:0/64] 2024-02-12 20:30:17,782 (trainer:756) INFO: 35epoch:train:7101-7200batch: iter_time=8.361e-05, forward_time=0.143, loss_ctc=83.701, loss_interctc_layer6=81.537, loss_interctc_layer12=67.415, loss_interctc_layer15=61.664, loss_interctc_layer21=87.093, loss=76.282, backward_time=0.208, grad_norm=71.429, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.813e-05, train_time=1.176 +[gpua006:0/64] 2024-02-12 20:32:45,230 (trainer:756) INFO: 35epoch:train:7201-7300batch: iter_time=3.195e-04, forward_time=0.233, loss_ctc=75.900, loss_interctc_layer6=78.349, loss_interctc_layer12=64.427, loss_interctc_layer15=58.973, loss_interctc_layer21=78.895, loss=71.309, backward_time=0.228, grad_norm=67.375, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.812e-05, train_time=1.474 +[gpua006:0/64] 2024-02-12 20:35:01,022 (trainer:756) INFO: 35epoch:train:7301-7400batch: iter_time=8.550e-05, forward_time=0.159, loss_ctc=89.200, loss_interctc_layer6=80.214, loss_interctc_layer12=66.034, loss_interctc_layer15=60.237, loss_interctc_layer21=92.685, loss=77.674, backward_time=0.212, grad_norm=73.359, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.812e-05, train_time=1.358 +[gpua006:0/64] 2024-02-12 20:37:12,564 (trainer:756) INFO: 35epoch:train:7401-7500batch: iter_time=8.423e-05, forward_time=0.142, loss_ctc=74.035, loss_interctc_layer6=77.547, loss_interctc_layer12=63.676, loss_interctc_layer15=58.159, loss_interctc_layer21=76.775, loss=70.038, backward_time=0.209, grad_norm=79.814, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.811e-05, train_time=1.314 +[gpua006:0/64] 2024-02-12 20:37:32,594 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-12 20:37:51,617 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 20:37:55,077 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 20:37:55,078 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-12 20:37:55,081 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 20:44:04,343 (trainer:756) INFO: 35epoch:train:7501-7600batch: iter_time=2.962, forward_time=0.142, loss_ctc=54.130, loss_interctc_layer6=67.518, loss_interctc_layer12=55.390, loss_interctc_layer15=50.567, loss_interctc_layer21=55.978, loss=56.716, backward_time=0.208, grad_norm=62.930, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.811e-05, train_time=4.118 +[gpua006:0/64] 2024-02-12 20:45:49,361 (trainer:756) INFO: 35epoch:train:7601-7700batch: iter_time=8.231e-05, forward_time=0.144, loss_ctc=80.351, loss_interctc_layer6=88.256, loss_interctc_layer12=73.441, loss_interctc_layer15=67.712, loss_interctc_layer21=82.926, loss=78.537, backward_time=0.210, grad_norm=77.245, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.810e-05, train_time=1.049 +[gpua006:0/64] 2024-02-12 20:48:07,083 (trainer:756) INFO: 35epoch:train:7701-7800batch: iter_time=9.086e-05, forward_time=0.190, loss_ctc=90.516, loss_interctc_layer6=97.186, loss_interctc_layer12=81.088, loss_interctc_layer15=74.769, loss_interctc_layer21=93.270, loss=87.366, backward_time=0.263, grad_norm=93.552, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.809e-05, train_time=1.377 +[gpua006:0/64] 2024-02-12 20:50:11,245 (trainer:756) INFO: 35epoch:train:7801-7900batch: iter_time=9.044e-05, forward_time=0.157, loss_ctc=57.457, loss_interctc_layer6=70.936, loss_interctc_layer12=58.781, loss_interctc_layer15=53.855, loss_interctc_layer21=59.487, loss=60.103, backward_time=0.213, grad_norm=63.089, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.809e-05, train_time=1.242 +[gpua006:0/64] 2024-02-12 20:52:00,018 (trainer:756) INFO: 35epoch:train:7901-8000batch: iter_time=8.573e-05, forward_time=0.143, loss_ctc=75.284, loss_interctc_layer6=83.228, loss_interctc_layer12=68.346, loss_interctc_layer15=62.218, loss_interctc_layer21=78.100, loss=73.435, backward_time=0.208, grad_norm=56.878, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.808e-05, train_time=1.087 +[gpua006:0/64] 2024-02-12 20:54:03,560 (trainer:756) INFO: 35epoch:train:8001-8100batch: iter_time=8.825e-05, forward_time=0.141, loss_ctc=63.473, loss_interctc_layer6=70.286, loss_interctc_layer12=58.204, loss_interctc_layer15=53.209, loss_interctc_layer21=65.904, loss=62.215, backward_time=0.208, grad_norm=64.822, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.807e-05, train_time=1.236 +[gpua006:0/64] 2024-02-12 20:56:08,153 (trainer:756) INFO: 35epoch:train:8101-8200batch: iter_time=8.625e-05, forward_time=0.142, loss_ctc=63.824, loss_interctc_layer6=76.343, loss_interctc_layer12=63.245, loss_interctc_layer15=57.881, loss_interctc_layer21=66.075, loss=65.473, backward_time=0.208, grad_norm=69.185, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.807e-05, train_time=1.246 +[gpua006:0/64] 2024-02-12 20:58:19,977 (trainer:756) INFO: 35epoch:train:8201-8300batch: iter_time=8.563e-05, forward_time=0.143, loss_ctc=64.087, loss_interctc_layer6=69.470, loss_interctc_layer12=57.054, loss_interctc_layer15=51.968, loss_interctc_layer21=66.512, loss=61.818, backward_time=0.208, grad_norm=84.131, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.806e-05, train_time=1.318 +[gpua006:0/64] 2024-02-12 21:00:38,787 (trainer:756) INFO: 35epoch:train:8301-8400batch: iter_time=8.581e-05, forward_time=0.155, loss_ctc=63.463, loss_interctc_layer6=69.670, loss_interctc_layer12=57.533, loss_interctc_layer15=52.566, loss_interctc_layer21=66.029, loss=61.852, backward_time=0.209, grad_norm=57.171, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.805e-05, train_time=1.388 +[gpua006:0/64] 2024-02-12 21:03:10,889 (trainer:756) INFO: 35epoch:train:8401-8500batch: iter_time=1.621e-04, forward_time=0.240, loss_ctc=77.972, loss_interctc_layer6=85.391, loss_interctc_layer12=70.319, loss_interctc_layer15=64.222, loss_interctc_layer21=81.282, loss=75.837, backward_time=0.227, grad_norm=78.983, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.154, optim0_lr0=6.805e-05, train_time=1.521 +[gpua006:0/64] 2024-02-12 21:04:53,524 (trainer:756) INFO: 35epoch:train:8501-8600batch: iter_time=8.660e-05, forward_time=0.142, loss_ctc=78.637, loss_interctc_layer6=78.313, loss_interctc_layer12=64.497, loss_interctc_layer15=58.818, loss_interctc_layer21=81.831, loss=72.419, backward_time=0.208, grad_norm=69.010, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.804e-05, train_time=1.025 +[gpua006:0/64] 2024-02-12 21:06:55,063 (trainer:756) INFO: 35epoch:train:8601-8700batch: iter_time=8.256e-05, forward_time=0.143, loss_ctc=69.949, loss_interctc_layer6=76.994, loss_interctc_layer12=63.471, loss_interctc_layer15=57.876, loss_interctc_layer21=72.713, loss=68.200, backward_time=0.209, grad_norm=66.973, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.803e-05, train_time=1.216 +[gpua006:0/64] 2024-02-12 21:08:16,582 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-12 21:08:35,297 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 21:08:38,767 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 21:08:38,767 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-12 21:08:38,781 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 21:13:26,790 (trainer:756) INFO: 35epoch:train:8701-8800batch: iter_time=2.707, forward_time=0.169, loss_ctc=61.367, loss_interctc_layer6=70.987, loss_interctc_layer12=58.015, loss_interctc_layer15=52.658, loss_interctc_layer21=63.664, loss=61.338, backward_time=0.214, grad_norm=54.183, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.803e-05, train_time=3.917 +[gpua006:0/64] 2024-02-12 21:14:21,380 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 21:15:17,003 (trainer:756) INFO: 35epoch:train:8801-8900batch: iter_time=8.052e-05, forward_time=0.143, loss_ctc=67.968, loss_interctc_layer6=77.750, loss_interctc_layer12=63.929, loss_interctc_layer15=58.449, loss_interctc_layer21=70.280, loss=67.675, backward_time=0.209, grad_norm=70.255, clip=100.000, loss_scale=1.537e+31, optim_step_time=0.136, optim0_lr0=6.802e-05, train_time=1.102 +[gpua006:0/64] 2024-02-12 21:17:15,040 (trainer:756) INFO: 35epoch:train:8901-9000batch: iter_time=8.069e-05, forward_time=0.146, loss_ctc=90.262, loss_interctc_layer6=93.328, loss_interctc_layer12=77.679, loss_interctc_layer15=71.522, loss_interctc_layer21=93.384, loss=85.235, backward_time=0.209, grad_norm=88.594, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.801e-05, train_time=1.180 +[gpua006:0/64] 2024-02-12 21:19:25,384 (trainer:756) INFO: 35epoch:train:9001-9100batch: iter_time=8.342e-05, forward_time=0.260, loss_ctc=76.781, loss_interctc_layer6=83.206, loss_interctc_layer12=69.888, loss_interctc_layer15=64.558, loss_interctc_layer21=80.320, loss=74.951, backward_time=0.255, grad_norm=81.701, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.801e-05, train_time=1.303 +[gpua006:0/64] 2024-02-12 21:21:39,777 (trainer:756) INFO: 35epoch:train:9101-9200batch: iter_time=8.434e-05, forward_time=0.143, loss_ctc=78.385, loss_interctc_layer6=81.313, loss_interctc_layer12=66.918, loss_interctc_layer15=61.126, loss_interctc_layer21=81.549, loss=73.858, backward_time=0.209, grad_norm=70.484, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.800e-05, train_time=1.344 +[gpua006:0/64] 2024-02-12 21:23:57,196 (trainer:756) INFO: 35epoch:train:9201-9300batch: iter_time=8.338e-05, forward_time=0.142, loss_ctc=62.238, loss_interctc_layer6=72.172, loss_interctc_layer12=59.381, loss_interctc_layer15=54.119, loss_interctc_layer21=64.592, loss=62.500, backward_time=0.209, grad_norm=60.971, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.799e-05, train_time=1.374 +[gpua006:0/64] 2024-02-12 21:26:09,397 (trainer:756) INFO: 35epoch:train:9301-9400batch: iter_time=2.070e-04, forward_time=0.204, loss_ctc=66.691, loss_interctc_layer6=69.862, loss_interctc_layer12=57.482, loss_interctc_layer15=52.457, loss_interctc_layer21=69.212, loss=63.141, backward_time=0.254, grad_norm=55.812, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.145, optim0_lr0=6.799e-05, train_time=1.321 +[gpua006:0/64] 2024-02-12 21:28:21,292 (trainer:756) INFO: 35epoch:train:9401-9500batch: iter_time=8.175e-05, forward_time=0.143, loss_ctc=62.545, loss_interctc_layer6=77.174, loss_interctc_layer12=63.991, loss_interctc_layer15=58.614, loss_interctc_layer21=64.811, loss=65.427, backward_time=0.208, grad_norm=115.499, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.798e-05, train_time=1.320 +[gpua006:0/64] 2024-02-12 21:30:40,237 (trainer:756) INFO: 35epoch:train:9501-9600batch: iter_time=8.170e-05, forward_time=0.143, loss_ctc=64.417, loss_interctc_layer6=69.282, loss_interctc_layer12=57.158, loss_interctc_layer15=52.239, loss_interctc_layer21=66.880, loss=61.995, backward_time=0.209, grad_norm=55.874, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.797e-05, train_time=1.389 +[gpua006:0/64] 2024-02-12 21:33:08,093 (trainer:756) INFO: 35epoch:train:9601-9700batch: iter_time=8.217e-05, forward_time=0.188, loss_ctc=83.198, loss_interctc_layer6=80.915, loss_interctc_layer12=66.793, loss_interctc_layer15=61.037, loss_interctc_layer21=86.749, loss=75.738, backward_time=0.294, grad_norm=86.788, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=6.797e-05, train_time=1.478 +[gpua006:0/64] 2024-02-12 21:34:52,797 (trainer:756) INFO: 35epoch:train:9701-9800batch: iter_time=8.206e-05, forward_time=0.143, loss_ctc=76.126, loss_interctc_layer6=78.376, loss_interctc_layer12=64.452, loss_interctc_layer15=58.813, loss_interctc_layer21=79.280, loss=71.409, backward_time=0.209, grad_norm=68.658, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.796e-05, train_time=1.048 +[gpua006:0/64] 2024-02-12 21:37:12,963 (trainer:756) INFO: 35epoch:train:9801-9900batch: iter_time=8.961e-05, forward_time=0.219, loss_ctc=88.813, loss_interctc_layer6=80.185, loss_interctc_layer12=66.082, loss_interctc_layer15=60.221, loss_interctc_layer21=92.419, loss=77.544, backward_time=0.250, grad_norm=69.482, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.795e-05, train_time=1.401 +[gpua006:0/64] 2024-02-12 21:39:07,477 (trainer:756) INFO: 35epoch:train:9901-10000batch: iter_time=8.975e-05, forward_time=0.143, loss_ctc=74.552, loss_interctc_layer6=76.885, loss_interctc_layer12=63.174, loss_interctc_layer15=57.599, loss_interctc_layer21=77.359, loss=69.914, backward_time=0.209, grad_norm=63.698, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.795e-05, train_time=1.146 +[gpua006:0/64] 2024-02-12 21:39:27,504 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-12 21:39:46,605 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 21:39:50,103 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 21:39:50,103 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-12 21:39:50,107 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 21:45:40,271 (trainer:756) INFO: 35epoch:train:10001-10100batch: iter_time=2.922, forward_time=0.164, loss_ctc=54.380, loss_interctc_layer6=67.091, loss_interctc_layer12=55.048, loss_interctc_layer15=50.230, loss_interctc_layer21=56.208, loss=56.591, backward_time=0.217, grad_norm=57.254, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.794e-05, train_time=3.927 +[gpua006:0/64] 2024-02-12 21:47:40,669 (trainer:756) INFO: 35epoch:train:10101-10200batch: iter_time=8.171e-05, forward_time=0.146, loss_ctc=80.045, loss_interctc_layer6=88.322, loss_interctc_layer12=73.767, loss_interctc_layer15=67.918, loss_interctc_layer21=82.741, loss=78.559, backward_time=0.209, grad_norm=81.792, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.793e-05, train_time=1.204 +[gpua006:0/64] 2024-02-12 21:49:47,928 (trainer:756) INFO: 35epoch:train:10201-10300batch: iter_time=8.031e-05, forward_time=0.259, loss_ctc=89.785, loss_interctc_layer6=96.655, loss_interctc_layer12=80.951, loss_interctc_layer15=74.445, loss_interctc_layer21=93.056, loss=86.978, backward_time=0.231, grad_norm=113.465, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=6.793e-05, train_time=1.272 +[gpua006:0/64] 2024-02-12 21:51:28,017 (trainer:756) INFO: 35epoch:train:10301-10400batch: iter_time=8.177e-05, forward_time=0.142, loss_ctc=56.897, loss_interctc_layer6=70.966, loss_interctc_layer12=58.764, loss_interctc_layer15=53.790, loss_interctc_layer21=58.706, loss=59.825, backward_time=0.209, grad_norm=73.345, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.792e-05, train_time=1.001 +[gpua006:0/64] 2024-02-12 21:53:34,952 (trainer:756) INFO: 35epoch:train:10401-10500batch: iter_time=8.263e-05, forward_time=0.150, loss_ctc=74.608, loss_interctc_layer6=83.396, loss_interctc_layer12=68.443, loss_interctc_layer15=62.284, loss_interctc_layer21=77.325, loss=73.211, backward_time=0.208, grad_norm=68.553, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.792e-05, train_time=1.269 +[gpua006:0/64] 2024-02-12 21:56:04,955 (trainer:756) INFO: 35epoch:train:10501-10600batch: iter_time=9.793e-05, forward_time=0.229, loss_ctc=62.947, loss_interctc_layer6=69.553, loss_interctc_layer12=57.303, loss_interctc_layer15=52.305, loss_interctc_layer21=65.376, loss=61.497, backward_time=0.259, grad_norm=56.638, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.791e-05, train_time=1.500 +[gpua006:0/64] 2024-02-12 21:58:12,943 (trainer:756) INFO: 35epoch:train:10601-10700batch: iter_time=8.598e-05, forward_time=0.142, loss_ctc=63.545, loss_interctc_layer6=76.195, loss_interctc_layer12=63.171, loss_interctc_layer15=57.837, loss_interctc_layer21=65.845, loss=65.319, backward_time=0.208, grad_norm=67.944, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.790e-05, train_time=1.279 +[gpua006:0/64] 2024-02-12 22:00:47,875 (trainer:756) INFO: 35epoch:train:10701-10800batch: iter_time=8.314e-05, forward_time=0.142, loss_ctc=63.160, loss_interctc_layer6=68.965, loss_interctc_layer12=56.680, loss_interctc_layer15=51.680, loss_interctc_layer21=65.610, loss=61.219, backward_time=0.208, grad_norm=82.816, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.790e-05, train_time=1.549 +[gpua006:0/64] 2024-02-12 22:03:07,265 (trainer:756) INFO: 35epoch:train:10801-10900batch: iter_time=5.344e-04, forward_time=0.227, loss_ctc=63.867, loss_interctc_layer6=70.080, loss_interctc_layer12=57.940, loss_interctc_layer15=52.905, loss_interctc_layer21=66.429, loss=62.244, backward_time=0.251, grad_norm=73.669, clip=100.000, loss_scale=1.501e+31, optim_step_time=0.152, optim0_lr0=6.789e-05, train_time=1.391 +[gpua006:0/64] 2024-02-12 22:04:56,775 (trainer:756) INFO: 35epoch:train:10901-11000batch: iter_time=8.080e-05, forward_time=0.143, loss_ctc=78.478, loss_interctc_layer6=85.700, loss_interctc_layer12=70.441, loss_interctc_layer15=64.397, loss_interctc_layer21=81.563, loss=76.116, backward_time=0.208, grad_norm=72.564, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.788e-05, train_time=1.097 +[gpua006:0/64] 2024-02-12 22:06:54,409 (trainer:756) INFO: 35epoch:train:11001-11100batch: iter_time=8.480e-05, forward_time=0.142, loss_ctc=77.574, loss_interctc_layer6=78.290, loss_interctc_layer12=64.344, loss_interctc_layer15=58.691, loss_interctc_layer21=80.425, loss=71.865, backward_time=0.208, grad_norm=88.662, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.788e-05, train_time=1.176 +[gpua006:0/64] 2024-02-12 22:09:13,902 (trainer:756) INFO: 35epoch:train:11101-11200batch: iter_time=8.106e-05, forward_time=0.143, loss_ctc=70.562, loss_interctc_layer6=77.194, loss_interctc_layer12=63.670, loss_interctc_layer15=58.174, loss_interctc_layer21=73.235, loss=68.567, backward_time=0.210, grad_norm=94.228, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.787e-05, train_time=1.395 +[gpua006:0/64] 2024-02-12 22:10:50,474 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-12 22:11:09,560 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 22:11:13,020 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 22:11:13,021 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-12 22:11:13,024 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 22:16:17,909 (trainer:756) INFO: 35epoch:train:11201-11300batch: iter_time=2.744, forward_time=0.254, loss_ctc=62.170, loss_interctc_layer6=71.709, loss_interctc_layer12=58.623, loss_interctc_layer15=53.353, loss_interctc_layer21=64.481, loss=62.067, backward_time=0.235, grad_norm=54.842, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.786e-05, train_time=4.240 +[gpua006:0/64] 2024-02-12 22:17:54,935 (trainer:756) INFO: 35epoch:train:11301-11400batch: iter_time=7.850e-05, forward_time=0.142, loss_ctc=66.740, loss_interctc_layer6=77.323, loss_interctc_layer12=63.663, loss_interctc_layer15=58.224, loss_interctc_layer21=69.000, loss=66.990, backward_time=0.209, grad_norm=115.269, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.786e-05, train_time=0.970 +[gpua006:0/64] 2024-02-12 22:19:29,815 (trainer:756) INFO: 35epoch:train:11401-11500batch: iter_time=7.835e-05, forward_time=0.143, loss_ctc=86.490, loss_interctc_layer6=93.779, loss_interctc_layer12=78.060, loss_interctc_layer15=71.990, loss_interctc_layer21=89.476, loss=83.959, backward_time=0.210, grad_norm=143.615, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.785e-05, train_time=0.948 +[gpua006:0/64] 2024-02-12 22:21:38,079 (trainer:756) INFO: 35epoch:train:11501-11600batch: iter_time=8.416e-05, forward_time=0.145, loss_ctc=74.153, loss_interctc_layer6=84.072, loss_interctc_layer12=70.567, loss_interctc_layer15=65.902, loss_interctc_layer21=76.651, loss=74.269, backward_time=0.210, grad_norm=94.015, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.784e-05, train_time=1.282 +[gpua006:0/64] 2024-02-12 22:24:07,746 (trainer:756) INFO: 35epoch:train:11601-11700batch: iter_time=9.108e-05, forward_time=0.259, loss_ctc=75.219, loss_interctc_layer6=81.974, loss_interctc_layer12=67.518, loss_interctc_layer15=61.640, loss_interctc_layer21=77.997, loss=72.870, backward_time=0.238, grad_norm=67.118, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.784e-05, train_time=1.495 +[gpua006:0/64] 2024-02-12 22:26:19,245 (trainer:756) INFO: 35epoch:train:11701-11800batch: iter_time=8.681e-05, forward_time=0.143, loss_ctc=61.202, loss_interctc_layer6=72.349, loss_interctc_layer12=59.612, loss_interctc_layer15=54.266, loss_interctc_layer21=63.398, loss=62.165, backward_time=0.209, grad_norm=68.974, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.783e-05, train_time=1.316 +[gpua006:0/64] 2024-02-12 22:28:20,601 (trainer:756) INFO: 35epoch:train:11801-11900batch: iter_time=8.114e-05, forward_time=0.142, loss_ctc=66.115, loss_interctc_layer6=70.386, loss_interctc_layer12=57.922, loss_interctc_layer15=52.853, loss_interctc_layer21=68.618, loss=63.179, backward_time=0.209, grad_norm=62.703, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.782e-05, train_time=1.213 +[gpua006:0/64] 2024-02-12 22:30:56,611 (trainer:756) INFO: 35epoch:train:11901-12000batch: iter_time=8.770e-05, forward_time=0.142, loss_ctc=61.103, loss_interctc_layer6=76.987, loss_interctc_layer12=63.878, loss_interctc_layer15=58.430, loss_interctc_layer21=63.317, loss=64.743, backward_time=0.207, grad_norm=89.979, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.782e-05, train_time=1.560 +[gpua006:0/64] 2024-02-12 22:33:18,234 (trainer:756) INFO: 35epoch:train:12001-12100batch: iter_time=8.070e-05, forward_time=0.154, loss_ctc=60.589, loss_interctc_layer6=69.515, loss_interctc_layer12=57.262, loss_interctc_layer15=52.252, loss_interctc_layer21=62.864, loss=60.497, backward_time=0.208, grad_norm=72.340, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.781e-05, train_time=1.415 +[gpua006:0/64] 2024-02-12 22:35:19,045 (trainer:756) INFO: 35epoch:train:12101-12200batch: iter_time=8.886e-05, forward_time=0.235, loss_ctc=75.929, loss_interctc_layer6=80.692, loss_interctc_layer12=66.603, loss_interctc_layer15=60.833, loss_interctc_layer21=79.244, loss=72.660, backward_time=0.236, grad_norm=84.058, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.145, optim0_lr0=6.780e-05, train_time=1.208 +[gpua006:0/64] 2024-02-12 22:37:36,204 (trainer:756) INFO: 35epoch:train:12201-12300batch: iter_time=8.680e-05, forward_time=0.143, loss_ctc=70.840, loss_interctc_layer6=78.369, loss_interctc_layer12=64.444, loss_interctc_layer15=58.786, loss_interctc_layer21=73.640, loss=69.216, backward_time=0.207, grad_norm=84.682, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.780e-05, train_time=1.372 +[gpua006:0/64] 2024-02-12 22:39:19,082 (trainer:756) INFO: 35epoch:train:12301-12400batch: iter_time=8.380e-05, forward_time=0.143, loss_ctc=85.586, loss_interctc_layer6=80.597, loss_interctc_layer12=66.414, loss_interctc_layer15=60.520, loss_interctc_layer21=89.169, loss=76.457, backward_time=0.210, grad_norm=64.815, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.779e-05, train_time=1.029 +[gpua006:0/64] 2024-02-12 22:41:57,241 (trainer:756) INFO: 35epoch:train:12401-12500batch: iter_time=8.171e-05, forward_time=0.143, loss_ctc=68.554, loss_interctc_layer6=77.146, loss_interctc_layer12=63.223, loss_interctc_layer15=57.567, loss_interctc_layer21=71.119, loss=67.522, backward_time=0.209, grad_norm=70.837, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.779e-05, train_time=1.581 +[gpua006:0/64] 2024-02-12 22:42:17,271 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-12 22:42:35,850 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 22:42:39,301 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 22:42:39,301 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-12 22:42:39,305 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 22:48:26,960 (trainer:756) INFO: 35epoch:train:12501-12600batch: iter_time=2.853, forward_time=0.166, loss_ctc=53.848, loss_interctc_layer6=67.534, loss_interctc_layer12=55.500, loss_interctc_layer15=50.680, loss_interctc_layer21=55.644, loss=56.641, backward_time=0.216, grad_norm=67.230, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.778e-05, train_time=3.897 +[gpua006:0/64] 2024-02-12 22:50:15,403 (trainer:756) INFO: 35epoch:train:12601-12700batch: iter_time=8.053e-05, forward_time=0.144, loss_ctc=82.317, loss_interctc_layer6=87.851, loss_interctc_layer12=73.469, loss_interctc_layer15=67.730, loss_interctc_layer21=85.287, loss=79.331, backward_time=0.209, grad_norm=81.897, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.777e-05, train_time=1.085 +[gpua006:0/64] 2024-02-12 22:52:03,104 (trainer:756) INFO: 35epoch:train:12701-12800batch: iter_time=8.323e-05, forward_time=0.143, loss_ctc=92.590, loss_interctc_layer6=96.645, loss_interctc_layer12=80.916, loss_interctc_layer15=74.330, loss_interctc_layer21=96.228, loss=88.142, backward_time=0.210, grad_norm=90.058, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.777e-05, train_time=1.077 +[gpua006:0/64] 2024-02-12 22:54:03,500 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-12 22:54:24,559 (trainer:756) INFO: 35epoch:train:12801-12900batch: iter_time=8.816e-05, forward_time=0.143, loss_ctc=59.187, loss_interctc_layer6=70.699, loss_interctc_layer12=58.567, loss_interctc_layer15=53.807, loss_interctc_layer21=61.012, loss=60.654, backward_time=0.209, grad_norm=59.462, clip=100.000, loss_scale=2.561e+31, optim_step_time=0.136, optim0_lr0=6.776e-05, train_time=1.414 +[gpua006:0/64] 2024-02-12 22:57:08,281 (trainer:756) INFO: 35epoch:train:12901-13000batch: iter_time=8.529e-05, forward_time=0.207, loss_ctc=76.935, loss_interctc_layer6=82.428, loss_interctc_layer12=67.613, loss_interctc_layer15=61.599, loss_interctc_layer21=79.883, loss=73.692, backward_time=0.236, grad_norm=67.397, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.147, optim0_lr0=6.775e-05, train_time=1.636 +[gpua006:0/64] 2024-02-12 22:59:45,605 (trainer:756) INFO: 35epoch:train:13001-13100batch: iter_time=8.778e-05, forward_time=0.142, loss_ctc=62.696, loss_interctc_layer6=69.041, loss_interctc_layer12=56.898, loss_interctc_layer15=51.992, loss_interctc_layer21=64.990, loss=61.123, backward_time=0.207, grad_norm=67.052, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.775e-05, train_time=1.574 +[gpua006:0/64] 2024-02-12 23:01:59,895 (trainer:756) INFO: 35epoch:train:13101-13200batch: iter_time=9.608e-05, forward_time=0.143, loss_ctc=65.025, loss_interctc_layer6=75.671, loss_interctc_layer12=62.630, loss_interctc_layer15=57.324, loss_interctc_layer21=67.440, loss=65.618, backward_time=0.208, grad_norm=78.528, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.774e-05, train_time=1.343 +[gpua006:0/64] 2024-02-12 23:03:56,842 (trainer:756) INFO: 35epoch:train:13201-13300batch: iter_time=9.138e-05, forward_time=0.142, loss_ctc=67.775, loss_interctc_layer6=68.891, loss_interctc_layer12=56.504, loss_interctc_layer15=51.526, loss_interctc_layer21=70.432, loss=63.026, backward_time=0.209, grad_norm=83.062, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.773e-05, train_time=1.169 +[gpua006:0/64] 2024-02-12 23:05:45,981 (trainer:756) INFO: 35epoch:train:13301-13400batch: iter_time=8.822e-05, forward_time=0.143, loss_ctc=67.694, loss_interctc_layer6=69.044, loss_interctc_layer12=56.974, loss_interctc_layer15=52.044, loss_interctc_layer21=70.519, loss=63.255, backward_time=0.209, grad_norm=68.786, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.773e-05, train_time=1.091 +[gpua006:0/64] 2024-02-12 23:07:39,759 (trainer:756) INFO: 35epoch:train:13401-13500batch: iter_time=8.285e-05, forward_time=0.143, loss_ctc=84.693, loss_interctc_layer6=85.466, loss_interctc_layer12=70.212, loss_interctc_layer15=64.104, loss_interctc_layer21=87.990, loss=78.493, backward_time=0.208, grad_norm=64.843, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.772e-05, train_time=1.138 +[gpua006:0/64] 2024-02-12 23:09:49,377 (trainer:756) INFO: 35epoch:train:13501-13600batch: iter_time=9.036e-05, forward_time=0.231, loss_ctc=82.452, loss_interctc_layer6=78.451, loss_interctc_layer12=64.411, loss_interctc_layer15=58.725, loss_interctc_layer21=85.598, loss=73.927, backward_time=0.254, grad_norm=75.713, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=6.771e-05, train_time=1.295 +[gpua006:0/64] 2024-02-12 23:12:00,152 (trainer:756) INFO: 35epoch:train:13601-13700batch: iter_time=9.196e-05, forward_time=0.143, loss_ctc=76.602, loss_interctc_layer6=78.088, loss_interctc_layer12=64.411, loss_interctc_layer15=58.744, loss_interctc_layer21=79.559, loss=71.481, backward_time=0.207, grad_norm=64.684, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.771e-05, train_time=1.308 +[gpua006:0/64] 2024-02-12 23:13:56,334 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-12 23:14:15,187 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-12 23:14:18,674 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-12 23:14:18,675 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-12 23:14:18,678 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-12 23:19:32,970 (trainer:756) INFO: 35epoch:train:13701-13800batch: iter_time=2.892, forward_time=0.143, loss_ctc=63.040, loss_interctc_layer6=71.278, loss_interctc_layer12=58.263, loss_interctc_layer15=53.007, loss_interctc_layer21=65.264, loss=62.170, backward_time=0.208, grad_norm=67.407, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.770e-05, train_time=4.528 +[gpua006:0/64] 2024-02-12 23:21:08,102 (trainer:756) INFO: 35epoch:train:13801-13900batch: iter_time=1.001e-04, forward_time=0.143, loss_ctc=67.740, loss_interctc_layer6=77.749, loss_interctc_layer12=64.013, loss_interctc_layer15=58.541, loss_interctc_layer21=70.008, loss=67.610, backward_time=0.208, grad_norm=87.144, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.769e-05, train_time=0.951 +[gpua006:0/64] 2024-02-12 23:22:47,725 (trainer:756) INFO: 35epoch:train:13901-14000batch: iter_time=9.024e-05, forward_time=0.144, loss_ctc=89.239, loss_interctc_layer6=93.412, loss_interctc_layer12=77.615, loss_interctc_layer15=71.257, loss_interctc_layer21=92.905, loss=84.885, backward_time=0.209, grad_norm=85.856, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.769e-05, train_time=0.996 +[gpua006:0/64] 2024-02-12 23:25:08,300 (trainer:756) INFO: 35epoch:train:14001-14100batch: iter_time=1.012e-04, forward_time=0.237, loss_ctc=75.963, loss_interctc_layer6=82.876, loss_interctc_layer12=69.676, loss_interctc_layer15=64.371, loss_interctc_layer21=78.772, loss=74.332, backward_time=0.231, grad_norm=90.493, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.150, optim0_lr0=6.768e-05, train_time=1.406 +[gpua006:0/64] 2024-02-12 23:27:19,765 (trainer:756) INFO: 35epoch:train:14101-14200batch: iter_time=9.418e-05, forward_time=0.143, loss_ctc=78.657, loss_interctc_layer6=81.400, loss_interctc_layer12=67.062, loss_interctc_layer15=61.209, loss_interctc_layer21=81.782, loss=74.022, backward_time=0.207, grad_norm=77.030, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.768e-05, train_time=1.314 +[gpua006:0/64] 2024-02-12 23:29:29,815 (trainer:756) INFO: 35epoch:train:14201-14300batch: iter_time=9.622e-05, forward_time=0.143, loss_ctc=62.536, loss_interctc_layer6=72.336, loss_interctc_layer12=59.491, loss_interctc_layer15=54.177, loss_interctc_layer21=64.777, loss=62.664, backward_time=0.207, grad_norm=66.042, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.767e-05, train_time=1.301 +[gpua006:0/64] 2024-02-12 23:32:04,119 (trainer:756) INFO: 35epoch:train:14301-14400batch: iter_time=9.091e-05, forward_time=0.142, loss_ctc=66.889, loss_interctc_layer6=69.768, loss_interctc_layer12=57.315, loss_interctc_layer15=52.327, loss_interctc_layer21=69.419, loss=63.144, backward_time=0.208, grad_norm=70.866, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.766e-05, train_time=1.543 +[gpua006:0/64] 2024-02-12 23:34:47,297 (trainer:756) INFO: 35epoch:train:14401-14500batch: iter_time=8.959e-05, forward_time=0.142, loss_ctc=62.917, loss_interctc_layer6=77.230, loss_interctc_layer12=64.111, loss_interctc_layer15=58.622, loss_interctc_layer21=65.094, loss=65.595, backward_time=0.207, grad_norm=113.779, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.766e-05, train_time=1.632 +[gpua006:0/64] 2024-02-12 23:36:44,696 (trainer:756) INFO: 35epoch:train:14501-14600batch: iter_time=8.902e-05, forward_time=0.188, loss_ctc=64.484, loss_interctc_layer6=69.164, loss_interctc_layer12=57.042, loss_interctc_layer15=52.020, loss_interctc_layer21=66.852, loss=61.912, backward_time=0.221, grad_norm=93.542, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.148, optim0_lr0=6.765e-05, train_time=1.174 +[gpua006:0/64] 2024-02-12 23:38:38,956 (trainer:756) INFO: 35epoch:train:14601-14700batch: iter_time=8.965e-05, forward_time=0.184, loss_ctc=83.321, loss_interctc_layer6=81.059, loss_interctc_layer12=66.852, loss_interctc_layer15=61.000, loss_interctc_layer21=86.761, loss=75.799, backward_time=0.231, grad_norm=70.444, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.764e-05, train_time=1.142 +[gpua006:0/64] 2024-02-12 23:41:08,920 (trainer:756) INFO: 35epoch:train:14701-14800batch: iter_time=8.952e-05, forward_time=0.143, loss_ctc=76.152, loss_interctc_layer6=78.925, loss_interctc_layer12=64.915, loss_interctc_layer15=59.271, loss_interctc_layer21=79.180, loss=71.689, backward_time=0.208, grad_norm=76.376, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.764e-05, train_time=1.500 +[gpua006:0/64] 2024-02-12 23:42:59,116 (trainer:756) INFO: 35epoch:train:14801-14900batch: iter_time=8.999e-05, forward_time=0.143, loss_ctc=88.591, loss_interctc_layer6=79.893, loss_interctc_layer12=65.728, loss_interctc_layer15=59.905, loss_interctc_layer21=92.171, loss=77.257, backward_time=0.209, grad_norm=74.918, clip=100.000, loss_scale=2.454e+31, optim_step_time=0.136, optim0_lr0=6.763e-05, train_time=1.102 +[gpua006:0/64] 2024-02-12 23:45:00,575 (trainer:756) INFO: 35epoch:train:14901-15000batch: iter_time=8.475e-05, forward_time=0.144, loss_ctc=74.638, loss_interctc_layer6=77.128, loss_interctc_layer12=63.232, loss_interctc_layer15=57.587, loss_interctc_layer21=77.364, loss=69.990, backward_time=0.209, grad_norm=70.662, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.762e-05, train_time=1.214 +[gpua006:0/64] 2024-02-13 00:16:02,654 (trainer:355) INFO: 35epoch results: [train] iter_time=0.231, forward_time=0.165, loss_ctc=71.962, loss_interctc_layer6=78.185, loss_interctc_layer12=64.652, loss_interctc_layer15=59.177, loss_interctc_layer21=74.616, loss=69.718, backward_time=0.218, grad_norm=77.781, clip=100.000, loss_scale=1.875e+31, optim_step_time=0.138, optim0_lr0=6.811e-05, train_time=1.512, time=6 hours, 18 minutes and 33.84 seconds, total_count=525000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=41.565, cer_ctc=0.194, loss_interctc_layer6=47.432, cer_interctc_layer6=0.212, loss_interctc_layer12=34.748, cer_interctc_layer12=0.145, loss_interctc_layer15=30.396, cer_interctc_layer15=0.121, loss_interctc_layer21=44.043, cer_interctc_layer21=0.206, loss=39.637, time=30 minutes and 38.16 seconds, total_count=163485, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-13 00:16:23,370 (trainer:410) INFO: The best model has been updated: valid.total_count +[gpua006:0/64] 2024-02-13 00:16:23,628 (average_nbest_models:69) INFO: Averaging 5best models: criterion="valid.cer_ctc": exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/valid.cer_ctc.ave_5best.till35epoch.pth +[gpua006:0/64] 2024-02-13 00:17:07,665 (average_nbest_models:69) INFO: Averaging 5best models: criterion="valid.loss_ctc": exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/valid.loss_ctc.ave_5best.till35epoch.pth +[gpua006:0/64] 2024-02-13 00:17:16,021 (average_nbest_models:69) INFO: Averaging 5best models: criterion="valid.total_count": exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/valid.total_count.ave_5best.till35epoch.pth +[gpua006:0/64] 2024-02-13 00:17:23,431 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/30epoch.pth +[gpua006:0/64] 2024-02-13 00:17:23,722 (trainer:289) INFO: 36/45epoch started. Estimated time to finish: 2 days, 21 hours and 56 minutes +[gpua006:0/64] 2024-02-13 00:17:24,577 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-13 00:17:43,432 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 00:17:46,876 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 00:17:46,876 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-13 00:17:46,904 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 00:22:50,707 (trainer:756) INFO: 36epoch:train:1-100batch: iter_time=2.171, forward_time=0.172, loss_ctc=83.645, loss_interctc_layer6=87.629, loss_interctc_layer12=72.792, loss_interctc_layer15=66.824, loss_interctc_layer21=86.776, loss=79.533, backward_time=0.218, grad_norm=75.959, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.762e-05, train_time=3.261 +[gpua006:0/64] 2024-02-13 00:24:27,266 (trainer:756) INFO: 36epoch:train:101-200batch: iter_time=8.492e-05, forward_time=0.145, loss_ctc=101.134, loss_interctc_layer6=94.253, loss_interctc_layer12=78.906, loss_interctc_layer15=72.553, loss_interctc_layer21=104.730, loss=90.315, backward_time=0.210, grad_norm=83.348, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.135, optim0_lr0=6.761e-05, train_time=0.966 +[gpua006:0/64] 2024-02-13 00:26:35,270 (trainer:756) INFO: 36epoch:train:201-300batch: iter_time=8.466e-05, forward_time=0.143, loss_ctc=80.756, loss_interctc_layer6=86.354, loss_interctc_layer12=71.685, loss_interctc_layer15=65.765, loss_interctc_layer21=83.529, loss=77.618, backward_time=0.208, grad_norm=84.604, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.760e-05, train_time=1.280 +[gpua006:0/64] 2024-02-13 00:28:03,575 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 00:28:23,099 (trainer:756) INFO: 36epoch:train:301-400batch: iter_time=8.778e-05, forward_time=0.142, loss_ctc=73.023, loss_interctc_layer6=79.638, loss_interctc_layer12=66.267, loss_interctc_layer15=60.847, loss_interctc_layer21=75.763, loss=71.107, backward_time=0.207, grad_norm=91.494, clip=100.000, loss_scale=3.647e+31, optim_step_time=0.136, optim0_lr0=6.760e-05, train_time=1.078 +[gpua006:0/64] 2024-02-13 00:30:15,278 (trainer:756) INFO: 36epoch:train:401-500batch: iter_time=9.000e-05, forward_time=0.142, loss_ctc=84.338, loss_interctc_layer6=83.764, loss_interctc_layer12=69.624, loss_interctc_layer15=63.835, loss_interctc_layer21=87.429, loss=77.798, backward_time=0.208, grad_norm=124.083, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.759e-05, train_time=1.122 +[gpua006:0/64] 2024-02-13 00:32:20,852 (trainer:756) INFO: 36epoch:train:501-600batch: iter_time=8.730e-05, forward_time=0.144, loss_ctc=92.995, loss_interctc_layer6=96.330, loss_interctc_layer12=79.733, loss_interctc_layer15=72.812, loss_interctc_layer21=96.300, loss=87.634, backward_time=0.209, grad_norm=108.333, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.758e-05, train_time=1.256 +[gpua006:0/64] 2024-02-13 00:34:17,637 (trainer:756) INFO: 36epoch:train:601-700batch: iter_time=9.070e-05, forward_time=0.145, loss_ctc=79.447, loss_interctc_layer6=89.174, loss_interctc_layer12=74.126, loss_interctc_layer15=68.136, loss_interctc_layer21=82.001, loss=78.577, backward_time=0.208, grad_norm=89.936, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.758e-05, train_time=1.168 +[gpua006:0/64] 2024-02-13 00:36:31,754 (trainer:756) INFO: 36epoch:train:701-800batch: iter_time=9.826e-05, forward_time=0.142, loss_ctc=65.048, loss_interctc_layer6=65.566, loss_interctc_layer12=53.990, loss_interctc_layer15=49.234, loss_interctc_layer21=67.593, loss=60.286, backward_time=0.208, grad_norm=56.322, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.757e-05, train_time=1.341 +[gpua006:0/64] 2024-02-13 00:38:54,423 (trainer:756) INFO: 36epoch:train:801-900batch: iter_time=1.079e-04, forward_time=0.160, loss_ctc=88.588, loss_interctc_layer6=90.851, loss_interctc_layer12=75.060, loss_interctc_layer15=68.682, loss_interctc_layer21=91.957, loss=83.028, backward_time=0.209, grad_norm=91.524, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.757e-05, train_time=1.426 +[gpua006:0/64] 2024-02-13 00:41:19,344 (trainer:756) INFO: 36epoch:train:901-1000batch: iter_time=8.640e-05, forward_time=0.239, loss_ctc=74.616, loss_interctc_layer6=81.097, loss_interctc_layer12=67.144, loss_interctc_layer15=61.541, loss_interctc_layer21=77.338, loss=72.347, backward_time=0.232, grad_norm=78.811, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.756e-05, train_time=1.449 +[gpua006:0/64] 2024-02-13 00:43:22,221 (trainer:756) INFO: 36epoch:train:1001-1100batch: iter_time=9.185e-05, forward_time=0.143, loss_ctc=94.903, loss_interctc_layer6=87.666, loss_interctc_layer12=72.067, loss_interctc_layer15=65.813, loss_interctc_layer21=98.279, loss=83.746, backward_time=0.205, grad_norm=83.665, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.755e-05, train_time=1.228 +[gpua006:0/64] 2024-02-13 00:45:33,353 (trainer:756) INFO: 36epoch:train:1101-1200batch: iter_time=8.838e-05, forward_time=0.143, loss_ctc=93.380, loss_interctc_layer6=91.932, loss_interctc_layer12=77.196, loss_interctc_layer15=71.297, loss_interctc_layer21=96.720, loss=86.105, backward_time=0.206, grad_norm=92.143, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.755e-05, train_time=1.312 +[gpua006:0/64] 2024-02-13 00:46:55,097 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-13 00:47:13,917 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 00:47:17,345 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 00:47:17,345 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-13 00:47:17,348 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 00:52:09,213 (trainer:756) INFO: 36epoch:train:1201-1300batch: iter_time=2.662, forward_time=0.206, loss_ctc=84.927, loss_interctc_layer6=93.245, loss_interctc_layer12=77.215, loss_interctc_layer15=70.794, loss_interctc_layer21=88.088, loss=82.854, backward_time=0.224, grad_norm=91.461, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.754e-05, train_time=3.958 +[gpua006:0/64] 2024-02-13 00:53:59,390 (trainer:756) INFO: 36epoch:train:1301-1400batch: iter_time=8.375e-05, forward_time=0.143, loss_ctc=77.821, loss_interctc_layer6=84.823, loss_interctc_layer12=70.570, loss_interctc_layer15=64.943, loss_interctc_layer21=80.675, loss=75.766, backward_time=0.208, grad_norm=65.034, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.753e-05, train_time=1.102 +[gpua006:0/64] 2024-02-13 00:55:46,285 (trainer:756) INFO: 36epoch:train:1401-1500batch: iter_time=4.054e-04, forward_time=0.144, loss_ctc=90.834, loss_interctc_layer6=90.573, loss_interctc_layer12=75.260, loss_interctc_layer15=69.089, loss_interctc_layer21=93.980, loss=83.947, backward_time=0.209, grad_norm=87.990, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.144, optim0_lr0=6.753e-05, train_time=1.069 +[gpua006:0/64] 2024-02-13 00:58:30,786 (trainer:756) INFO: 36epoch:train:1501-1600batch: iter_time=8.256e-05, forward_time=0.276, loss_ctc=68.260, loss_interctc_layer6=80.330, loss_interctc_layer12=66.711, loss_interctc_layer15=61.185, loss_interctc_layer21=70.791, loss=69.455, backward_time=0.231, grad_norm=63.375, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.752e-05, train_time=1.645 +[gpua006:0/64] 2024-02-13 01:00:25,760 (trainer:756) INFO: 36epoch:train:1601-1700batch: iter_time=8.515e-05, forward_time=0.143, loss_ctc=74.235, loss_interctc_layer6=86.024, loss_interctc_layer12=71.666, loss_interctc_layer15=65.874, loss_interctc_layer21=77.093, loss=74.978, backward_time=0.207, grad_norm=141.605, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.751e-05, train_time=1.150 +[gpua006:0/64] 2024-02-13 01:02:19,627 (trainer:756) INFO: 36epoch:train:1701-1800batch: iter_time=8.424e-05, forward_time=0.144, loss_ctc=89.227, loss_interctc_layer6=89.368, loss_interctc_layer12=73.389, loss_interctc_layer15=66.778, loss_interctc_layer21=92.534, loss=82.259, backward_time=0.208, grad_norm=91.150, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.751e-05, train_time=1.139 +[gpua006:0/64] 2024-02-13 01:04:47,273 (trainer:756) INFO: 36epoch:train:1801-1900batch: iter_time=8.193e-05, forward_time=0.143, loss_ctc=76.056, loss_interctc_layer6=87.691, loss_interctc_layer12=72.315, loss_interctc_layer15=66.208, loss_interctc_layer21=78.591, loss=76.172, backward_time=0.206, grad_norm=117.070, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.750e-05, train_time=1.476 +[gpua006:0/64] 2024-02-13 01:06:44,802 (trainer:756) INFO: 36epoch:train:1901-2000batch: iter_time=8.443e-05, forward_time=0.188, loss_ctc=74.915, loss_interctc_layer6=76.575, loss_interctc_layer12=63.148, loss_interctc_layer15=57.712, loss_interctc_layer21=77.750, loss=70.020, backward_time=0.221, grad_norm=64.347, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.750e-05, train_time=1.175 +[gpua006:0/64] 2024-02-13 01:08:45,867 (trainer:756) INFO: 36epoch:train:2001-2100batch: iter_time=8.256e-05, forward_time=0.208, loss_ctc=71.969, loss_interctc_layer6=78.814, loss_interctc_layer12=64.934, loss_interctc_layer15=59.288, loss_interctc_layer21=74.782, loss=69.957, backward_time=0.226, grad_norm=83.717, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.749e-05, train_time=1.210 +[gpua006:0/64] 2024-02-13 01:11:19,062 (trainer:756) INFO: 36epoch:train:2101-2200batch: iter_time=8.735e-05, forward_time=0.143, loss_ctc=74.635, loss_interctc_layer6=85.272, loss_interctc_layer12=70.446, loss_interctc_layer15=64.437, loss_interctc_layer21=77.539, loss=74.466, backward_time=0.208, grad_norm=86.803, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.748e-05, train_time=1.532 +[gpua006:0/64] 2024-02-13 01:13:27,111 (trainer:756) INFO: 36epoch:train:2201-2300batch: iter_time=8.715e-05, forward_time=0.143, loss_ctc=80.750, loss_interctc_layer6=82.638, loss_interctc_layer12=67.975, loss_interctc_layer15=62.016, loss_interctc_layer21=83.652, loss=75.406, backward_time=0.207, grad_norm=79.688, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.748e-05, train_time=1.281 +[gpua006:0/64] 2024-02-13 01:15:16,836 (trainer:756) INFO: 36epoch:train:2301-2400batch: iter_time=8.165e-05, forward_time=0.144, loss_ctc=85.327, loss_interctc_layer6=88.262, loss_interctc_layer12=73.399, loss_interctc_layer15=67.426, loss_interctc_layer21=88.581, loss=80.599, backward_time=0.209, grad_norm=119.900, clip=100.000, loss_scale=2.434e+31, optim_step_time=0.137, optim0_lr0=6.747e-05, train_time=1.097 +[gpua006:0/64] 2024-02-13 01:16:58,240 (trainer:756) INFO: 36epoch:train:2401-2500batch: iter_time=8.057e-05, forward_time=0.143, loss_ctc=82.648, loss_interctc_layer6=90.978, loss_interctc_layer12=75.868, loss_interctc_layer15=69.799, loss_interctc_layer21=85.507, loss=80.960, backward_time=0.209, grad_norm=93.872, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.746e-05, train_time=1.014 +[gpua006:0/64] 2024-02-13 01:17:18,270 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-13 01:17:36,954 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 01:17:40,493 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 01:17:40,493 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-13 01:17:40,498 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 01:23:46,871 (trainer:756) INFO: 36epoch:train:2501-2600batch: iter_time=2.387, forward_time=0.229, loss_ctc=81.322, loss_interctc_layer6=85.447, loss_interctc_layer12=70.693, loss_interctc_layer15=64.819, loss_interctc_layer21=84.314, loss=77.319, backward_time=0.241, grad_norm=78.303, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=6.746e-05, train_time=4.086 +[gpua006:0/64] 2024-02-13 01:25:30,478 (trainer:756) INFO: 36epoch:train:2601-2700batch: iter_time=8.301e-05, forward_time=0.143, loss_ctc=99.501, loss_interctc_layer6=93.347, loss_interctc_layer12=77.742, loss_interctc_layer15=71.580, loss_interctc_layer21=103.158, loss=89.066, backward_time=0.209, grad_norm=82.000, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.745e-05, train_time=1.036 +[gpua006:0/64] 2024-02-13 01:28:02,422 (trainer:756) INFO: 36epoch:train:2701-2800batch: iter_time=8.235e-05, forward_time=0.144, loss_ctc=81.156, loss_interctc_layer6=85.174, loss_interctc_layer12=70.515, loss_interctc_layer15=64.738, loss_interctc_layer21=83.920, loss=77.101, backward_time=0.208, grad_norm=82.176, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.744e-05, train_time=1.520 +[gpua006:0/64] 2024-02-13 01:29:43,769 (trainer:756) INFO: 36epoch:train:2801-2900batch: iter_time=8.337e-05, forward_time=0.143, loss_ctc=70.528, loss_interctc_layer6=77.502, loss_interctc_layer12=64.180, loss_interctc_layer15=58.872, loss_interctc_layer21=73.164, loss=68.849, backward_time=0.208, grad_norm=97.807, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.744e-05, train_time=1.013 +[gpua006:0/64] 2024-02-13 01:31:48,254 (trainer:756) INFO: 36epoch:train:2901-3000batch: iter_time=7.909e-05, forward_time=0.142, loss_ctc=81.193, loss_interctc_layer6=82.325, loss_interctc_layer12=68.146, loss_interctc_layer15=62.305, loss_interctc_layer21=84.385, loss=75.671, backward_time=0.208, grad_norm=94.760, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.743e-05, train_time=1.245 +[gpua006:0/64] 2024-02-13 01:33:48,984 (trainer:756) INFO: 36epoch:train:3001-3100batch: iter_time=7.946e-05, forward_time=0.144, loss_ctc=91.622, loss_interctc_layer6=95.444, loss_interctc_layer12=78.618, loss_interctc_layer15=71.728, loss_interctc_layer21=94.896, loss=86.462, backward_time=0.208, grad_norm=81.739, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.742e-05, train_time=1.207 +[gpua006:0/64] 2024-02-13 01:35:59,489 (trainer:756) INFO: 36epoch:train:3101-3200batch: iter_time=7.600e-05, forward_time=0.145, loss_ctc=76.793, loss_interctc_layer6=86.611, loss_interctc_layer12=71.402, loss_interctc_layer15=65.335, loss_interctc_layer21=79.485, loss=75.925, backward_time=0.211, grad_norm=100.147, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.742e-05, train_time=1.305 +[gpua006:0/64] 2024-02-13 01:37:51,587 (trainer:756) INFO: 36epoch:train:3201-3300batch: iter_time=8.381e-05, forward_time=0.141, loss_ctc=64.896, loss_interctc_layer6=65.014, loss_interctc_layer12=53.392, loss_interctc_layer15=48.716, loss_interctc_layer21=67.504, loss=59.904, backward_time=0.207, grad_norm=106.638, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.741e-05, train_time=1.121 +[gpua006:0/64] 2024-02-13 01:38:38,263 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 01:39:49,724 (trainer:756) INFO: 36epoch:train:3301-3400batch: iter_time=1.046e-04, forward_time=0.204, loss_ctc=86.579, loss_interctc_layer6=90.032, loss_interctc_layer12=74.214, loss_interctc_layer15=67.868, loss_interctc_layer21=89.971, loss=81.733, backward_time=0.223, grad_norm=75.474, clip=100.000, loss_scale=2.930e+31, optim_step_time=0.137, optim0_lr0=6.741e-05, train_time=1.181 +[gpua006:0/64] 2024-02-13 01:41:48,990 (trainer:756) INFO: 36epoch:train:3401-3500batch: iter_time=9.350e-05, forward_time=0.175, loss_ctc=73.054, loss_interctc_layer6=80.409, loss_interctc_layer12=66.337, loss_interctc_layer15=60.680, loss_interctc_layer21=75.759, loss=71.248, backward_time=0.234, grad_norm=71.683, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.740e-05, train_time=1.192 +[gpua006:0/64] 2024-02-13 01:44:07,460 (trainer:756) INFO: 36epoch:train:3501-3600batch: iter_time=8.274e-05, forward_time=0.143, loss_ctc=94.399, loss_interctc_layer6=87.143, loss_interctc_layer12=71.668, loss_interctc_layer15=65.420, loss_interctc_layer21=97.853, loss=83.297, backward_time=0.207, grad_norm=83.981, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.739e-05, train_time=1.383 +[gpua006:0/64] 2024-02-13 01:45:54,066 (trainer:756) INFO: 36epoch:train:3601-3700batch: iter_time=7.745e-05, forward_time=0.149, loss_ctc=91.180, loss_interctc_layer6=90.626, loss_interctc_layer12=75.608, loss_interctc_layer15=69.600, loss_interctc_layer21=94.374, loss=84.278, backward_time=0.209, grad_norm=87.554, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.739e-05, train_time=1.067 +[gpua006:0/64] 2024-02-13 01:47:08,511 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-13 01:47:27,425 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 01:47:30,873 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 01:47:30,874 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-13 01:47:30,938 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 01:53:07,137 (trainer:756) INFO: 36epoch:train:3701-3800batch: iter_time=2.908, forward_time=0.144, loss_ctc=83.262, loss_interctc_layer6=91.858, loss_interctc_layer12=75.853, loss_interctc_layer15=69.470, loss_interctc_layer21=86.242, loss=81.337, backward_time=0.209, grad_norm=85.374, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.738e-05, train_time=4.330 +[gpua006:0/64] 2024-02-13 01:54:46,763 (trainer:756) INFO: 36epoch:train:3801-3900batch: iter_time=8.257e-05, forward_time=0.143, loss_ctc=77.953, loss_interctc_layer6=84.047, loss_interctc_layer12=69.933, loss_interctc_layer15=64.267, loss_interctc_layer21=80.668, loss=75.373, backward_time=0.209, grad_norm=120.491, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.737e-05, train_time=0.996 +[gpua006:0/64] 2024-02-13 01:56:29,284 (trainer:756) INFO: 36epoch:train:3901-4000batch: iter_time=8.230e-05, forward_time=0.145, loss_ctc=90.127, loss_interctc_layer6=90.071, loss_interctc_layer12=74.517, loss_interctc_layer15=68.358, loss_interctc_layer21=93.608, loss=83.336, backward_time=0.210, grad_norm=71.581, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.737e-05, train_time=1.025 +[gpua006:0/64] 2024-02-13 01:58:39,510 (trainer:756) INFO: 36epoch:train:4001-4100batch: iter_time=8.598e-05, forward_time=0.144, loss_ctc=67.552, loss_interctc_layer6=80.258, loss_interctc_layer12=66.594, loss_interctc_layer15=61.091, loss_interctc_layer21=69.903, loss=69.080, backward_time=0.209, grad_norm=69.466, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.736e-05, train_time=1.302 +[gpua006:0/64] 2024-02-13 02:00:26,439 (trainer:756) INFO: 36epoch:train:4101-4200batch: iter_time=8.985e-05, forward_time=0.144, loss_ctc=72.916, loss_interctc_layer6=85.363, loss_interctc_layer12=70.934, loss_interctc_layer15=65.140, loss_interctc_layer21=75.757, loss=74.022, backward_time=0.208, grad_norm=76.781, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.735e-05, train_time=1.069 +[gpua006:0/64] 2024-02-13 02:02:33,585 (trainer:756) INFO: 36epoch:train:4201-4300batch: iter_time=2.668e-04, forward_time=0.196, loss_ctc=90.760, loss_interctc_layer6=89.241, loss_interctc_layer12=73.300, loss_interctc_layer15=66.706, loss_interctc_layer21=94.287, loss=82.859, backward_time=0.223, grad_norm=105.129, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.735e-05, train_time=1.271 +[gpua006:0/64] 2024-02-13 02:04:40,538 (trainer:756) INFO: 36epoch:train:4301-4400batch: iter_time=9.278e-05, forward_time=0.179, loss_ctc=74.891, loss_interctc_layer6=86.879, loss_interctc_layer12=71.648, loss_interctc_layer15=65.622, loss_interctc_layer21=77.480, loss=75.304, backward_time=0.232, grad_norm=79.123, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.734e-05, train_time=1.269 +[gpua006:0/64] 2024-02-13 02:06:50,907 (trainer:756) INFO: 36epoch:train:4401-4500batch: iter_time=9.088e-05, forward_time=0.142, loss_ctc=74.534, loss_interctc_layer6=76.178, loss_interctc_layer12=62.622, loss_interctc_layer15=57.180, loss_interctc_layer21=77.438, loss=69.590, backward_time=0.207, grad_norm=58.100, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.734e-05, train_time=1.303 +[gpua006:0/64] 2024-02-13 02:08:58,831 (trainer:756) INFO: 36epoch:train:4501-4600batch: iter_time=8.246e-05, forward_time=0.145, loss_ctc=70.823, loss_interctc_layer6=78.181, loss_interctc_layer12=64.273, loss_interctc_layer15=58.593, loss_interctc_layer21=73.523, loss=69.079, backward_time=0.208, grad_norm=72.650, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.733e-05, train_time=1.280 +[gpua006:0/64] 2024-02-13 02:11:07,024 (trainer:756) INFO: 36epoch:train:4601-4700batch: iter_time=8.327e-05, forward_time=0.143, loss_ctc=75.159, loss_interctc_layer6=85.558, loss_interctc_layer12=70.575, loss_interctc_layer15=64.531, loss_interctc_layer21=78.139, loss=74.792, backward_time=0.208, grad_norm=95.051, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.732e-05, train_time=1.282 +[gpua006:0/64] 2024-02-13 02:12:51,571 (trainer:756) INFO: 36epoch:train:4701-4800batch: iter_time=8.344e-05, forward_time=0.143, loss_ctc=80.233, loss_interctc_layer6=82.545, loss_interctc_layer12=67.696, loss_interctc_layer15=61.715, loss_interctc_layer21=83.201, loss=75.078, backward_time=0.208, grad_norm=78.805, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.732e-05, train_time=1.045 +[gpua006:0/64] 2024-02-13 02:14:55,187 (trainer:756) INFO: 36epoch:train:4801-4900batch: iter_time=8.549e-05, forward_time=0.143, loss_ctc=83.935, loss_interctc_layer6=87.122, loss_interctc_layer12=72.115, loss_interctc_layer15=66.217, loss_interctc_layer21=86.921, loss=79.262, backward_time=0.208, grad_norm=90.690, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.731e-05, train_time=1.236 +[gpua006:0/64] 2024-02-13 02:16:46,809 (trainer:756) INFO: 36epoch:train:4901-5000batch: iter_time=8.468e-05, forward_time=0.144, loss_ctc=82.285, loss_interctc_layer6=90.337, loss_interctc_layer12=75.098, loss_interctc_layer15=69.031, loss_interctc_layer21=84.978, loss=80.346, backward_time=0.209, grad_norm=136.019, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.730e-05, train_time=1.116 +[gpua006:0/64] 2024-02-13 02:17:06,835 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-13 02:17:25,688 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 02:17:29,155 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 02:17:29,155 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-13 02:17:29,162 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 02:22:30,434 (trainer:756) INFO: 36epoch:train:5001-5100batch: iter_time=2.413, forward_time=0.164, loss_ctc=77.759, loss_interctc_layer6=84.977, loss_interctc_layer12=70.208, loss_interctc_layer15=64.307, loss_interctc_layer21=80.662, loss=75.583, backward_time=0.212, grad_norm=84.087, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.730e-05, train_time=3.436 +[gpua006:0/64] 2024-02-13 02:24:12,054 (trainer:756) INFO: 36epoch:train:5101-5200batch: iter_time=8.968e-05, forward_time=0.160, loss_ctc=90.676, loss_interctc_layer6=92.628, loss_interctc_layer12=77.207, loss_interctc_layer15=70.958, loss_interctc_layer21=93.924, loss=85.079, backward_time=0.216, grad_norm=80.972, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.729e-05, train_time=1.016 +[gpua006:0/64] 2024-02-13 02:26:19,446 (trainer:756) INFO: 36epoch:train:5201-5300batch: iter_time=8.983e-05, forward_time=0.144, loss_ctc=77.519, loss_interctc_layer6=84.974, loss_interctc_layer12=70.330, loss_interctc_layer15=64.526, loss_interctc_layer21=80.324, loss=75.535, backward_time=0.209, grad_norm=76.851, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.728e-05, train_time=1.274 +[gpua006:0/64] 2024-02-13 02:28:11,538 (trainer:756) INFO: 36epoch:train:5301-5400batch: iter_time=9.817e-05, forward_time=0.144, loss_ctc=66.677, loss_interctc_layer6=77.529, loss_interctc_layer12=64.177, loss_interctc_layer15=58.819, loss_interctc_layer21=69.267, loss=67.294, backward_time=0.210, grad_norm=64.687, clip=100.000, loss_scale=3.144e+31, optim_step_time=0.136, optim0_lr0=6.728e-05, train_time=1.121 +[gpua006:0/64] 2024-02-13 02:30:24,521 (trainer:756) INFO: 36epoch:train:5401-5500batch: iter_time=9.715e-05, forward_time=0.143, loss_ctc=76.837, loss_interctc_layer6=82.438, loss_interctc_layer12=68.316, loss_interctc_layer15=62.546, loss_interctc_layer21=79.828, loss=73.993, backward_time=0.208, grad_norm=73.091, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.727e-05, train_time=1.327 +[gpua006:0/64] 2024-02-13 02:32:23,468 (trainer:756) INFO: 36epoch:train:5501-5600batch: iter_time=8.972e-05, forward_time=0.155, loss_ctc=89.717, loss_interctc_layer6=95.391, loss_interctc_layer12=78.550, loss_interctc_layer15=71.635, loss_interctc_layer21=92.977, loss=85.654, backward_time=0.209, grad_norm=83.162, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.727e-05, train_time=1.191 +[gpua006:0/64] 2024-02-13 02:34:37,214 (trainer:756) INFO: 36epoch:train:5601-5700batch: iter_time=9.267e-05, forward_time=0.144, loss_ctc=73.907, loss_interctc_layer6=86.776, loss_interctc_layer12=71.601, loss_interctc_layer15=65.523, loss_interctc_layer21=76.523, loss=74.866, backward_time=0.209, grad_norm=69.527, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.726e-05, train_time=1.337 +[gpua006:0/64] 2024-02-13 02:36:55,892 (trainer:756) INFO: 36epoch:train:5701-5800batch: iter_time=8.989e-05, forward_time=0.143, loss_ctc=63.326, loss_interctc_layer6=64.743, loss_interctc_layer12=53.073, loss_interctc_layer15=48.212, loss_interctc_layer21=65.920, loss=59.055, backward_time=0.208, grad_norm=63.410, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.725e-05, train_time=1.387 +[gpua006:0/64] 2024-02-13 02:38:48,796 (trainer:756) INFO: 36epoch:train:5801-5900batch: iter_time=8.212e-05, forward_time=0.144, loss_ctc=82.860, loss_interctc_layer6=89.389, loss_interctc_layer12=73.639, loss_interctc_layer15=67.204, loss_interctc_layer21=85.861, loss=79.791, backward_time=0.208, grad_norm=75.638, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.725e-05, train_time=1.129 +[gpua006:0/64] 2024-02-13 02:40:45,602 (trainer:756) INFO: 36epoch:train:5901-6000batch: iter_time=8.805e-05, forward_time=0.143, loss_ctc=67.145, loss_interctc_layer6=79.886, loss_interctc_layer12=66.007, loss_interctc_layer15=60.311, loss_interctc_layer21=69.749, loss=68.620, backward_time=0.209, grad_norm=73.141, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.724e-05, train_time=1.168 +[gpua006:0/64] 2024-02-13 02:42:53,393 (trainer:756) INFO: 36epoch:train:6001-6100batch: iter_time=8.755e-05, forward_time=0.197, loss_ctc=90.587, loss_interctc_layer6=87.073, loss_interctc_layer12=71.645, loss_interctc_layer15=65.385, loss_interctc_layer21=93.957, loss=81.729, backward_time=0.217, grad_norm=86.910, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.723e-05, train_time=1.278 +[gpua006:0/64] 2024-02-13 02:44:43,153 (trainer:756) INFO: 36epoch:train:6101-6200batch: iter_time=8.352e-04, forward_time=0.186, loss_ctc=86.392, loss_interctc_layer6=89.761, loss_interctc_layer12=74.720, loss_interctc_layer15=68.702, loss_interctc_layer21=89.614, loss=81.838, backward_time=0.225, grad_norm=89.159, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=6.723e-05, train_time=1.097 +[gpua006:0/64] 2024-02-13 02:45:52,242 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-13 02:46:11,246 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 02:46:14,713 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 02:46:14,713 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-13 02:46:14,716 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 02:50:56,076 (trainer:756) INFO: 36epoch:train:6201-6300batch: iter_time=2.586, forward_time=0.161, loss_ctc=81.052, loss_interctc_layer6=91.552, loss_interctc_layer12=75.395, loss_interctc_layer15=68.908, loss_interctc_layer21=83.866, loss=80.155, backward_time=0.212, grad_norm=94.959, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.722e-05, train_time=3.728 +[gpua006:0/64] 2024-02-13 02:52:47,009 (trainer:756) INFO: 36epoch:train:6301-6400batch: iter_time=8.209e-05, forward_time=0.142, loss_ctc=76.433, loss_interctc_layer6=83.667, loss_interctc_layer12=69.509, loss_interctc_layer15=63.835, loss_interctc_layer21=78.967, loss=74.482, backward_time=0.208, grad_norm=87.619, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.722e-05, train_time=1.110 +[gpua006:0/64] 2024-02-13 02:54:40,109 (trainer:756) INFO: 36epoch:train:6401-6500batch: iter_time=8.297e-05, forward_time=0.143, loss_ctc=88.875, loss_interctc_layer6=89.482, loss_interctc_layer12=74.212, loss_interctc_layer15=67.951, loss_interctc_layer21=92.102, loss=82.524, backward_time=0.209, grad_norm=84.422, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.721e-05, train_time=1.131 +[gpua006:0/64] 2024-02-13 02:56:33,368 (trainer:756) INFO: 36epoch:train:6501-6600batch: iter_time=8.564e-05, forward_time=0.142, loss_ctc=67.719, loss_interctc_layer6=79.807, loss_interctc_layer12=66.309, loss_interctc_layer15=60.780, loss_interctc_layer21=70.345, loss=68.992, backward_time=0.208, grad_norm=87.071, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.720e-05, train_time=1.132 +[gpua006:0/64] 2024-02-13 02:58:25,069 (trainer:756) INFO: 36epoch:train:6601-6700batch: iter_time=8.440e-05, forward_time=0.145, loss_ctc=72.862, loss_interctc_layer6=85.407, loss_interctc_layer12=70.948, loss_interctc_layer15=65.262, loss_interctc_layer21=75.630, loss=74.022, backward_time=0.208, grad_norm=125.256, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.720e-05, train_time=1.117 +[gpua006:0/64] 2024-02-13 03:00:38,385 (trainer:756) INFO: 36epoch:train:6701-6800batch: iter_time=8.160e-05, forward_time=0.143, loss_ctc=88.914, loss_interctc_layer6=88.786, loss_interctc_layer12=72.880, loss_interctc_layer15=66.243, loss_interctc_layer21=92.313, loss=81.827, backward_time=0.208, grad_norm=110.774, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.719e-05, train_time=1.333 +[gpua006:0/64] 2024-02-13 03:02:40,796 (trainer:756) INFO: 36epoch:train:6801-6900batch: iter_time=9.356e-05, forward_time=0.143, loss_ctc=74.160, loss_interctc_layer6=86.487, loss_interctc_layer12=71.236, loss_interctc_layer15=65.166, loss_interctc_layer21=76.730, loss=74.756, backward_time=0.207, grad_norm=92.051, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.718e-05, train_time=1.224 +[gpua006:0/64] 2024-02-13 03:03:23,889 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 03:05:13,677 (trainer:756) INFO: 36epoch:train:6901-7000batch: iter_time=8.554e-05, forward_time=0.179, loss_ctc=74.202, loss_interctc_layer6=75.871, loss_interctc_layer12=62.438, loss_interctc_layer15=56.971, loss_interctc_layer21=77.162, loss=69.329, backward_time=0.225, grad_norm=127.951, clip=100.000, loss_scale=2.499e+31, optim_step_time=0.137, optim0_lr0=6.718e-05, train_time=1.529 +[gpua006:0/64] 2024-02-13 03:07:34,414 (trainer:756) INFO: 36epoch:train:7001-7100batch: iter_time=8.852e-05, forward_time=0.166, loss_ctc=71.274, loss_interctc_layer6=78.710, loss_interctc_layer12=64.654, loss_interctc_layer15=59.015, loss_interctc_layer21=74.024, loss=69.535, backward_time=0.229, grad_norm=63.187, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.717e-05, train_time=1.407 +[gpua006:0/64] 2024-02-13 03:09:47,699 (trainer:756) INFO: 36epoch:train:7101-7200batch: iter_time=9.488e-05, forward_time=0.147, loss_ctc=74.231, loss_interctc_layer6=84.788, loss_interctc_layer12=69.969, loss_interctc_layer15=63.829, loss_interctc_layer21=77.246, loss=74.013, backward_time=0.207, grad_norm=91.946, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.716e-05, train_time=1.333 +[gpua006:0/64] 2024-02-13 03:11:47,186 (trainer:756) INFO: 36epoch:train:7201-7300batch: iter_time=8.906e-05, forward_time=0.155, loss_ctc=80.646, loss_interctc_layer6=82.418, loss_interctc_layer12=67.597, loss_interctc_layer15=61.581, loss_interctc_layer21=83.576, loss=75.164, backward_time=0.211, grad_norm=73.762, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.716e-05, train_time=1.194 +[gpua006:0/64] 2024-02-13 03:14:26,372 (trainer:756) INFO: 36epoch:train:7301-7400batch: iter_time=8.372e-05, forward_time=0.143, loss_ctc=84.421, loss_interctc_layer6=86.855, loss_interctc_layer12=71.967, loss_interctc_layer15=65.981, loss_interctc_layer21=87.633, loss=79.371, backward_time=0.206, grad_norm=71.018, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.715e-05, train_time=1.591 +[gpua006:0/64] 2024-02-13 03:16:38,849 (trainer:756) INFO: 36epoch:train:7401-7500batch: iter_time=8.518e-05, forward_time=0.143, loss_ctc=80.147, loss_interctc_layer6=89.359, loss_interctc_layer12=74.143, loss_interctc_layer15=68.052, loss_interctc_layer21=83.013, loss=78.943, backward_time=0.208, grad_norm=112.336, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.715e-05, train_time=1.326 +[gpua006:0/64] 2024-02-13 03:16:58,879 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-13 03:17:17,596 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 03:17:21,052 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 03:17:21,053 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-13 03:17:21,061 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 03:23:15,573 (trainer:756) INFO: 36epoch:train:7501-7600batch: iter_time=2.903, forward_time=0.157, loss_ctc=81.352, loss_interctc_layer6=85.181, loss_interctc_layer12=70.378, loss_interctc_layer15=64.472, loss_interctc_layer21=84.250, loss=77.127, backward_time=0.210, grad_norm=82.550, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.714e-05, train_time=3.967 +[gpua006:0/64] 2024-02-13 03:25:20,687 (trainer:756) INFO: 36epoch:train:7601-7700batch: iter_time=7.889e-05, forward_time=0.150, loss_ctc=99.500, loss_interctc_layer6=92.688, loss_interctc_layer12=77.098, loss_interctc_layer15=70.818, loss_interctc_layer21=103.353, loss=88.691, backward_time=0.224, grad_norm=77.600, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.713e-05, train_time=1.251 +[gpua006:0/64] 2024-02-13 03:25:56,618 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 03:27:15,430 (trainer:756) INFO: 36epoch:train:7701-7800batch: iter_time=7.663e-05, forward_time=0.180, loss_ctc=80.676, loss_interctc_layer6=84.455, loss_interctc_layer12=69.986, loss_interctc_layer15=64.148, loss_interctc_layer21=83.616, loss=76.576, backward_time=0.226, grad_norm=86.201, clip=100.000, loss_scale=1.270e+31, optim_step_time=0.140, optim0_lr0=6.713e-05, train_time=1.147 +[gpua006:0/64] 2024-02-13 03:29:03,146 (trainer:756) INFO: 36epoch:train:7801-7900batch: iter_time=7.903e-05, forward_time=0.148, loss_ctc=70.641, loss_interctc_layer6=77.006, loss_interctc_layer12=63.684, loss_interctc_layer15=58.441, loss_interctc_layer21=73.068, loss=68.568, backward_time=0.210, grad_norm=67.977, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.712e-05, train_time=1.077 +[gpua006:0/64] 2024-02-13 03:31:11,293 (trainer:756) INFO: 36epoch:train:7901-8000batch: iter_time=8.188e-05, forward_time=0.157, loss_ctc=81.122, loss_interctc_layer6=82.220, loss_interctc_layer12=67.889, loss_interctc_layer15=62.039, loss_interctc_layer21=84.277, loss=75.509, backward_time=0.215, grad_norm=80.546, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.711e-05, train_time=1.280 +[gpua006:0/64] 2024-02-13 03:33:44,054 (trainer:756) INFO: 36epoch:train:8001-8100batch: iter_time=8.548e-05, forward_time=0.146, loss_ctc=90.698, loss_interctc_layer6=94.443, loss_interctc_layer12=77.801, loss_interctc_layer15=70.929, loss_interctc_layer21=94.053, loss=85.585, backward_time=0.209, grad_norm=74.916, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.711e-05, train_time=1.528 +[gpua006:0/64] 2024-02-13 03:36:11,101 (trainer:756) INFO: 36epoch:train:8101-8200batch: iter_time=8.829e-05, forward_time=0.144, loss_ctc=76.126, loss_interctc_layer6=85.754, loss_interctc_layer12=70.664, loss_interctc_layer15=64.680, loss_interctc_layer21=78.818, loss=75.208, backward_time=0.208, grad_norm=76.886, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.710e-05, train_time=1.471 +[gpua006:0/64] 2024-02-13 03:38:07,788 (trainer:756) INFO: 36epoch:train:8201-8300batch: iter_time=8.219e-05, forward_time=0.142, loss_ctc=64.313, loss_interctc_layer6=64.510, loss_interctc_layer12=52.800, loss_interctc_layer15=48.038, loss_interctc_layer21=66.996, loss=59.331, backward_time=0.208, grad_norm=51.580, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.710e-05, train_time=1.167 +[gpua006:0/64] 2024-02-13 03:40:14,660 (trainer:756) INFO: 36epoch:train:8301-8400batch: iter_time=8.140e-05, forward_time=0.155, loss_ctc=87.281, loss_interctc_layer6=89.427, loss_interctc_layer12=73.739, loss_interctc_layer15=67.325, loss_interctc_layer21=90.683, loss=81.691, backward_time=0.209, grad_norm=88.489, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.709e-05, train_time=1.268 +[gpua006:0/64] 2024-02-13 03:42:11,274 (trainer:756) INFO: 36epoch:train:8401-8500batch: iter_time=8.138e-05, forward_time=0.143, loss_ctc=73.078, loss_interctc_layer6=79.908, loss_interctc_layer12=65.923, loss_interctc_layer15=60.407, loss_interctc_layer21=75.769, loss=71.017, backward_time=0.209, grad_norm=67.445, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.708e-05, train_time=1.166 +[gpua006:0/64] 2024-02-13 03:44:32,014 (trainer:756) INFO: 36epoch:train:8501-8600batch: iter_time=8.015e-05, forward_time=0.179, loss_ctc=93.679, loss_interctc_layer6=86.112, loss_interctc_layer12=70.701, loss_interctc_layer15=64.481, loss_interctc_layer21=97.294, loss=82.453, backward_time=0.216, grad_norm=85.201, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.708e-05, train_time=1.407 +[gpua006:0/64] 2024-02-13 03:46:38,432 (trainer:756) INFO: 36epoch:train:8601-8700batch: iter_time=8.229e-05, forward_time=0.176, loss_ctc=90.245, loss_interctc_layer6=89.573, loss_interctc_layer12=74.458, loss_interctc_layer15=68.470, loss_interctc_layer21=93.493, loss=83.248, backward_time=0.221, grad_norm=73.671, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.707e-05, train_time=1.264 +[gpua006:0/64] 2024-02-13 03:48:02,189 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-13 03:48:21,065 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 03:48:24,528 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 03:48:24,528 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-13 03:48:24,531 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 03:53:17,918 (trainer:756) INFO: 36epoch:train:8701-8800batch: iter_time=2.755, forward_time=0.152, loss_ctc=84.189, loss_interctc_layer6=90.655, loss_interctc_layer12=74.466, loss_interctc_layer15=68.067, loss_interctc_layer21=87.203, loss=80.916, backward_time=0.212, grad_norm=75.347, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.706e-05, train_time=3.994 +[gpua006:0/64] 2024-02-13 03:54:56,241 (trainer:756) INFO: 36epoch:train:8801-8900batch: iter_time=8.870e-05, forward_time=0.145, loss_ctc=83.331, loss_interctc_layer6=83.826, loss_interctc_layer12=69.677, loss_interctc_layer15=63.981, loss_interctc_layer21=86.174, loss=77.398, backward_time=0.208, grad_norm=86.985, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.706e-05, train_time=0.984 +[gpua006:0/64] 2024-02-13 03:56:56,848 (trainer:756) INFO: 36epoch:train:8901-9000batch: iter_time=8.529e-05, forward_time=0.144, loss_ctc=93.052, loss_interctc_layer6=88.552, loss_interctc_layer12=73.278, loss_interctc_layer15=67.111, loss_interctc_layer21=96.416, loss=83.682, backward_time=0.209, grad_norm=66.493, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.705e-05, train_time=1.206 +[gpua006:0/64] 2024-02-13 03:58:43,974 (trainer:756) INFO: 36epoch:train:9001-9100batch: iter_time=8.642e-05, forward_time=0.143, loss_ctc=70.868, loss_interctc_layer6=78.996, loss_interctc_layer12=65.343, loss_interctc_layer15=59.847, loss_interctc_layer21=73.460, loss=69.703, backward_time=0.208, grad_norm=75.297, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.705e-05, train_time=1.071 +[gpua006:0/64] 2024-02-13 04:01:14,344 (trainer:756) INFO: 36epoch:train:9101-9200batch: iter_time=8.526e-05, forward_time=0.143, loss_ctc=78.133, loss_interctc_layer6=84.828, loss_interctc_layer12=70.411, loss_interctc_layer15=64.566, loss_interctc_layer21=81.253, loss=75.838, backward_time=0.207, grad_norm=77.310, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.704e-05, train_time=1.503 +[gpua006:0/64] 2024-02-13 04:03:47,026 (trainer:756) INFO: 36epoch:train:9201-9300batch: iter_time=8.692e-05, forward_time=0.157, loss_ctc=93.214, loss_interctc_layer6=88.872, loss_interctc_layer12=73.058, loss_interctc_layer15=66.446, loss_interctc_layer21=96.745, loss=83.667, backward_time=0.211, grad_norm=85.938, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.703e-05, train_time=1.527 +[gpua006:0/64] 2024-02-13 04:06:11,390 (trainer:756) INFO: 36epoch:train:9301-9400batch: iter_time=8.588e-05, forward_time=0.159, loss_ctc=76.291, loss_interctc_layer6=85.899, loss_interctc_layer12=70.773, loss_interctc_layer15=64.776, loss_interctc_layer21=78.891, loss=75.326, backward_time=0.215, grad_norm=88.269, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.703e-05, train_time=1.443 +[gpua006:0/64] 2024-02-13 04:08:31,768 (trainer:756) INFO: 36epoch:train:9401-9500batch: iter_time=8.905e-05, forward_time=0.169, loss_ctc=76.443, loss_interctc_layer6=75.955, loss_interctc_layer12=62.429, loss_interctc_layer15=56.953, loss_interctc_layer21=79.286, loss=70.213, backward_time=0.224, grad_norm=79.946, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.702e-05, train_time=1.404 +[gpua006:0/64] 2024-02-13 04:10:38,508 (trainer:756) INFO: 36epoch:train:9501-9600batch: iter_time=8.456e-05, forward_time=0.150, loss_ctc=73.307, loss_interctc_layer6=78.021, loss_interctc_layer12=64.112, loss_interctc_layer15=58.530, loss_interctc_layer21=76.135, loss=70.021, backward_time=0.214, grad_norm=70.826, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.701e-05, train_time=1.267 +[gpua006:0/64] 2024-02-13 04:12:40,711 (trainer:756) INFO: 36epoch:train:9601-9700batch: iter_time=9.322e-05, forward_time=0.144, loss_ctc=80.805, loss_interctc_layer6=84.556, loss_interctc_layer12=69.766, loss_interctc_layer15=63.973, loss_interctc_layer21=83.783, loss=76.577, backward_time=0.209, grad_norm=94.632, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.701e-05, train_time=1.222 +[gpua006:0/64] 2024-02-13 04:14:40,789 (trainer:756) INFO: 36epoch:train:9701-9800batch: iter_time=8.656e-05, forward_time=0.146, loss_ctc=82.235, loss_interctc_layer6=82.448, loss_interctc_layer12=67.648, loss_interctc_layer15=61.699, loss_interctc_layer21=85.176, loss=75.841, backward_time=0.211, grad_norm=88.512, clip=100.000, loss_scale=1.765e+31, optim_step_time=0.137, optim0_lr0=6.700e-05, train_time=1.201 +[gpua006:0/64] 2024-02-13 04:17:00,620 (trainer:756) INFO: 36epoch:train:9801-9900batch: iter_time=8.668e-05, forward_time=0.151, loss_ctc=90.912, loss_interctc_layer6=86.638, loss_interctc_layer12=71.799, loss_interctc_layer15=65.806, loss_interctc_layer21=94.421, loss=81.915, backward_time=0.213, grad_norm=81.442, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.699e-05, train_time=1.395 +[gpua006:0/64] 2024-02-13 04:19:11,795 (trainer:756) INFO: 36epoch:train:9901-10000batch: iter_time=8.604e-05, forward_time=0.144, loss_ctc=83.408, loss_interctc_layer6=88.956, loss_interctc_layer12=73.695, loss_interctc_layer15=67.488, loss_interctc_layer21=86.326, loss=79.975, backward_time=0.209, grad_norm=84.928, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.699e-05, train_time=1.314 +[gpua006:0/64] 2024-02-13 04:19:31,825 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-13 04:19:50,592 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 04:19:54,439 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 04:19:54,439 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-13 04:19:54,465 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 04:25:37,766 (trainer:756) INFO: 36epoch:train:10001-10100batch: iter_time=2.762, forward_time=0.184, loss_ctc=81.815, loss_interctc_layer6=85.508, loss_interctc_layer12=70.547, loss_interctc_layer15=64.566, loss_interctc_layer21=84.824, loss=77.452, backward_time=0.219, grad_norm=122.500, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.698e-05, train_time=3.859 +[gpua006:0/64] 2024-02-13 04:27:33,434 (trainer:756) INFO: 36epoch:train:10101-10200batch: iter_time=8.823e-05, forward_time=0.146, loss_ctc=98.812, loss_interctc_layer6=91.994, loss_interctc_layer12=76.513, loss_interctc_layer15=70.168, loss_interctc_layer21=102.419, loss=87.981, backward_time=0.213, grad_norm=90.220, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.698e-05, train_time=1.156 +[gpua006:0/64] 2024-02-13 04:29:27,737 (trainer:756) INFO: 36epoch:train:10201-10300batch: iter_time=9.017e-05, forward_time=0.146, loss_ctc=79.423, loss_interctc_layer6=84.490, loss_interctc_layer12=69.945, loss_interctc_layer15=64.052, loss_interctc_layer21=82.202, loss=76.023, backward_time=0.210, grad_norm=66.908, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.697e-05, train_time=1.143 +[gpua006:0/64] 2024-02-13 04:31:37,513 (trainer:756) INFO: 36epoch:train:10301-10400batch: iter_time=9.559e-05, forward_time=0.164, loss_ctc=70.031, loss_interctc_layer6=76.839, loss_interctc_layer12=63.441, loss_interctc_layer15=58.092, loss_interctc_layer21=72.735, loss=68.227, backward_time=0.215, grad_norm=64.945, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.696e-05, train_time=1.297 +[gpua006:0/64] 2024-02-13 04:33:55,895 (trainer:756) INFO: 36epoch:train:10401-10500batch: iter_time=9.395e-05, forward_time=0.143, loss_ctc=81.853, loss_interctc_layer6=81.453, loss_interctc_layer12=67.327, loss_interctc_layer15=61.571, loss_interctc_layer21=85.214, loss=75.484, backward_time=0.209, grad_norm=62.257, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.696e-05, train_time=1.385 +[gpua006:0/64] 2024-02-13 04:36:53,068 (trainer:756) INFO: 36epoch:train:10501-10600batch: iter_time=8.636e-05, forward_time=0.144, loss_ctc=91.263, loss_interctc_layer6=94.752, loss_interctc_layer12=78.066, loss_interctc_layer15=71.146, loss_interctc_layer21=94.325, loss=85.910, backward_time=0.208, grad_norm=92.081, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.695e-05, train_time=1.772 +[gpua006:0/64] 2024-02-13 04:38:36,652 (trainer:756) INFO: 36epoch:train:10601-10700batch: iter_time=8.586e-05, forward_time=0.154, loss_ctc=75.566, loss_interctc_layer6=85.084, loss_interctc_layer12=70.186, loss_interctc_layer15=64.165, loss_interctc_layer21=78.286, loss=74.657, backward_time=0.212, grad_norm=124.031, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.694e-05, train_time=1.036 +[gpua006:0/64] 2024-02-13 04:40:50,163 (trainer:756) INFO: 36epoch:train:10701-10800batch: iter_time=8.587e-05, forward_time=0.225, loss_ctc=63.983, loss_interctc_layer6=64.200, loss_interctc_layer12=52.601, loss_interctc_layer15=47.814, loss_interctc_layer21=66.548, loss=59.029, backward_time=0.235, grad_norm=57.678, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.694e-05, train_time=1.335 +[gpua006:0/64] 2024-02-13 04:43:07,960 (trainer:756) INFO: 36epoch:train:10801-10900batch: iter_time=8.349e-05, forward_time=0.146, loss_ctc=86.179, loss_interctc_layer6=88.907, loss_interctc_layer12=73.246, loss_interctc_layer15=66.807, loss_interctc_layer21=89.511, loss=80.930, backward_time=0.210, grad_norm=75.905, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.693e-05, train_time=1.378 +[gpua006:0/64] 2024-02-13 04:45:29,395 (trainer:756) INFO: 36epoch:train:10901-11000batch: iter_time=8.411e-05, forward_time=0.143, loss_ctc=73.189, loss_interctc_layer6=79.549, loss_interctc_layer12=65.760, loss_interctc_layer15=60.123, loss_interctc_layer21=75.825, loss=70.889, backward_time=0.207, grad_norm=71.458, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.693e-05, train_time=1.414 +[gpua006:0/64] 2024-02-13 04:47:33,358 (trainer:756) INFO: 36epoch:train:11001-11100batch: iter_time=8.381e-05, forward_time=0.158, loss_ctc=92.583, loss_interctc_layer6=85.839, loss_interctc_layer12=70.416, loss_interctc_layer15=64.201, loss_interctc_layer21=95.995, loss=81.807, backward_time=0.212, grad_norm=81.086, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.692e-05, train_time=1.239 +[gpua006:0/64] 2024-02-13 04:49:20,879 (trainer:756) INFO: 36epoch:train:11101-11200batch: iter_time=8.198e-05, forward_time=0.144, loss_ctc=91.202, loss_interctc_layer6=90.011, loss_interctc_layer12=74.853, loss_interctc_layer15=68.778, loss_interctc_layer21=94.589, loss=83.887, backward_time=0.210, grad_norm=115.683, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.691e-05, train_time=1.074 +[gpua006:0/64] 2024-02-13 04:50:46,757 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-13 04:51:05,796 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 04:51:09,229 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 04:51:09,229 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-13 04:51:09,252 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 04:55:41,625 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 04:56:09,842 (trainer:756) INFO: 36epoch:train:11201-11300batch: iter_time=2.831, forward_time=0.146, loss_ctc=82.606, loss_interctc_layer6=90.178, loss_interctc_layer12=74.163, loss_interctc_layer15=67.772, loss_interctc_layer21=85.573, loss=80.058, backward_time=0.208, grad_norm=79.375, clip=100.000, loss_scale=1.731e+31, optim_step_time=0.136, optim0_lr0=6.691e-05, train_time=4.091 +[gpua006:0/64] 2024-02-13 04:58:07,370 (trainer:756) INFO: 36epoch:train:11301-11400batch: iter_time=8.991e-05, forward_time=0.144, loss_ctc=77.524, loss_interctc_layer6=83.504, loss_interctc_layer12=69.591, loss_interctc_layer15=63.865, loss_interctc_layer21=80.374, loss=74.971, backward_time=0.209, grad_norm=71.366, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.690e-05, train_time=1.175 +[gpua006:0/64] 2024-02-13 05:00:00,967 (trainer:756) INFO: 36epoch:train:11401-11500batch: iter_time=8.252e-05, forward_time=0.148, loss_ctc=88.876, loss_interctc_layer6=89.196, loss_interctc_layer12=73.763, loss_interctc_layer15=67.518, loss_interctc_layer21=92.226, loss=82.316, backward_time=0.210, grad_norm=95.718, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.689e-05, train_time=1.136 +[gpua006:0/64] 2024-02-13 05:02:28,093 (trainer:756) INFO: 36epoch:train:11501-11600batch: iter_time=8.709e-05, forward_time=0.178, loss_ctc=66.578, loss_interctc_layer6=79.134, loss_interctc_layer12=65.652, loss_interctc_layer15=60.149, loss_interctc_layer21=69.042, loss=68.111, backward_time=0.219, grad_norm=87.754, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.689e-05, train_time=1.471 +[gpua006:0/64] 2024-02-13 05:04:15,371 (trainer:756) INFO: 36epoch:train:11601-11700batch: iter_time=8.608e-05, forward_time=0.185, loss_ctc=72.407, loss_interctc_layer6=84.359, loss_interctc_layer12=70.069, loss_interctc_layer15=64.187, loss_interctc_layer21=75.045, loss=73.213, backward_time=0.229, grad_norm=74.979, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.688e-05, train_time=1.073 +[gpua006:0/64] 2024-02-13 05:06:31,283 (trainer:756) INFO: 36epoch:train:11701-11800batch: iter_time=9.186e-05, forward_time=0.148, loss_ctc=89.214, loss_interctc_layer6=87.848, loss_interctc_layer12=71.897, loss_interctc_layer15=65.251, loss_interctc_layer21=92.755, loss=81.393, backward_time=0.209, grad_norm=65.954, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.688e-05, train_time=1.359 +[gpua006:0/64] 2024-02-13 05:08:43,307 (trainer:756) INFO: 36epoch:train:11801-11900batch: iter_time=9.529e-05, forward_time=0.157, loss_ctc=73.733, loss_interctc_layer6=85.547, loss_interctc_layer12=70.328, loss_interctc_layer15=64.278, loss_interctc_layer21=76.232, loss=74.024, backward_time=0.213, grad_norm=77.575, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.687e-05, train_time=1.320 +[gpua006:0/64] 2024-02-13 05:10:42,134 (trainer:756) INFO: 36epoch:train:11901-12000batch: iter_time=8.728e-05, forward_time=0.147, loss_ctc=74.969, loss_interctc_layer6=75.983, loss_interctc_layer12=62.414, loss_interctc_layer15=56.837, loss_interctc_layer21=77.885, loss=69.618, backward_time=0.210, grad_norm=63.253, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.686e-05, train_time=1.187 +[gpua006:0/64] 2024-02-13 05:13:07,245 (trainer:756) INFO: 36epoch:train:12001-12100batch: iter_time=8.880e-05, forward_time=0.145, loss_ctc=69.890, loss_interctc_layer6=77.201, loss_interctc_layer12=63.436, loss_interctc_layer15=57.866, loss_interctc_layer21=72.756, loss=68.230, backward_time=0.207, grad_norm=67.076, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.686e-05, train_time=1.452 +[gpua006:0/64] 2024-02-13 05:15:07,818 (trainer:756) INFO: 36epoch:train:12101-12200batch: iter_time=9.046e-05, forward_time=0.142, loss_ctc=73.750, loss_interctc_layer6=84.641, loss_interctc_layer12=69.782, loss_interctc_layer15=63.773, loss_interctc_layer21=76.529, loss=73.695, backward_time=0.208, grad_norm=78.672, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.685e-05, train_time=1.206 +[gpua006:0/64] 2024-02-13 05:17:38,867 (trainer:756) INFO: 36epoch:train:12201-12300batch: iter_time=8.841e-05, forward_time=0.144, loss_ctc=79.198, loss_interctc_layer6=81.617, loss_interctc_layer12=66.837, loss_interctc_layer15=60.872, loss_interctc_layer21=82.163, loss=74.137, backward_time=0.210, grad_norm=79.667, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.685e-05, train_time=1.510 +[gpua006:0/64] 2024-02-13 05:19:42,413 (trainer:756) INFO: 36epoch:train:12301-12400batch: iter_time=1.777e-04, forward_time=0.229, loss_ctc=84.217, loss_interctc_layer6=87.024, loss_interctc_layer12=71.911, loss_interctc_layer15=65.959, loss_interctc_layer21=87.436, loss=79.309, backward_time=0.238, grad_norm=91.039, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.684e-05, train_time=1.235 +[gpua006:0/64] 2024-02-13 05:21:58,645 (trainer:756) INFO: 36epoch:train:12401-12500batch: iter_time=8.731e-05, forward_time=0.144, loss_ctc=80.589, loss_interctc_layer6=88.908, loss_interctc_layer12=73.682, loss_interctc_layer15=67.527, loss_interctc_layer21=83.392, loss=78.819, backward_time=0.210, grad_norm=95.669, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.683e-05, train_time=1.362 +[gpua006:0/64] 2024-02-13 05:22:18,675 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-13 05:22:37,497 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 05:22:40,940 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 05:22:40,940 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-13 05:22:40,955 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 05:28:13,741 (trainer:756) INFO: 36epoch:train:12501-12600batch: iter_time=2.749, forward_time=0.179, loss_ctc=80.412, loss_interctc_layer6=84.888, loss_interctc_layer12=69.984, loss_interctc_layer15=64.041, loss_interctc_layer21=83.391, loss=76.543, backward_time=0.218, grad_norm=82.339, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.683e-05, train_time=3.751 +[gpua006:0/64] 2024-02-13 05:30:10,849 (trainer:756) INFO: 36epoch:train:12601-12700batch: iter_time=2.614e-04, forward_time=0.246, loss_ctc=98.737, loss_interctc_layer6=91.928, loss_interctc_layer12=76.590, loss_interctc_layer15=70.136, loss_interctc_layer21=102.488, loss=87.976, backward_time=0.231, grad_norm=87.913, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.162, optim0_lr0=6.682e-05, train_time=1.170 +[gpua006:0/64] 2024-02-13 05:32:33,437 (trainer:756) INFO: 36epoch:train:12701-12800batch: iter_time=9.026e-05, forward_time=0.155, loss_ctc=79.356, loss_interctc_layer6=83.740, loss_interctc_layer12=69.243, loss_interctc_layer15=63.408, loss_interctc_layer21=82.275, loss=75.604, backward_time=0.212, grad_norm=70.281, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.681e-05, train_time=1.426 +[gpua006:0/64] 2024-02-13 05:35:27,699 (trainer:756) INFO: 36epoch:train:12801-12900batch: iter_time=8.867e-05, forward_time=0.143, loss_ctc=69.563, loss_interctc_layer6=76.855, loss_interctc_layer12=63.509, loss_interctc_layer15=58.003, loss_interctc_layer21=72.343, loss=68.055, backward_time=0.207, grad_norm=71.885, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.681e-05, train_time=1.742 +[gpua006:0/64] 2024-02-13 05:37:08,506 (trainer:756) INFO: 36epoch:train:12901-13000batch: iter_time=8.841e-05, forward_time=0.143, loss_ctc=80.142, loss_interctc_layer6=81.509, loss_interctc_layer12=67.260, loss_interctc_layer15=61.400, loss_interctc_layer21=83.380, loss=74.738, backward_time=0.209, grad_norm=94.084, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.680e-05, train_time=1.008 +[gpua006:0/64] 2024-02-13 05:39:44,174 (trainer:756) INFO: 36epoch:train:13001-13100batch: iter_time=6.869e-04, forward_time=0.254, loss_ctc=92.001, loss_interctc_layer6=95.345, loss_interctc_layer12=78.672, loss_interctc_layer15=71.815, loss_interctc_layer21=95.291, loss=86.625, backward_time=0.231, grad_norm=76.635, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=6.680e-05, train_time=1.556 +[gpua006:0/64] 2024-02-13 05:41:48,562 (trainer:756) INFO: 36epoch:train:13101-13200batch: iter_time=8.686e-05, forward_time=0.144, loss_ctc=75.053, loss_interctc_layer6=85.050, loss_interctc_layer12=70.032, loss_interctc_layer15=63.916, loss_interctc_layer21=77.684, loss=74.347, backward_time=0.209, grad_norm=73.069, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.679e-05, train_time=1.244 +[gpua006:0/64] 2024-02-13 05:43:48,381 (trainer:756) INFO: 36epoch:train:13201-13300batch: iter_time=8.869e-05, forward_time=0.142, loss_ctc=63.896, loss_interctc_layer6=63.973, loss_interctc_layer12=52.266, loss_interctc_layer15=47.559, loss_interctc_layer21=66.506, loss=58.840, backward_time=0.209, grad_norm=75.080, clip=100.000, loss_scale=1.308e+31, optim_step_time=0.136, optim0_lr0=6.678e-05, train_time=1.198 +[gpua006:0/64] 2024-02-13 05:46:16,641 (trainer:756) INFO: 36epoch:train:13301-13400batch: iter_time=9.138e-05, forward_time=0.144, loss_ctc=85.882, loss_interctc_layer6=88.890, loss_interctc_layer12=73.309, loss_interctc_layer15=66.931, loss_interctc_layer21=89.215, loss=80.845, backward_time=0.207, grad_norm=78.678, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.678e-05, train_time=1.482 +[gpua006:0/64] 2024-02-13 05:48:29,673 (trainer:756) INFO: 36epoch:train:13401-13500batch: iter_time=2.163e-04, forward_time=0.222, loss_ctc=72.220, loss_interctc_layer6=79.057, loss_interctc_layer12=65.201, loss_interctc_layer15=59.646, loss_interctc_layer21=74.761, loss=70.177, backward_time=0.241, grad_norm=99.001, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.677e-05, train_time=1.329 +[gpua006:0/64] 2024-02-13 05:50:47,028 (trainer:756) INFO: 36epoch:train:13501-13600batch: iter_time=8.744e-05, forward_time=0.156, loss_ctc=94.066, loss_interctc_layer6=86.362, loss_interctc_layer12=70.878, loss_interctc_layer15=64.577, loss_interctc_layer21=97.560, loss=82.689, backward_time=0.214, grad_norm=75.691, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.676e-05, train_time=1.374 +[gpua006:0/64] 2024-02-13 05:52:33,796 (trainer:756) INFO: 36epoch:train:13601-13700batch: iter_time=8.501e-05, forward_time=0.191, loss_ctc=89.391, loss_interctc_layer6=89.573, loss_interctc_layer12=74.502, loss_interctc_layer15=68.422, loss_interctc_layer21=92.655, loss=82.909, backward_time=0.217, grad_norm=104.503, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.676e-05, train_time=1.068 +[gpua006:0/64] 2024-02-13 05:54:10,972 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-13 05:54:30,404 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 05:54:33,854 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 05:54:33,855 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-13 05:54:33,858 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 05:59:34,524 (trainer:756) INFO: 36epoch:train:13701-13800batch: iter_time=2.825, forward_time=0.217, loss_ctc=81.243, loss_interctc_layer6=90.002, loss_interctc_layer12=74.133, loss_interctc_layer15=67.760, loss_interctc_layer21=84.222, loss=79.472, backward_time=0.227, grad_norm=92.519, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=6.675e-05, train_time=4.207 +[gpua006:0/64] 2024-02-13 06:01:01,199 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 06:01:09,765 (trainer:756) INFO: 36epoch:train:13801-13900batch: iter_time=8.545e-05, forward_time=0.146, loss_ctc=77.220, loss_interctc_layer6=83.242, loss_interctc_layer12=69.350, loss_interctc_layer15=63.528, loss_interctc_layer21=79.824, loss=74.633, backward_time=0.210, grad_norm=148.956, clip=100.000, loss_scale=1.936e+31, optim_step_time=0.137, optim0_lr0=6.675e-05, train_time=0.952 +[gpua006:0/64] 2024-02-13 06:02:49,811 (trainer:756) INFO: 36epoch:train:13901-14000batch: iter_time=8.382e-05, forward_time=0.153, loss_ctc=89.595, loss_interctc_layer6=89.732, loss_interctc_layer12=74.277, loss_interctc_layer15=67.842, loss_interctc_layer21=92.986, loss=82.886, backward_time=0.210, grad_norm=105.637, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.674e-05, train_time=1.001 +[gpua006:0/64] 2024-02-13 06:05:23,371 (trainer:756) INFO: 36epoch:train:14001-14100batch: iter_time=3.317e-04, forward_time=0.240, loss_ctc=66.389, loss_interctc_layer6=79.298, loss_interctc_layer12=65.550, loss_interctc_layer15=59.983, loss_interctc_layer21=68.801, loss=68.004, backward_time=0.234, grad_norm=87.834, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.148, optim0_lr0=6.673e-05, train_time=1.535 +[gpua006:0/64] 2024-02-13 06:07:34,440 (trainer:756) INFO: 36epoch:train:14101-14200batch: iter_time=8.598e-05, forward_time=0.143, loss_ctc=71.691, loss_interctc_layer6=83.939, loss_interctc_layer12=69.637, loss_interctc_layer15=63.780, loss_interctc_layer21=74.367, loss=72.683, backward_time=0.207, grad_norm=95.152, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.673e-05, train_time=1.310 +[gpua006:0/64] 2024-02-13 06:09:29,822 (trainer:756) INFO: 36epoch:train:14201-14300batch: iter_time=8.837e-05, forward_time=0.144, loss_ctc=89.883, loss_interctc_layer6=88.303, loss_interctc_layer12=72.470, loss_interctc_layer15=65.664, loss_interctc_layer21=93.387, loss=81.941, backward_time=0.209, grad_norm=94.408, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.672e-05, train_time=1.155 +[gpua006:0/64] 2024-02-13 06:11:52,022 (trainer:756) INFO: 36epoch:train:14301-14400batch: iter_time=8.380e-05, forward_time=0.143, loss_ctc=73.679, loss_interctc_layer6=85.347, loss_interctc_layer12=70.346, loss_interctc_layer15=64.262, loss_interctc_layer21=76.109, loss=73.948, backward_time=0.208, grad_norm=78.086, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.671e-05, train_time=1.422 +[gpua006:0/64] 2024-02-13 06:13:40,844 (trainer:756) INFO: 36epoch:train:14401-14500batch: iter_time=8.384e-05, forward_time=0.144, loss_ctc=73.639, loss_interctc_layer6=75.480, loss_interctc_layer12=61.983, loss_interctc_layer15=56.442, loss_interctc_layer21=76.455, loss=68.800, backward_time=0.211, grad_norm=52.311, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.671e-05, train_time=1.088 +[gpua006:0/64] 2024-02-13 06:16:17,208 (trainer:756) INFO: 36epoch:train:14501-14600batch: iter_time=8.651e-05, forward_time=0.228, loss_ctc=69.527, loss_interctc_layer6=77.272, loss_interctc_layer12=63.390, loss_interctc_layer15=57.789, loss_interctc_layer21=72.292, loss=68.054, backward_time=0.230, grad_norm=72.447, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.670e-05, train_time=1.564 +[gpua006:0/64] 2024-02-13 06:18:52,811 (trainer:756) INFO: 36epoch:train:14601-14700batch: iter_time=8.664e-05, forward_time=0.168, loss_ctc=73.910, loss_interctc_layer6=84.488, loss_interctc_layer12=69.719, loss_interctc_layer15=63.699, loss_interctc_layer21=76.896, loss=73.742, backward_time=0.216, grad_norm=74.877, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.670e-05, train_time=1.555 +[gpua006:0/64] 2024-02-13 06:20:34,469 (trainer:756) INFO: 36epoch:train:14701-14800batch: iter_time=8.568e-05, forward_time=0.143, loss_ctc=79.925, loss_interctc_layer6=82.064, loss_interctc_layer12=67.188, loss_interctc_layer15=61.175, loss_interctc_layer21=82.854, loss=74.641, backward_time=0.209, grad_norm=85.820, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.669e-05, train_time=1.016 +[gpua006:0/64] 2024-02-13 06:22:45,855 (trainer:756) INFO: 36epoch:train:14801-14900batch: iter_time=8.388e-05, forward_time=0.143, loss_ctc=83.659, loss_interctc_layer6=86.700, loss_interctc_layer12=71.765, loss_interctc_layer15=65.815, loss_interctc_layer21=87.000, loss=78.988, backward_time=0.209, grad_norm=76.623, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.668e-05, train_time=1.314 +[gpua006:0/64] 2024-02-13 06:24:59,515 (trainer:756) INFO: 36epoch:train:14901-15000batch: iter_time=8.108e-05, forward_time=0.144, loss_ctc=80.405, loss_interctc_layer6=88.890, loss_interctc_layer12=73.614, loss_interctc_layer15=67.488, loss_interctc_layer21=83.295, loss=78.738, backward_time=0.209, grad_norm=81.042, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.668e-05, train_time=1.336 +[gpua006:0/64] 2024-02-13 06:54:53,185 (trainer:355) INFO: 36epoch results: [train] iter_time=0.213, forward_time=0.158, loss_ctc=80.429, loss_interctc_layer6=84.730, loss_interctc_layer12=70.018, loss_interctc_layer15=64.067, loss_interctc_layer21=83.407, loss=76.530, backward_time=0.213, grad_norm=84.843, clip=100.000, loss_scale=2.078e+31, optim_step_time=0.137, optim0_lr0=6.714e-05, train_time=1.470, time=6 hours, 7 minutes and 59.74 seconds, total_count=540000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=41.569, cer_ctc=0.196, loss_interctc_layer6=47.693, cer_interctc_layer6=0.211, loss_interctc_layer12=34.805, cer_interctc_layer12=0.145, loss_interctc_layer15=30.491, cer_interctc_layer15=0.120, loss_interctc_layer21=44.183, cer_interctc_layer21=0.204, loss=39.748, time=29 minutes and 29.29 seconds, total_count=168156, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-13 06:55:15,297 (trainer:410) INFO: The best model has been updated: valid.total_count +[gpua006:0/64] 2024-02-13 06:55:15,428 (trainer:464) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/31epoch.pth +[gpua006:0/64] 2024-02-13 06:55:15,495 (trainer:289) INFO: 37/45epoch started. Estimated time to finish: 2 days, 14 hours and 17 minutes +[gpua006:0/64] 2024-02-13 06:55:16,425 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-13 06:55:34,201 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 06:55:37,669 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 06:55:37,669 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, +[gpua006:0/64] 2024-02-13 06:55:37,673 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 07:01:04,891 (trainer:756) INFO: 37epoch:train:1-100batch: iter_time=2.345, forward_time=0.177, loss_ctc=83.341, loss_interctc_layer6=83.632, loss_interctc_layer12=69.212, loss_interctc_layer15=63.440, loss_interctc_layer21=86.318, loss=77.189, backward_time=0.214, grad_norm=90.731, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.667e-05, train_time=3.486 +[gpua006:0/64] 2024-02-13 07:02:47,564 (trainer:756) INFO: 37epoch:train:101-200batch: iter_time=9.840e-05, forward_time=0.145, loss_ctc=69.877, loss_interctc_layer6=75.516, loss_interctc_layer12=62.543, loss_interctc_layer15=57.352, loss_interctc_layer21=72.201, loss=67.498, backward_time=0.209, grad_norm=67.415, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.667e-05, train_time=1.027 +[gpua006:0/64] 2024-02-13 07:04:38,171 (trainer:756) INFO: 37epoch:train:201-300batch: iter_time=1.014e-04, forward_time=0.142, loss_ctc=75.078, loss_interctc_layer6=82.763, loss_interctc_layer12=68.966, loss_interctc_layer15=63.724, loss_interctc_layer21=77.610, loss=73.628, backward_time=0.208, grad_norm=90.215, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.666e-05, train_time=1.106 +[gpua006:0/64] 2024-02-13 07:06:45,293 (trainer:756) INFO: 37epoch:train:301-400batch: iter_time=1.018e-04, forward_time=0.143, loss_ctc=79.160, loss_interctc_layer6=85.544, loss_interctc_layer12=71.002, loss_interctc_layer15=64.895, loss_interctc_layer21=82.013, loss=76.523, backward_time=0.208, grad_norm=81.033, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.665e-05, train_time=1.271 +[gpua006:0/64] 2024-02-13 07:08:38,214 (trainer:756) INFO: 37epoch:train:401-500batch: iter_time=1.045e-04, forward_time=0.144, loss_ctc=87.570, loss_interctc_layer6=90.167, loss_interctc_layer12=75.551, loss_interctc_layer15=69.622, loss_interctc_layer21=90.720, loss=82.726, backward_time=0.209, grad_norm=91.281, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.665e-05, train_time=1.129 +[gpua006:0/64] 2024-02-13 07:10:36,505 (trainer:756) INFO: 37epoch:train:501-600batch: iter_time=9.687e-05, forward_time=0.143, loss_ctc=65.504, loss_interctc_layer6=71.858, loss_interctc_layer12=59.801, loss_interctc_layer15=54.769, loss_interctc_layer21=67.778, loss=63.942, backward_time=0.211, grad_norm=204.189, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.664e-05, train_time=1.181 +[gpua006:0/64] 2024-02-13 07:12:31,534 (trainer:756) INFO: 37epoch:train:601-700batch: iter_time=1.033e-04, forward_time=0.147, loss_ctc=97.676, loss_interctc_layer6=95.353, loss_interctc_layer12=79.861, loss_interctc_layer15=73.362, loss_interctc_layer21=101.376, loss=89.526, backward_time=0.210, grad_norm=85.259, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.663e-05, train_time=1.152 +[gpua006:0/64] 2024-02-13 07:14:46,674 (trainer:756) INFO: 37epoch:train:701-800batch: iter_time=1.030e-04, forward_time=0.173, loss_ctc=83.438, loss_interctc_layer6=85.778, loss_interctc_layer12=71.776, loss_interctc_layer15=66.270, loss_interctc_layer21=86.486, loss=78.750, backward_time=0.212, grad_norm=84.030, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.663e-05, train_time=1.351 +[gpua006:0/64] 2024-02-13 07:16:46,270 (trainer:756) INFO: 37epoch:train:801-900batch: iter_time=5.109e-04, forward_time=0.188, loss_ctc=84.709, loss_interctc_layer6=82.071, loss_interctc_layer12=67.827, loss_interctc_layer15=62.070, loss_interctc_layer21=87.964, loss=76.928, backward_time=0.234, grad_norm=71.313, clip=100.000, loss_scale=1.105e+31, optim_step_time=0.140, optim0_lr0=6.662e-05, train_time=1.195 +[gpua006:0/64] 2024-02-13 07:19:17,204 (trainer:756) INFO: 37epoch:train:901-1000batch: iter_time=9.838e-05, forward_time=0.150, loss_ctc=79.384, loss_interctc_layer6=88.951, loss_interctc_layer12=73.864, loss_interctc_layer15=68.016, loss_interctc_layer21=81.992, loss=78.441, backward_time=0.209, grad_norm=94.836, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.662e-05, train_time=1.509 +[gpua006:0/64] 2024-02-13 07:21:26,787 (trainer:756) INFO: 37epoch:train:1001-1100batch: iter_time=9.984e-05, forward_time=0.141, loss_ctc=74.061, loss_interctc_layer6=74.840, loss_interctc_layer12=61.789, loss_interctc_layer15=56.665, loss_interctc_layer21=76.725, loss=68.816, backward_time=0.206, grad_norm=108.502, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.661e-05, train_time=1.296 +[gpua006:0/64] 2024-02-13 07:23:29,988 (trainer:756) INFO: 37epoch:train:1101-1200batch: iter_time=9.982e-05, forward_time=0.144, loss_ctc=81.351, loss_interctc_layer6=81.187, loss_interctc_layer12=67.311, loss_interctc_layer15=61.769, loss_interctc_layer21=84.297, loss=75.183, backward_time=0.211, grad_norm=81.996, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.660e-05, train_time=1.232 +[gpua006:0/64] 2024-02-13 07:24:50,643 (multiple_iter_factory:32) INFO: Building 1th iter-factory... +[gpua006:0/64] 2024-02-13 07:25:09,284 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 07:25:12,702 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 07:25:12,702 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, +[gpua006:0/64] 2024-02-13 07:25:12,709 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 07:30:21,873 (trainer:756) INFO: 37epoch:train:1201-1300batch: iter_time=2.482, forward_time=0.169, loss_ctc=71.725, loss_interctc_layer6=77.216, loss_interctc_layer12=64.310, loss_interctc_layer15=59.290, loss_interctc_layer21=74.131, loss=69.334, backward_time=0.215, grad_norm=68.667, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.660e-05, train_time=4.119 +[gpua006:0/64] 2024-02-13 07:31:59,296 (trainer:756) INFO: 37epoch:train:1301-1400batch: iter_time=8.272e-05, forward_time=0.143, loss_ctc=71.877, loss_interctc_layer6=80.625, loss_interctc_layer12=66.686, loss_interctc_layer15=61.003, loss_interctc_layer21=74.264, loss=70.891, backward_time=0.210, grad_norm=77.283, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.659e-05, train_time=0.974 +[gpua006:0/64] 2024-02-13 07:34:07,814 (trainer:756) INFO: 37epoch:train:1401-1500batch: iter_time=7.863e-05, forward_time=0.278, loss_ctc=77.626, loss_interctc_layer6=80.566, loss_interctc_layer12=66.749, loss_interctc_layer15=61.551, loss_interctc_layer21=80.340, loss=73.366, backward_time=0.229, grad_norm=77.730, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=6.659e-05, train_time=1.283 +[gpua006:0/64] 2024-02-13 07:35:49,335 (trainer:756) INFO: 37epoch:train:1501-1600batch: iter_time=8.020e-05, forward_time=0.145, loss_ctc=68.487, loss_interctc_layer6=82.779, loss_interctc_layer12=68.229, loss_interctc_layer15=62.478, loss_interctc_layer21=70.673, loss=70.529, backward_time=0.209, grad_norm=68.781, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.658e-05, train_time=1.017 +[gpua006:0/64] 2024-02-13 07:37:49,984 (trainer:756) INFO: 37epoch:train:1601-1700batch: iter_time=8.111e-05, forward_time=0.143, loss_ctc=81.632, loss_interctc_layer6=83.500, loss_interctc_layer12=69.521, loss_interctc_layer15=64.167, loss_interctc_layer21=84.743, loss=76.713, backward_time=0.208, grad_norm=69.000, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.657e-05, train_time=1.206 +[gpua006:0/64] 2024-02-13 07:39:53,580 (trainer:756) INFO: 37epoch:train:1701-1800batch: iter_time=9.345e-05, forward_time=0.144, loss_ctc=84.561, loss_interctc_layer6=84.747, loss_interctc_layer12=70.251, loss_interctc_layer15=64.364, loss_interctc_layer21=87.575, loss=78.300, backward_time=0.208, grad_norm=96.648, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.657e-05, train_time=1.236 +[gpua006:0/64] 2024-02-13 07:42:02,351 (trainer:756) INFO: 37epoch:train:1801-1900batch: iter_time=8.761e-05, forward_time=0.200, loss_ctc=86.701, loss_interctc_layer6=82.894, loss_interctc_layer12=69.174, loss_interctc_layer15=63.630, loss_interctc_layer21=90.018, loss=78.483, backward_time=0.227, grad_norm=92.527, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.656e-05, train_time=1.287 +[gpua006:0/64] 2024-02-13 07:44:20,397 (trainer:756) INFO: 37epoch:train:1901-2000batch: iter_time=8.980e-05, forward_time=0.160, loss_ctc=72.044, loss_interctc_layer6=77.279, loss_interctc_layer12=63.815, loss_interctc_layer15=58.348, loss_interctc_layer21=74.631, loss=69.223, backward_time=0.220, grad_norm=59.145, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.655e-05, train_time=1.379 +[gpua006:0/64] 2024-02-13 07:46:20,872 (trainer:756) INFO: 37epoch:train:2001-2100batch: iter_time=8.797e-05, forward_time=0.144, loss_ctc=90.936, loss_interctc_layer6=94.186, loss_interctc_layer12=79.010, loss_interctc_layer15=72.860, loss_interctc_layer21=94.233, loss=86.245, backward_time=0.208, grad_norm=98.770, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.655e-05, train_time=1.206 +[gpua006:0/64] 2024-02-13 07:48:29,263 (trainer:756) INFO: 37epoch:train:2101-2200batch: iter_time=8.133e-05, forward_time=0.145, loss_ctc=77.128, loss_interctc_layer6=80.158, loss_interctc_layer12=65.961, loss_interctc_layer15=60.232, loss_interctc_layer21=80.002, loss=72.696, backward_time=0.207, grad_norm=58.459, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.654e-05, train_time=1.284 +[gpua006:0/64] 2024-02-13 07:50:40,224 (trainer:756) INFO: 37epoch:train:2201-2300batch: iter_time=9.959e-05, forward_time=0.169, loss_ctc=84.207, loss_interctc_layer6=87.293, loss_interctc_layer12=72.316, loss_interctc_layer15=66.396, loss_interctc_layer21=87.031, loss=79.448, backward_time=0.220, grad_norm=78.857, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.654e-05, train_time=1.309 +[gpua006:0/64] 2024-02-13 07:52:51,511 (trainer:756) INFO: 37epoch:train:2301-2400batch: iter_time=8.642e-05, forward_time=0.199, loss_ctc=76.754, loss_interctc_layer6=77.139, loss_interctc_layer12=63.630, loss_interctc_layer15=58.201, loss_interctc_layer21=79.467, loss=71.038, backward_time=0.255, grad_norm=70.786, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.653e-05, train_time=1.313 +[gpua006:0/64] 2024-02-13 07:54:40,259 (trainer:756) INFO: 37epoch:train:2401-2500batch: iter_time=8.922e-05, forward_time=0.144, loss_ctc=73.999, loss_interctc_layer6=77.831, loss_interctc_layer12=65.157, loss_interctc_layer15=60.153, loss_interctc_layer21=76.637, loss=70.755, backward_time=0.209, grad_norm=73.563, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.652e-05, train_time=1.087 +[gpua006:0/64] 2024-02-13 07:55:00,389 (multiple_iter_factory:32) INFO: Building 2th iter-factory... +[gpua006:0/64] 2024-02-13 07:55:19,039 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 07:55:22,538 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 07:55:22,538 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, +[gpua006:0/64] 2024-02-13 07:55:22,545 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 08:01:03,612 (trainer:756) INFO: 37epoch:train:2501-2600batch: iter_time=2.753, forward_time=0.144, loss_ctc=82.160, loss_interctc_layer6=82.405, loss_interctc_layer12=68.060, loss_interctc_layer15=62.373, loss_interctc_layer21=85.157, loss=76.031, backward_time=0.209, grad_norm=77.846, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.652e-05, train_time=3.834 +[gpua006:0/64] 2024-02-13 08:02:42,829 (trainer:756) INFO: 37epoch:train:2601-2700batch: iter_time=8.991e-05, forward_time=0.143, loss_ctc=69.239, loss_interctc_layer6=74.935, loss_interctc_layer12=61.902, loss_interctc_layer15=56.680, loss_interctc_layer21=71.556, loss=66.862, backward_time=0.209, grad_norm=73.131, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.651e-05, train_time=0.992 +[gpua006:0/64] 2024-02-13 08:03:43,751 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 08:04:36,092 (trainer:756) INFO: 37epoch:train:2701-2800batch: iter_time=9.039e-05, forward_time=0.143, loss_ctc=73.588, loss_interctc_layer6=80.655, loss_interctc_layer12=66.761, loss_interctc_layer15=61.404, loss_interctc_layer21=76.212, loss=71.724, backward_time=0.208, grad_norm=90.463, clip=100.000, loss_scale=1.567e+31, optim_step_time=0.136, optim0_lr0=6.651e-05, train_time=1.132 +[gpua006:0/64] 2024-02-13 08:06:31,726 (trainer:756) INFO: 37epoch:train:2801-2900batch: iter_time=9.770e-05, forward_time=0.146, loss_ctc=78.551, loss_interctc_layer6=85.592, loss_interctc_layer12=70.744, loss_interctc_layer15=64.973, loss_interctc_layer21=81.389, loss=76.250, backward_time=0.208, grad_norm=74.961, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.650e-05, train_time=1.156 +[gpua006:0/64] 2024-02-13 08:08:29,577 (trainer:756) INFO: 37epoch:train:2901-3000batch: iter_time=8.792e-05, forward_time=0.162, loss_ctc=87.413, loss_interctc_layer6=88.747, loss_interctc_layer12=73.672, loss_interctc_layer15=67.694, loss_interctc_layer21=90.498, loss=81.605, backward_time=0.208, grad_norm=101.159, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.649e-05, train_time=1.178 +[gpua006:0/64] 2024-02-13 08:10:52,409 (trainer:756) INFO: 37epoch:train:3001-3100batch: iter_time=8.668e-05, forward_time=0.242, loss_ctc=65.158, loss_interctc_layer6=71.489, loss_interctc_layer12=59.142, loss_interctc_layer15=54.221, loss_interctc_layer21=67.461, loss=63.494, backward_time=0.239, grad_norm=66.554, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=6.649e-05, train_time=1.428 +[gpua006:0/64] 2024-02-13 08:13:39,872 (trainer:756) INFO: 37epoch:train:3101-3200batch: iter_time=8.760e-05, forward_time=0.145, loss_ctc=95.532, loss_interctc_layer6=94.138, loss_interctc_layer12=78.563, loss_interctc_layer15=72.168, loss_interctc_layer21=99.238, loss=87.928, backward_time=0.208, grad_norm=86.997, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.648e-05, train_time=1.674 +[gpua006:0/64] 2024-02-13 08:15:32,761 (trainer:756) INFO: 37epoch:train:3201-3300batch: iter_time=8.786e-05, forward_time=0.143, loss_ctc=82.362, loss_interctc_layer6=85.006, loss_interctc_layer12=71.090, loss_interctc_layer15=64.798, loss_interctc_layer21=85.650, loss=77.781, backward_time=0.208, grad_norm=78.352, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.647e-05, train_time=1.130 +[gpua006:0/64] 2024-02-13 08:17:52,595 (trainer:756) INFO: 37epoch:train:3301-3400batch: iter_time=9.089e-05, forward_time=0.143, loss_ctc=83.515, loss_interctc_layer6=81.397, loss_interctc_layer12=67.129, loss_interctc_layer15=61.312, loss_interctc_layer21=86.732, loss=76.017, backward_time=0.206, grad_norm=88.321, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.647e-05, train_time=1.398 +[gpua006:0/64] 2024-02-13 08:19:36,389 (trainer:756) INFO: 37epoch:train:3401-3500batch: iter_time=8.799e-05, forward_time=0.143, loss_ctc=78.687, loss_interctc_layer6=88.144, loss_interctc_layer12=72.980, loss_interctc_layer15=66.967, loss_interctc_layer21=81.261, loss=77.608, backward_time=0.209, grad_norm=80.071, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.646e-05, train_time=1.038 +[gpua006:0/64] 2024-02-13 08:21:23,121 (trainer:756) INFO: 37epoch:train:3501-3600batch: iter_time=8.324e-05, forward_time=0.142, loss_ctc=73.857, loss_interctc_layer6=74.035, loss_interctc_layer12=61.060, loss_interctc_layer15=55.829, loss_interctc_layer21=76.498, loss=68.256, backward_time=0.209, grad_norm=67.631, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.646e-05, train_time=1.067 +[gpua006:0/64] 2024-02-13 08:23:11,574 (trainer:756) INFO: 37epoch:train:3601-3700batch: iter_time=8.092e-05, forward_time=0.144, loss_ctc=79.847, loss_interctc_layer6=80.518, loss_interctc_layer12=66.473, loss_interctc_layer15=60.827, loss_interctc_layer21=82.742, loss=74.081, backward_time=0.208, grad_norm=96.186, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.645e-05, train_time=1.084 +[gpua006:0/64] 2024-02-13 08:24:27,550 (multiple_iter_factory:32) INFO: Building 3th iter-factory... +[gpua006:0/64] 2024-02-13 08:24:46,149 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 08:24:49,577 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 08:24:49,577 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, +[gpua006:0/64] 2024-02-13 08:24:49,581 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 08:30:03,518 (trainer:756) INFO: 37epoch:train:3701-3800batch: iter_time=2.954, forward_time=0.167, loss_ctc=71.947, loss_interctc_layer6=76.617, loss_interctc_layer12=63.877, loss_interctc_layer15=58.935, loss_interctc_layer21=74.467, loss=69.168, backward_time=0.212, grad_norm=199.634, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.644e-05, train_time=4.119 +[gpua006:0/64] 2024-02-13 08:31:40,566 (trainer:756) INFO: 37epoch:train:3801-3900batch: iter_time=8.113e-05, forward_time=0.143, loss_ctc=70.944, loss_interctc_layer6=80.018, loss_interctc_layer12=66.060, loss_interctc_layer15=60.340, loss_interctc_layer21=73.396, loss=70.152, backward_time=0.209, grad_norm=75.744, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.644e-05, train_time=0.970 +[gpua006:0/64] 2024-02-13 08:33:30,537 (trainer:756) INFO: 37epoch:train:3901-4000batch: iter_time=8.116e-05, forward_time=0.168, loss_ctc=76.400, loss_interctc_layer6=79.287, loss_interctc_layer12=65.788, loss_interctc_layer15=60.564, loss_interctc_layer21=78.700, loss=72.148, backward_time=0.212, grad_norm=81.791, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=6.643e-05, train_time=1.099 +[gpua006:0/64] 2024-02-13 08:35:46,975 (trainer:756) INFO: 37epoch:train:4001-4100batch: iter_time=1.592e-04, forward_time=0.201, loss_ctc=68.912, loss_interctc_layer6=82.720, loss_interctc_layer12=68.285, loss_interctc_layer15=62.739, loss_interctc_layer21=71.069, loss=70.745, backward_time=0.237, grad_norm=65.465, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.643e-05, train_time=1.364 +[gpua006:0/64] 2024-02-13 08:37:35,165 (trainer:756) INFO: 37epoch:train:4101-4200batch: iter_time=8.172e-05, forward_time=0.143, loss_ctc=81.961, loss_interctc_layer6=83.079, loss_interctc_layer12=69.127, loss_interctc_layer15=63.626, loss_interctc_layer21=85.128, loss=76.584, backward_time=0.208, grad_norm=73.853, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.642e-05, train_time=1.082 +[gpua006:0/64] 2024-02-13 08:39:42,184 (trainer:756) INFO: 37epoch:train:4201-4300batch: iter_time=8.366e-05, forward_time=0.144, loss_ctc=84.505, loss_interctc_layer6=84.662, loss_interctc_layer12=69.995, loss_interctc_layer15=64.122, loss_interctc_layer21=87.628, loss=78.182, backward_time=0.208, grad_norm=82.230, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.641e-05, train_time=1.270 +[gpua006:0/64] 2024-02-13 08:41:40,784 (trainer:756) INFO: 37epoch:train:4301-4400batch: iter_time=8.691e-05, forward_time=0.143, loss_ctc=87.194, loss_interctc_layer6=82.685, loss_interctc_layer12=68.813, loss_interctc_layer15=63.256, loss_interctc_layer21=90.712, loss=78.532, backward_time=0.207, grad_norm=77.673, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.641e-05, train_time=1.186 +[gpua006:0/64] 2024-02-13 08:43:51,825 (trainer:756) INFO: 37epoch:train:4401-4500batch: iter_time=3.785e-04, forward_time=0.177, loss_ctc=71.478, loss_interctc_layer6=77.532, loss_interctc_layer12=63.815, loss_interctc_layer15=58.345, loss_interctc_layer21=74.008, loss=69.036, backward_time=0.263, grad_norm=73.981, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.143, optim0_lr0=6.640e-05, train_time=1.309 +[gpua006:0/64] 2024-02-13 08:46:04,861 (trainer:756) INFO: 37epoch:train:4501-4600batch: iter_time=8.216e-05, forward_time=0.172, loss_ctc=90.930, loss_interctc_layer6=93.622, loss_interctc_layer12=78.278, loss_interctc_layer15=72.035, loss_interctc_layer21=94.074, loss=85.788, backward_time=0.222, grad_norm=94.790, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.640e-05, train_time=1.331 +[gpua006:0/64] 2024-02-13 08:48:01,980 (trainer:756) INFO: 37epoch:train:4601-4700batch: iter_time=7.962e-05, forward_time=0.143, loss_ctc=77.355, loss_interctc_layer6=79.963, loss_interctc_layer12=65.784, loss_interctc_layer15=59.958, loss_interctc_layer21=80.320, loss=72.676, backward_time=0.208, grad_norm=80.917, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.639e-05, train_time=1.171 +[gpua006:0/64] 2024-02-13 08:50:16,249 (trainer:756) INFO: 37epoch:train:4701-4800batch: iter_time=8.617e-05, forward_time=0.143, loss_ctc=82.636, loss_interctc_layer6=86.473, loss_interctc_layer12=71.556, loss_interctc_layer15=65.809, loss_interctc_layer21=85.382, loss=78.371, backward_time=0.208, grad_norm=97.984, clip=100.000, loss_scale=1.470e+31, optim_step_time=0.136, optim0_lr0=6.638e-05, train_time=1.343 +[gpua006:0/64] 2024-02-13 08:52:38,802 (trainer:756) INFO: 37epoch:train:4801-4900batch: iter_time=8.520e-05, forward_time=0.142, loss_ctc=75.910, loss_interctc_layer6=76.694, loss_interctc_layer12=63.300, loss_interctc_layer15=58.160, loss_interctc_layer21=78.546, loss=70.522, backward_time=0.208, grad_norm=77.543, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.638e-05, train_time=1.425 +[gpua006:0/64] 2024-02-13 08:54:55,576 (trainer:756) INFO: 37epoch:train:4901-5000batch: iter_time=9.859e-05, forward_time=0.190, loss_ctc=71.690, loss_interctc_layer6=77.069, loss_interctc_layer12=64.296, loss_interctc_layer15=59.287, loss_interctc_layer21=74.209, loss=69.310, backward_time=0.244, grad_norm=67.554, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.637e-05, train_time=1.366 +[gpua006:0/64] 2024-02-13 08:55:15,890 (multiple_iter_factory:32) INFO: Building 4th iter-factory... +[gpua006:0/64] 2024-02-13 08:55:34,373 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 08:55:37,861 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 08:55:37,861 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, +[gpua006:0/64] 2024-02-13 08:55:37,864 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 09:01:44,778 (trainer:756) INFO: 37epoch:train:5001-5100batch: iter_time=3.010, forward_time=0.161, loss_ctc=81.186, loss_interctc_layer6=82.825, loss_interctc_layer12=68.530, loss_interctc_layer15=62.769, loss_interctc_layer21=84.008, loss=75.864, backward_time=0.211, grad_norm=79.778, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.637e-05, train_time=4.093 +[gpua006:0/64] 2024-02-13 09:03:13,856 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 09:03:19,521 (trainer:756) INFO: 37epoch:train:5101-5200batch: iter_time=7.952e-05, forward_time=0.142, loss_ctc=69.144, loss_interctc_layer6=75.253, loss_interctc_layer12=62.180, loss_interctc_layer15=56.835, loss_interctc_layer21=71.595, loss=67.002, backward_time=0.209, grad_norm=69.164, clip=100.000, loss_scale=1.967e+31, optim_step_time=0.136, optim0_lr0=6.636e-05, train_time=0.947 +[gpua006:0/64] 2024-02-13 09:05:09,951 (trainer:756) INFO: 37epoch:train:5201-5300batch: iter_time=7.499e-05, forward_time=0.142, loss_ctc=73.895, loss_interctc_layer6=80.673, loss_interctc_layer12=66.735, loss_interctc_layer15=61.428, loss_interctc_layer21=76.257, loss=71.798, backward_time=0.209, grad_norm=81.936, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.635e-05, train_time=1.104 +[gpua006:0/64] 2024-02-13 09:07:16,202 (trainer:756) INFO: 37epoch:train:5301-5400batch: iter_time=8.310e-05, forward_time=0.143, loss_ctc=78.150, loss_interctc_layer6=85.397, loss_interctc_layer12=70.420, loss_interctc_layer15=64.383, loss_interctc_layer21=80.902, loss=75.850, backward_time=0.208, grad_norm=76.785, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.635e-05, train_time=1.262 +[gpua006:0/64] 2024-02-13 09:09:48,385 (trainer:756) INFO: 37epoch:train:5401-5500batch: iter_time=8.250e-05, forward_time=0.212, loss_ctc=86.759, loss_interctc_layer6=87.945, loss_interctc_layer12=73.148, loss_interctc_layer15=67.110, loss_interctc_layer21=89.967, loss=80.986, backward_time=0.239, grad_norm=123.424, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=6.634e-05, train_time=1.519 +[gpua006:0/64] 2024-02-13 09:11:44,760 (trainer:756) INFO: 37epoch:train:5501-5600batch: iter_time=8.199e-05, forward_time=0.146, loss_ctc=64.299, loss_interctc_layer6=71.312, loss_interctc_layer12=58.973, loss_interctc_layer15=53.881, loss_interctc_layer21=66.594, loss=63.012, backward_time=0.217, grad_norm=64.509, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.633e-05, train_time=1.166 +[gpua006:0/64] 2024-02-13 09:14:04,903 (trainer:756) INFO: 37epoch:train:5601-5700batch: iter_time=8.078e-05, forward_time=0.146, loss_ctc=95.875, loss_interctc_layer6=93.791, loss_interctc_layer12=78.041, loss_interctc_layer15=71.722, loss_interctc_layer21=99.500, loss=87.786, backward_time=0.209, grad_norm=88.952, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.633e-05, train_time=1.401 +[gpua006:0/64] 2024-02-13 09:16:15,955 (trainer:756) INFO: 37epoch:train:5701-5800batch: iter_time=8.120e-05, forward_time=0.142, loss_ctc=81.382, loss_interctc_layer6=84.070, loss_interctc_layer12=70.116, loss_interctc_layer15=64.148, loss_interctc_layer21=84.450, loss=76.833, backward_time=0.208, grad_norm=110.858, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.632e-05, train_time=1.310 +[gpua006:0/64] 2024-02-13 09:18:59,873 (trainer:756) INFO: 37epoch:train:5801-5900batch: iter_time=8.238e-05, forward_time=0.143, loss_ctc=83.096, loss_interctc_layer6=81.214, loss_interctc_layer12=66.888, loss_interctc_layer15=60.994, loss_interctc_layer21=86.324, loss=75.703, backward_time=0.207, grad_norm=71.432, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.632e-05, train_time=1.639 +[gpua006:0/64] 2024-02-13 09:21:01,614 (trainer:756) INFO: 37epoch:train:5901-6000batch: iter_time=8.877e-05, forward_time=0.143, loss_ctc=78.147, loss_interctc_layer6=87.379, loss_interctc_layer12=72.283, loss_interctc_layer15=66.312, loss_interctc_layer21=81.007, loss=77.026, backward_time=0.209, grad_norm=87.469, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.631e-05, train_time=1.217 +[gpua006:0/64] 2024-02-13 09:23:02,375 (trainer:756) INFO: 37epoch:train:6001-6100batch: iter_time=1.014e-04, forward_time=0.143, loss_ctc=73.201, loss_interctc_layer6=73.963, loss_interctc_layer12=60.896, loss_interctc_layer15=55.748, loss_interctc_layer21=75.901, loss=67.942, backward_time=0.209, grad_norm=82.271, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.630e-05, train_time=1.207 +[gpua006:0/64] 2024-02-13 09:25:12,337 (trainer:756) INFO: 37epoch:train:6101-6200batch: iter_time=8.814e-05, forward_time=0.234, loss_ctc=79.905, loss_interctc_layer6=79.862, loss_interctc_layer12=65.758, loss_interctc_layer15=60.038, loss_interctc_layer21=82.751, loss=73.663, backward_time=0.248, grad_norm=88.408, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=6.630e-05, train_time=1.298 +[gpua006:0/64] 2024-02-13 09:26:22,144 (multiple_iter_factory:32) INFO: Building 5th iter-factory... +[gpua006:0/64] 2024-02-13 09:26:41,091 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 09:26:44,665 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 09:26:44,665 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, +[gpua006:0/64] 2024-02-13 09:26:44,693 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 09:31:40,926 (trainer:756) INFO: 37epoch:train:6201-6300batch: iter_time=2.693, forward_time=0.160, loss_ctc=71.856, loss_interctc_layer6=76.357, loss_interctc_layer12=63.714, loss_interctc_layer15=58.751, loss_interctc_layer21=74.495, loss=69.035, backward_time=0.212, grad_norm=90.577, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.629e-05, train_time=3.887 +[gpua006:0/64] 2024-02-13 09:33:17,369 (trainer:756) INFO: 37epoch:train:6301-6400batch: iter_time=7.846e-05, forward_time=0.143, loss_ctc=67.968, loss_interctc_layer6=80.049, loss_interctc_layer12=65.985, loss_interctc_layer15=60.278, loss_interctc_layer21=70.619, loss=68.980, backward_time=0.210, grad_norm=72.064, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.629e-05, train_time=0.964 +[gpua006:0/64] 2024-02-13 09:35:26,338 (trainer:756) INFO: 37epoch:train:6401-6500batch: iter_time=8.368e-05, forward_time=0.144, loss_ctc=71.511, loss_interctc_layer6=78.879, loss_interctc_layer12=65.404, loss_interctc_layer15=60.234, loss_interctc_layer21=74.454, loss=70.096, backward_time=0.209, grad_norm=89.188, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.628e-05, train_time=1.289 +[gpua006:0/64] 2024-02-13 09:37:51,515 (trainer:756) INFO: 37epoch:train:6501-6600batch: iter_time=8.962e-05, forward_time=0.143, loss_ctc=67.117, loss_interctc_layer6=83.043, loss_interctc_layer12=68.448, loss_interctc_layer15=62.560, loss_interctc_layer21=69.452, loss=70.124, backward_time=0.208, grad_norm=70.172, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.627e-05, train_time=1.452 +[gpua006:0/64] 2024-02-13 09:40:26,485 (trainer:756) INFO: 37epoch:train:6601-6700batch: iter_time=8.723e-05, forward_time=0.145, loss_ctc=77.948, loss_interctc_layer6=82.345, loss_interctc_layer12=68.421, loss_interctc_layer15=62.749, loss_interctc_layer21=80.854, loss=74.463, backward_time=0.208, grad_norm=123.216, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.627e-05, train_time=1.549 +[gpua006:0/64] 2024-02-13 09:42:28,664 (trainer:756) INFO: 37epoch:train:6701-6800batch: iter_time=8.398e-05, forward_time=0.144, loss_ctc=79.454, loss_interctc_layer6=84.525, loss_interctc_layer12=69.875, loss_interctc_layer15=63.872, loss_interctc_layer21=82.284, loss=76.002, backward_time=0.209, grad_norm=99.965, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.626e-05, train_time=1.221 +[gpua006:0/64] 2024-02-13 09:45:05,159 (trainer:756) INFO: 37epoch:train:6801-6900batch: iter_time=8.973e-05, forward_time=0.143, loss_ctc=82.667, loss_interctc_layer6=81.979, loss_interctc_layer12=68.194, loss_interctc_layer15=62.640, loss_interctc_layer21=86.011, loss=76.298, backward_time=0.208, grad_norm=76.149, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.626e-05, train_time=1.565 +[gpua006:0/64] 2024-02-13 09:47:12,105 (trainer:756) INFO: 37epoch:train:6901-7000batch: iter_time=2.646e-04, forward_time=0.194, loss_ctc=66.849, loss_interctc_layer6=76.787, loss_interctc_layer12=63.310, loss_interctc_layer15=57.713, loss_interctc_layer21=69.320, loss=66.796, backward_time=0.272, grad_norm=66.930, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.142, optim0_lr0=6.625e-05, train_time=1.269 +[gpua006:0/64] 2024-02-13 09:49:40,085 (trainer:756) INFO: 37epoch:train:7001-7100batch: iter_time=8.945e-05, forward_time=0.175, loss_ctc=88.388, loss_interctc_layer6=92.669, loss_interctc_layer12=77.181, loss_interctc_layer15=71.000, loss_interctc_layer21=91.449, loss=84.137, backward_time=0.214, grad_norm=93.636, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.624e-05, train_time=1.479 +[gpua006:0/64] 2024-02-13 09:51:41,767 (trainer:756) INFO: 37epoch:train:7101-7200batch: iter_time=8.782e-05, forward_time=0.143, loss_ctc=73.489, loss_interctc_layer6=79.149, loss_interctc_layer12=64.978, loss_interctc_layer15=59.164, loss_interctc_layer21=76.366, loss=70.629, backward_time=0.209, grad_norm=70.041, clip=100.000, loss_scale=1.075e+31, optim_step_time=0.136, optim0_lr0=6.624e-05, train_time=1.218 +[gpua006:0/64] 2024-02-13 09:53:36,600 (trainer:756) INFO: 37epoch:train:7201-7300batch: iter_time=8.751e-05, forward_time=0.142, loss_ctc=80.709, loss_interctc_layer6=85.695, loss_interctc_layer12=70.965, loss_interctc_layer15=65.116, loss_interctc_layer21=83.595, loss=77.216, backward_time=0.209, grad_norm=81.529, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.623e-05, train_time=1.148 +[gpua006:0/64] 2024-02-13 09:55:58,330 (trainer:756) INFO: 37epoch:train:7301-7400batch: iter_time=8.240e-05, forward_time=0.143, loss_ctc=70.753, loss_interctc_layer6=76.273, loss_interctc_layer12=62.675, loss_interctc_layer15=57.294, loss_interctc_layer21=73.157, loss=68.030, backward_time=0.209, grad_norm=67.753, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.623e-05, train_time=1.417 +[gpua006:0/64] 2024-02-13 09:58:15,088 (trainer:756) INFO: 37epoch:train:7401-7500batch: iter_time=8.858e-05, forward_time=0.142, loss_ctc=69.047, loss_interctc_layer6=76.725, loss_interctc_layer12=64.217, loss_interctc_layer15=59.101, loss_interctc_layer21=71.539, loss=68.126, backward_time=0.209, grad_norm=70.611, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.622e-05, train_time=1.367 +[gpua006:0/64] 2024-02-13 09:58:35,118 (multiple_iter_factory:32) INFO: Building 6th iter-factory... +[gpua006:0/64] 2024-02-13 09:58:54,163 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 09:58:57,631 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 09:58:57,631 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, +[gpua006:0/64] 2024-02-13 09:58:57,636 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 10:05:15,859 (trainer:756) INFO: 37epoch:train:7501-7600batch: iter_time=3.168, forward_time=0.172, loss_ctc=78.180, loss_interctc_layer6=81.903, loss_interctc_layer12=67.549, loss_interctc_layer15=61.724, loss_interctc_layer21=81.013, loss=74.074, backward_time=0.215, grad_norm=69.013, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=6.621e-05, train_time=4.207 +[gpua006:0/64] 2024-02-13 10:06:53,547 (trainer:756) INFO: 37epoch:train:7601-7700batch: iter_time=8.300e-05, forward_time=0.164, loss_ctc=64.847, loss_interctc_layer6=75.010, loss_interctc_layer12=61.853, loss_interctc_layer15=56.458, loss_interctc_layer21=67.166, loss=65.067, backward_time=0.214, grad_norm=76.773, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.621e-05, train_time=0.976 +[gpua006:0/64] 2024-02-13 10:08:54,527 (trainer:756) INFO: 37epoch:train:7701-7800batch: iter_time=8.637e-05, forward_time=0.144, loss_ctc=70.854, loss_interctc_layer6=80.502, loss_interctc_layer12=66.993, loss_interctc_layer15=61.740, loss_interctc_layer21=73.777, loss=70.773, backward_time=0.208, grad_norm=86.991, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.620e-05, train_time=1.210 +[gpua006:0/64] 2024-02-13 10:11:18,590 (trainer:756) INFO: 37epoch:train:7801-7900batch: iter_time=9.744e-05, forward_time=0.143, loss_ctc=75.508, loss_interctc_layer6=84.658, loss_interctc_layer12=69.745, loss_interctc_layer15=63.891, loss_interctc_layer21=78.062, loss=74.373, backward_time=0.207, grad_norm=66.179, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.620e-05, train_time=1.438 +[gpua006:0/64] 2024-02-13 10:13:43,739 (trainer:756) INFO: 37epoch:train:7901-8000batch: iter_time=9.320e-05, forward_time=0.144, loss_ctc=83.054, loss_interctc_layer6=88.104, loss_interctc_layer12=73.391, loss_interctc_layer15=67.355, loss_interctc_layer21=86.184, loss=79.618, backward_time=0.209, grad_norm=70.948, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.619e-05, train_time=1.453 +[gpua006:0/64] 2024-02-13 10:15:32,119 (trainer:756) INFO: 37epoch:train:8001-8100batch: iter_time=9.673e-05, forward_time=0.142, loss_ctc=61.735, loss_interctc_layer6=71.127, loss_interctc_layer12=58.867, loss_interctc_layer15=53.861, loss_interctc_layer21=63.972, loss=61.912, backward_time=0.208, grad_norm=69.714, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.618e-05, train_time=1.084 +[gpua006:0/64] 2024-02-13 10:17:41,996 (trainer:756) INFO: 37epoch:train:8101-8200batch: iter_time=9.779e-05, forward_time=0.145, loss_ctc=90.188, loss_interctc_layer6=93.675, loss_interctc_layer12=77.884, loss_interctc_layer15=71.676, loss_interctc_layer21=93.724, loss=85.429, backward_time=0.208, grad_norm=74.923, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.618e-05, train_time=1.299 +[gpua006:0/64] 2024-02-13 10:19:41,842 (trainer:756) INFO: 37epoch:train:8201-8300batch: iter_time=9.717e-05, forward_time=0.142, loss_ctc=76.750, loss_interctc_layer6=83.829, loss_interctc_layer12=69.411, loss_interctc_layer15=63.761, loss_interctc_layer21=79.514, loss=74.653, backward_time=0.208, grad_norm=90.433, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.617e-05, train_time=1.198 +[gpua006:0/64] 2024-02-13 10:19:47,443 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 10:22:18,053 (trainer:756) INFO: 37epoch:train:8301-8400batch: iter_time=9.132e-05, forward_time=0.160, loss_ctc=81.046, loss_interctc_layer6=80.467, loss_interctc_layer12=66.193, loss_interctc_layer15=60.344, loss_interctc_layer21=84.295, loss=74.469, backward_time=0.212, grad_norm=70.806, clip=100.000, loss_scale=1.045e+31, optim_step_time=0.138, optim0_lr0=6.617e-05, train_time=1.562 +[gpua006:0/64] 2024-02-13 10:24:43,404 (trainer:756) INFO: 37epoch:train:8401-8500batch: iter_time=9.006e-05, forward_time=0.189, loss_ctc=73.627, loss_interctc_layer6=86.577, loss_interctc_layer12=71.551, loss_interctc_layer15=65.514, loss_interctc_layer21=76.175, loss=74.689, backward_time=0.231, grad_norm=80.451, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.616e-05, train_time=1.453 +[gpua006:0/64] 2024-02-13 10:26:32,529 (trainer:756) INFO: 37epoch:train:8501-8600batch: iter_time=9.113e-05, forward_time=0.154, loss_ctc=71.251, loss_interctc_layer6=72.963, loss_interctc_layer12=60.009, loss_interctc_layer15=54.912, loss_interctc_layer21=73.933, loss=66.614, backward_time=0.214, grad_norm=64.462, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.615e-05, train_time=1.090 +[gpua006:0/64] 2024-02-13 10:28:34,187 (trainer:756) INFO: 37epoch:train:8601-8700batch: iter_time=9.296e-05, forward_time=0.160, loss_ctc=73.069, loss_interctc_layer6=79.685, loss_interctc_layer12=65.727, loss_interctc_layer15=60.036, loss_interctc_layer21=75.923, loss=70.888, backward_time=0.212, grad_norm=61.298, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.615e-05, train_time=1.217 +[gpua006:0/64] 2024-02-13 10:29:53,976 (multiple_iter_factory:32) INFO: Building 7th iter-factory... +[gpua006:0/64] 2024-02-13 10:30:12,997 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 10:30:16,462 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 10:30:16,462 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, +[gpua006:0/64] 2024-02-13 10:30:16,465 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 10:35:39,077 (trainer:756) INFO: 37epoch:train:8701-8800batch: iter_time=3.055, forward_time=0.143, loss_ctc=69.477, loss_interctc_layer6=75.923, loss_interctc_layer12=63.305, loss_interctc_layer15=58.173, loss_interctc_layer21=71.918, loss=67.759, backward_time=0.209, grad_norm=69.101, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.614e-05, train_time=4.248 +[gpua006:0/64] 2024-02-13 10:37:17,278 (trainer:756) INFO: 37epoch:train:8801-8900batch: iter_time=9.444e-05, forward_time=0.142, loss_ctc=67.060, loss_interctc_layer6=79.919, loss_interctc_layer12=65.976, loss_interctc_layer15=60.293, loss_interctc_layer21=69.389, loss=68.527, backward_time=0.209, grad_norm=64.648, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.613e-05, train_time=0.982 +[gpua006:0/64] 2024-02-13 10:39:10,417 (trainer:756) INFO: 37epoch:train:8901-9000batch: iter_time=8.676e-05, forward_time=0.142, loss_ctc=71.472, loss_interctc_layer6=79.626, loss_interctc_layer12=66.169, loss_interctc_layer15=60.819, loss_interctc_layer21=74.024, loss=70.422, backward_time=0.208, grad_norm=76.202, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.613e-05, train_time=1.131 +[gpua006:0/64] 2024-02-13 10:40:46,406 (trainer:756) INFO: 37epoch:train:9001-9100batch: iter_time=1.038e-04, forward_time=0.142, loss_ctc=66.311, loss_interctc_layer6=82.737, loss_interctc_layer12=68.246, loss_interctc_layer15=62.462, loss_interctc_layer21=68.569, loss=69.665, backward_time=0.209, grad_norm=78.674, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.612e-05, train_time=0.960 +[gpua006:0/64] 2024-02-13 10:42:44,403 (trainer:756) INFO: 37epoch:train:9101-9200batch: iter_time=9.752e-05, forward_time=0.143, loss_ctc=77.170, loss_interctc_layer6=81.563, loss_interctc_layer12=67.662, loss_interctc_layer15=62.092, loss_interctc_layer21=80.154, loss=73.728, backward_time=0.208, grad_norm=84.507, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.612e-05, train_time=1.180 +[gpua006:0/64] 2024-02-13 10:45:14,498 (trainer:756) INFO: 37epoch:train:9201-9300batch: iter_time=8.689e-05, forward_time=0.161, loss_ctc=78.649, loss_interctc_layer6=83.630, loss_interctc_layer12=69.083, loss_interctc_layer15=63.229, loss_interctc_layer21=81.619, loss=75.242, backward_time=0.208, grad_norm=186.921, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.611e-05, train_time=1.501 +[gpua006:0/64] 2024-02-13 10:47:22,946 (trainer:756) INFO: 37epoch:train:9301-9400batch: iter_time=8.770e-05, forward_time=0.173, loss_ctc=81.230, loss_interctc_layer6=81.808, loss_interctc_layer12=68.089, loss_interctc_layer15=62.587, loss_interctc_layer21=84.532, loss=75.649, backward_time=0.227, grad_norm=69.386, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=6.610e-05, train_time=1.284 +[gpua006:0/64] 2024-02-13 10:49:45,082 (trainer:756) INFO: 37epoch:train:9401-9500batch: iter_time=8.786e-05, forward_time=0.169, loss_ctc=65.387, loss_interctc_layer6=76.352, loss_interctc_layer12=62.779, loss_interctc_layer15=57.144, loss_interctc_layer21=67.821, loss=65.897, backward_time=0.215, grad_norm=71.145, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=6.610e-05, train_time=1.421 +[gpua006:0/64] 2024-02-13 10:52:25,592 (trainer:756) INFO: 37epoch:train:9501-9600batch: iter_time=8.604e-05, forward_time=0.170, loss_ctc=86.716, loss_interctc_layer6=92.603, loss_interctc_layer12=77.154, loss_interctc_layer15=70.882, loss_interctc_layer21=89.792, loss=83.430, backward_time=0.289, grad_norm=173.950, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.609e-05, train_time=1.605 +[gpua006:0/64] 2024-02-13 10:55:03,934 (trainer:756) INFO: 37epoch:train:9601-9700batch: iter_time=8.474e-05, forward_time=0.155, loss_ctc=73.616, loss_interctc_layer6=78.807, loss_interctc_layer12=64.603, loss_interctc_layer15=58.840, loss_interctc_layer21=76.542, loss=70.482, backward_time=0.211, grad_norm=64.076, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.609e-05, train_time=1.583 +[gpua006:0/64] 2024-02-13 10:57:21,475 (trainer:756) INFO: 37epoch:train:9701-9800batch: iter_time=8.586e-05, forward_time=0.143, loss_ctc=80.037, loss_interctc_layer6=85.365, loss_interctc_layer12=70.582, loss_interctc_layer15=64.585, loss_interctc_layer21=82.810, loss=76.676, backward_time=0.209, grad_norm=80.820, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.608e-05, train_time=1.374 +[gpua006:0/64] 2024-02-13 10:59:20,373 (trainer:756) INFO: 37epoch:train:9801-9900batch: iter_time=8.211e-05, forward_time=0.142, loss_ctc=69.597, loss_interctc_layer6=76.144, loss_interctc_layer12=62.661, loss_interctc_layer15=57.295, loss_interctc_layer21=72.227, loss=67.585, backward_time=0.209, grad_norm=64.887, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.607e-05, train_time=1.190 +[gpua006:0/64] 2024-02-13 11:00:59,722 (trainer:756) INFO: 37epoch:train:9901-10000batch: iter_time=7.998e-05, forward_time=0.142, loss_ctc=68.755, loss_interctc_layer6=76.701, loss_interctc_layer12=64.096, loss_interctc_layer15=59.071, loss_interctc_layer21=71.172, loss=67.959, backward_time=0.210, grad_norm=89.418, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.607e-05, train_time=0.993 +[gpua006:0/64] 2024-02-13 11:01:19,752 (multiple_iter_factory:32) INFO: Building 8th iter-factory... +[gpua006:0/64] 2024-02-13 11:01:38,750 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 11:01:42,451 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 11:01:42,451 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-13 11:01:42,455 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 11:08:22,368 (trainer:756) INFO: 37epoch:train:10001-10100batch: iter_time=3.319, forward_time=0.184, loss_ctc=77.880, loss_interctc_layer6=81.677, loss_interctc_layer12=67.387, loss_interctc_layer15=61.682, loss_interctc_layer21=80.572, loss=73.840, backward_time=0.214, grad_norm=66.587, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.606e-05, train_time=4.426 +[gpua006:0/64] 2024-02-13 11:10:08,922 (trainer:756) INFO: 37epoch:train:10101-10200batch: iter_time=8.375e-05, forward_time=0.142, loss_ctc=64.338, loss_interctc_layer6=74.349, loss_interctc_layer12=61.337, loss_interctc_layer15=56.047, loss_interctc_layer21=66.587, loss=64.532, backward_time=0.208, grad_norm=67.524, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.136, optim0_lr0=6.606e-05, train_time=1.065 +[gpua006:0/64] 2024-02-13 11:12:48,428 (trainer:756) INFO: 37epoch:train:10201-10300batch: iter_time=8.303e-05, forward_time=0.152, loss_ctc=69.474, loss_interctc_layer6=79.915, loss_interctc_layer12=66.151, loss_interctc_layer15=60.779, loss_interctc_layer21=72.193, loss=69.703, backward_time=0.214, grad_norm=72.364, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.137, optim0_lr0=6.605e-05, train_time=1.595 +[gpua006:0/64] 2024-02-13 11:14:42,668 (trainer:756) INFO: 37epoch:train:10301-10400batch: iter_time=8.542e-05, forward_time=0.168, loss_ctc=75.028, loss_interctc_layer6=84.211, loss_interctc_layer12=69.369, loss_interctc_layer15=63.483, loss_interctc_layer21=77.626, loss=73.943, backward_time=0.222, grad_norm=77.584, clip=100.000, loss_scale=1.988e+31, optim_step_time=0.137, optim0_lr0=6.604e-05, train_time=1.142 +[gpua006:0/64] 2024-02-13 11:17:11,626 (trainer:756) INFO: 37epoch:train:10401-10500batch: iter_time=8.764e-05, forward_time=0.144, loss_ctc=82.324, loss_interctc_layer6=87.327, loss_interctc_layer12=72.634, loss_interctc_layer15=66.557, loss_interctc_layer21=85.393, loss=78.847, backward_time=0.210, grad_norm=91.331, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.604e-05, train_time=1.490 +[gpua006:0/64] 2024-02-13 11:19:09,076 (trainer:756) INFO: 37epoch:train:10501-10600batch: iter_time=8.500e-05, forward_time=0.142, loss_ctc=61.301, loss_interctc_layer6=70.801, loss_interctc_layer12=58.548, loss_interctc_layer15=53.554, loss_interctc_layer21=63.530, loss=61.547, backward_time=0.208, grad_norm=73.108, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.603e-05, train_time=1.174 +[gpua006:0/64] 2024-02-13 11:21:26,426 (trainer:756) INFO: 37epoch:train:10601-10700batch: iter_time=8.486e-05, forward_time=0.145, loss_ctc=89.178, loss_interctc_layer6=93.079, loss_interctc_layer12=77.264, loss_interctc_layer15=70.919, loss_interctc_layer21=92.663, loss=84.621, backward_time=0.209, grad_norm=76.314, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.603e-05, train_time=1.372 +[gpua006:0/64] 2024-02-13 11:23:53,634 (trainer:756) INFO: 37epoch:train:10701-10800batch: iter_time=8.396e-05, forward_time=0.143, loss_ctc=76.319, loss_interctc_layer6=83.464, loss_interctc_layer12=69.124, loss_interctc_layer15=63.475, loss_interctc_layer21=79.117, loss=74.300, backward_time=0.208, grad_norm=122.604, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.602e-05, train_time=1.473 +[gpua006:0/64] 2024-02-13 11:26:24,795 (trainer:756) INFO: 37epoch:train:10801-10900batch: iter_time=8.685e-05, forward_time=0.148, loss_ctc=80.663, loss_interctc_layer6=80.344, loss_interctc_layer12=66.093, loss_interctc_layer15=60.193, loss_interctc_layer21=83.940, loss=74.247, backward_time=0.209, grad_norm=98.164, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.601e-05, train_time=1.511 +[gpua006:0/64] 2024-02-13 11:29:05,794 (trainer:756) INFO: 37epoch:train:10901-11000batch: iter_time=2.903e-04, forward_time=0.183, loss_ctc=73.751, loss_interctc_layer6=86.610, loss_interctc_layer12=71.460, loss_interctc_layer15=65.443, loss_interctc_layer21=76.418, loss=74.737, backward_time=0.221, grad_norm=84.940, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.144, optim0_lr0=6.601e-05, train_time=1.609 +[gpua006:0/64] 2024-02-13 11:31:12,436 (trainer:756) INFO: 37epoch:train:11001-11100batch: iter_time=8.795e-05, forward_time=0.142, loss_ctc=71.179, loss_interctc_layer6=73.097, loss_interctc_layer12=60.120, loss_interctc_layer15=55.109, loss_interctc_layer21=73.831, loss=66.667, backward_time=0.209, grad_norm=60.730, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.600e-05, train_time=1.266 +[gpua006:0/64] 2024-02-13 11:33:28,969 (trainer:756) INFO: 37epoch:train:11101-11200batch: iter_time=8.705e-05, forward_time=0.149, loss_ctc=73.602, loss_interctc_layer6=80.059, loss_interctc_layer12=65.961, loss_interctc_layer15=60.340, loss_interctc_layer21=76.377, loss=71.268, backward_time=0.215, grad_norm=59.890, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.600e-05, train_time=1.365 +[gpua006:0/64] 2024-02-13 11:34:39,211 (multiple_iter_factory:32) INFO: Building 9th iter-factory... +[gpua006:0/64] 2024-02-13 11:34:58,534 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 11:35:02,025 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 11:35:02,025 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, +[gpua006:0/64] 2024-02-13 11:35:02,029 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 11:39:51,988 (trainer:756) INFO: 37epoch:train:11201-11300batch: iter_time=2.685, forward_time=0.160, loss_ctc=70.126, loss_interctc_layer6=75.469, loss_interctc_layer12=62.794, loss_interctc_layer15=57.822, loss_interctc_layer21=72.646, loss=67.772, backward_time=0.212, grad_norm=91.957, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.599e-05, train_time=3.830 +[gpua006:0/64] 2024-02-13 11:41:27,089 (trainer:756) INFO: 37epoch:train:11301-11400batch: iter_time=8.492e-05, forward_time=0.145, loss_ctc=71.034, loss_interctc_layer6=79.473, loss_interctc_layer12=65.403, loss_interctc_layer15=59.798, loss_interctc_layer21=73.643, loss=69.870, backward_time=0.209, grad_norm=85.901, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.598e-05, train_time=0.951 +[gpua006:0/64] 2024-02-13 11:43:05,489 (trainer:756) INFO: 37epoch:train:11401-11500batch: iter_time=8.567e-05, forward_time=0.142, loss_ctc=77.029, loss_interctc_layer6=79.133, loss_interctc_layer12=65.740, loss_interctc_layer15=60.381, loss_interctc_layer21=80.022, loss=72.461, backward_time=0.209, grad_norm=89.870, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.598e-05, train_time=0.984 +[gpua006:0/64] 2024-02-13 11:45:15,717 (trainer:756) INFO: 37epoch:train:11501-11600batch: iter_time=9.411e-05, forward_time=0.146, loss_ctc=69.303, loss_interctc_layer6=83.060, loss_interctc_layer12=68.645, loss_interctc_layer15=62.702, loss_interctc_layer21=71.637, loss=71.069, backward_time=0.209, grad_norm=85.886, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.597e-05, train_time=1.301 +[gpua006:0/64] 2024-02-13 11:47:32,471 (trainer:756) INFO: 37epoch:train:11601-11700batch: iter_time=9.558e-05, forward_time=0.143, loss_ctc=81.387, loss_interctc_layer6=82.240, loss_interctc_layer12=68.367, loss_interctc_layer15=62.687, loss_interctc_layer21=84.496, loss=75.835, backward_time=0.209, grad_norm=76.922, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.597e-05, train_time=1.368 +[gpua006:0/64] 2024-02-13 11:50:11,237 (trainer:756) INFO: 37epoch:train:11701-11800batch: iter_time=9.304e-05, forward_time=0.144, loss_ctc=83.969, loss_interctc_layer6=83.705, loss_interctc_layer12=69.166, loss_interctc_layer15=63.306, loss_interctc_layer21=87.077, loss=77.445, backward_time=0.213, grad_norm=71.874, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.596e-05, train_time=1.587 +[gpua006:0/64] 2024-02-13 11:52:31,998 (trainer:756) INFO: 37epoch:train:11801-11900batch: iter_time=8.896e-05, forward_time=0.169, loss_ctc=85.972, loss_interctc_layer6=81.684, loss_interctc_layer12=67.926, loss_interctc_layer15=62.382, loss_interctc_layer21=89.342, loss=77.461, backward_time=0.294, grad_norm=92.553, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.595e-05, train_time=1.407 +[gpua006:0/64] 2024-02-13 11:54:21,559 (trainer:756) INFO: 37epoch:train:11901-12000batch: iter_time=9.838e-05, forward_time=0.150, loss_ctc=71.078, loss_interctc_layer6=76.305, loss_interctc_layer12=62.738, loss_interctc_layer15=57.163, loss_interctc_layer21=73.746, loss=68.206, backward_time=0.213, grad_norm=79.357, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.595e-05, train_time=1.096 +[gpua006:0/64] 2024-02-13 11:56:47,134 (trainer:756) INFO: 37epoch:train:12001-12100batch: iter_time=8.283e-05, forward_time=0.158, loss_ctc=88.265, loss_interctc_layer6=91.696, loss_interctc_layer12=76.285, loss_interctc_layer15=70.500, loss_interctc_layer21=91.444, loss=83.638, backward_time=0.210, grad_norm=91.786, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.594e-05, train_time=1.455 +[gpua006:0/64] 2024-02-13 11:59:21,019 (trainer:756) INFO: 37epoch:train:12101-12200batch: iter_time=9.237e-05, forward_time=0.143, loss_ctc=75.908, loss_interctc_layer6=78.880, loss_interctc_layer12=64.707, loss_interctc_layer15=58.949, loss_interctc_layer21=78.827, loss=71.454, backward_time=0.206, grad_norm=71.142, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.594e-05, train_time=1.538 +[gpua006:0/64] 2024-02-13 12:01:41,116 (trainer:756) INFO: 37epoch:train:12201-12300batch: iter_time=8.908e-05, forward_time=0.156, loss_ctc=82.127, loss_interctc_layer6=85.806, loss_interctc_layer12=70.845, loss_interctc_layer15=64.922, loss_interctc_layer21=85.046, loss=77.749, backward_time=0.215, grad_norm=81.402, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.593e-05, train_time=1.402 +[gpua006:0/64] 2024-02-13 12:03:45,648 (trainer:756) INFO: 37epoch:train:12301-12400batch: iter_time=9.338e-05, forward_time=0.143, loss_ctc=75.900, loss_interctc_layer6=75.800, loss_interctc_layer12=62.181, loss_interctc_layer15=56.684, loss_interctc_layer21=78.807, loss=69.874, backward_time=0.209, grad_norm=93.975, clip=100.000, loss_scale=3.975e+31, optim_step_time=0.136, optim0_lr0=6.593e-05, train_time=1.245 +[gpua006:0/64] 2024-02-13 12:05:53,266 (trainer:756) INFO: 37epoch:train:12401-12500batch: iter_time=8.787e-05, forward_time=0.142, loss_ctc=72.231, loss_interctc_layer6=76.160, loss_interctc_layer12=63.449, loss_interctc_layer15=58.380, loss_interctc_layer21=74.840, loss=69.012, backward_time=0.209, grad_norm=83.532, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.592e-05, train_time=1.276 +[gpua006:0/64] 2024-02-13 12:06:13,295 (multiple_iter_factory:32) INFO: Building 10th iter-factory... +[gpua006:0/64] 2024-02-13 12:06:32,484 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 12:06:35,995 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 12:06:35,995 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, +[gpua006:0/64] 2024-02-13 12:06:36,010 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 12:13:00,075 (trainer:756) INFO: 37epoch:train:12501-12600batch: iter_time=3.257, forward_time=0.171, loss_ctc=77.501, loss_interctc_layer6=81.836, loss_interctc_layer12=67.509, loss_interctc_layer15=61.686, loss_interctc_layer21=80.292, loss=73.765, backward_time=0.215, grad_norm=72.934, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.591e-05, train_time=4.266 +[gpua006:0/64] 2024-02-13 12:14:46,056 (trainer:756) INFO: 37epoch:train:12601-12700batch: iter_time=8.184e-05, forward_time=0.141, loss_ctc=63.987, loss_interctc_layer6=74.772, loss_interctc_layer12=61.716, loss_interctc_layer15=56.439, loss_interctc_layer21=66.136, loss=64.610, backward_time=0.208, grad_norm=72.896, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.591e-05, train_time=1.061 +[gpua006:0/64] 2024-02-13 12:16:32,142 (trainer:756) INFO: 37epoch:train:12701-12800batch: iter_time=8.134e-05, forward_time=0.142, loss_ctc=69.255, loss_interctc_layer6=80.060, loss_interctc_layer12=66.393, loss_interctc_layer15=60.770, loss_interctc_layer21=72.035, loss=69.703, backward_time=0.208, grad_norm=84.222, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.590e-05, train_time=1.060 +[gpua006:0/64] 2024-02-13 12:18:36,360 (trainer:756) INFO: 37epoch:train:12801-12900batch: iter_time=8.321e-05, forward_time=0.142, loss_ctc=74.704, loss_interctc_layer6=84.338, loss_interctc_layer12=69.509, loss_interctc_layer15=63.468, loss_interctc_layer21=77.394, loss=73.882, backward_time=0.207, grad_norm=98.290, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.590e-05, train_time=1.242 +[gpua006:0/64] 2024-02-13 12:22:04,688 (trainer:756) INFO: 37epoch:train:12901-13000batch: iter_time=8.492e-05, forward_time=0.236, loss_ctc=82.778, loss_interctc_layer6=87.835, loss_interctc_layer12=72.941, loss_interctc_layer15=66.877, loss_interctc_layer21=86.054, loss=79.297, backward_time=0.256, grad_norm=91.747, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=6.589e-05, train_time=2.081 +[gpua006:0/64] 2024-02-13 12:24:14,669 (trainer:756) INFO: 37epoch:train:13001-13100batch: iter_time=8.395e-05, forward_time=0.143, loss_ctc=60.908, loss_interctc_layer6=70.998, loss_interctc_layer12=58.750, loss_interctc_layer15=53.807, loss_interctc_layer21=63.424, loss=61.578, backward_time=0.207, grad_norm=81.427, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.588e-05, train_time=1.302 +[gpua006:0/64] 2024-02-13 12:26:28,687 (trainer:756) INFO: 37epoch:train:13101-13200batch: iter_time=8.431e-05, forward_time=0.143, loss_ctc=89.738, loss_interctc_layer6=93.583, loss_interctc_layer12=77.929, loss_interctc_layer15=71.572, loss_interctc_layer21=93.086, loss=85.182, backward_time=0.207, grad_norm=94.444, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.588e-05, train_time=1.340 +[gpua006:0/64] 2024-02-13 12:28:26,372 (trainer:756) INFO: 37epoch:train:13201-13300batch: iter_time=8.387e-05, forward_time=0.155, loss_ctc=74.985, loss_interctc_layer6=83.214, loss_interctc_layer12=68.815, loss_interctc_layer15=63.105, loss_interctc_layer21=77.742, loss=73.572, backward_time=0.211, grad_norm=88.274, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=6.587e-05, train_time=1.177 +[gpua006:0/64] 2024-02-13 12:30:42,731 (trainer:756) INFO: 37epoch:train:13301-13400batch: iter_time=9.130e-05, forward_time=0.241, loss_ctc=80.799, loss_interctc_layer6=80.072, loss_interctc_layer12=65.712, loss_interctc_layer15=59.881, loss_interctc_layer21=83.985, loss=74.090, backward_time=0.239, grad_norm=71.991, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=6.587e-05, train_time=1.361 +[gpua006:0/64] 2024-02-13 12:33:08,266 (trainer:756) INFO: 37epoch:train:13401-13500batch: iter_time=9.607e-05, forward_time=0.148, loss_ctc=73.764, loss_interctc_layer6=86.433, loss_interctc_layer12=71.298, loss_interctc_layer15=65.207, loss_interctc_layer21=76.372, loss=74.615, backward_time=0.209, grad_norm=141.289, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=6.586e-05, train_time=1.457 +[gpua006:0/64] 2024-02-13 12:35:01,095 (trainer:756) INFO: 37epoch:train:13501-13600batch: iter_time=9.071e-05, forward_time=0.142, loss_ctc=70.528, loss_interctc_layer6=72.870, loss_interctc_layer12=59.805, loss_interctc_layer15=54.627, loss_interctc_layer21=73.179, loss=66.202, backward_time=0.209, grad_norm=70.790, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.585e-05, train_time=1.128 +[gpua006:0/64] 2024-02-13 12:37:41,283 (trainer:756) INFO: 37epoch:train:13601-13700batch: iter_time=8.706e-05, forward_time=0.142, loss_ctc=73.050, loss_interctc_layer6=79.709, loss_interctc_layer12=65.726, loss_interctc_layer15=60.056, loss_interctc_layer21=75.817, loss=70.871, backward_time=0.207, grad_norm=66.273, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.136, optim0_lr0=6.585e-05, train_time=1.602 +[gpua006:0/64] 2024-02-13 12:39:00,664 (multiple_iter_factory:32) INFO: Building 11th iter-factory... +[gpua006:0/64] 2024-02-13 12:39:19,729 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 12:39:23,195 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 12:39:23,195 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, +[gpua006:0/64] 2024-02-13 12:39:23,199 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 12:44:11,671 (trainer:756) INFO: 37epoch:train:13701-13800batch: iter_time=2.778, forward_time=0.180, loss_ctc=68.859, loss_interctc_layer6=75.367, loss_interctc_layer12=62.787, loss_interctc_layer15=57.892, loss_interctc_layer21=71.446, loss=67.270, backward_time=0.220, grad_norm=71.287, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=6.584e-05, train_time=3.904 +[gpua006:0/64] 2024-02-13 12:45:46,801 (trainer:756) INFO: 37epoch:train:13801-13900batch: iter_time=8.390e-05, forward_time=0.143, loss_ctc=66.723, loss_interctc_layer6=79.478, loss_interctc_layer12=65.561, loss_interctc_layer15=59.892, loss_interctc_layer21=69.056, loss=68.142, backward_time=0.209, grad_norm=65.246, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=6.584e-05, train_time=0.951 +[gpua006:0/64] 2024-02-13 12:47:49,881 (trainer:756) INFO: 37epoch:train:13901-14000batch: iter_time=9.060e-05, forward_time=0.229, loss_ctc=70.285, loss_interctc_layer6=78.690, loss_interctc_layer12=65.222, loss_interctc_layer15=59.911, loss_interctc_layer21=72.839, loss=69.389, backward_time=0.252, grad_norm=76.766, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.143, optim0_lr0=6.583e-05, train_time=1.230 +[gpua006:0/64] 2024-02-13 12:49:47,552 (trainer:687) WARNING: The grad norm is nan. Skipping updating the model. +[gpua006:0/64] 2024-02-13 12:49:54,551 (trainer:756) INFO: 37epoch:train:14001-14100batch: iter_time=8.108e-05, forward_time=0.142, loss_ctc=65.888, loss_interctc_layer6=82.122, loss_interctc_layer12=67.830, loss_interctc_layer15=61.861, loss_interctc_layer21=68.090, loss=69.158, backward_time=0.208, grad_norm=112.189, clip=100.000, loss_scale=3.954e+31, optim_step_time=0.137, optim0_lr0=6.582e-05, train_time=1.247 +[gpua006:0/64] 2024-02-13 12:52:30,490 (trainer:756) INFO: 37epoch:train:14101-14200batch: iter_time=8.366e-05, forward_time=0.143, loss_ctc=77.358, loss_interctc_layer6=81.824, loss_interctc_layer12=68.108, loss_interctc_layer15=62.427, loss_interctc_layer21=80.208, loss=73.985, backward_time=0.208, grad_norm=71.292, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.582e-05, train_time=1.559 +[gpua006:0/64] 2024-02-13 12:55:07,364 (trainer:756) INFO: 37epoch:train:14201-14300batch: iter_time=8.366e-05, forward_time=0.245, loss_ctc=78.648, loss_interctc_layer6=83.277, loss_interctc_layer12=68.788, loss_interctc_layer15=62.821, loss_interctc_layer21=81.683, loss=75.044, backward_time=0.261, grad_norm=86.505, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=6.581e-05, train_time=1.568 +[gpua006:0/64] 2024-02-13 12:57:18,330 (trainer:756) INFO: 37epoch:train:14301-14400batch: iter_time=8.369e-05, forward_time=0.143, loss_ctc=81.195, loss_interctc_layer6=82.145, loss_interctc_layer12=68.387, loss_interctc_layer15=62.674, loss_interctc_layer21=84.425, loss=75.765, backward_time=0.207, grad_norm=80.684, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.581e-05, train_time=1.307 +[gpua006:0/64] 2024-02-13 12:59:36,575 (trainer:756) INFO: 37epoch:train:14401-14500batch: iter_time=8.450e-05, forward_time=0.142, loss_ctc=65.273, loss_interctc_layer6=76.216, loss_interctc_layer12=62.792, loss_interctc_layer15=57.174, loss_interctc_layer21=67.700, loss=65.831, backward_time=0.207, grad_norm=148.289, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.580e-05, train_time=1.385 +[gpua006:0/64] 2024-02-13 13:02:07,056 (trainer:756) INFO: 37epoch:train:14501-14600batch: iter_time=8.505e-05, forward_time=0.143, loss_ctc=85.605, loss_interctc_layer6=91.359, loss_interctc_layer12=76.265, loss_interctc_layer15=70.192, loss_interctc_layer21=88.599, loss=82.404, backward_time=0.207, grad_norm=212.427, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.579e-05, train_time=1.505 +[gpua006:0/64] 2024-02-13 13:04:36,729 (trainer:756) INFO: 37epoch:train:14601-14700batch: iter_time=8.215e-05, forward_time=0.253, loss_ctc=74.169, loss_interctc_layer6=79.386, loss_interctc_layer12=65.222, loss_interctc_layer15=59.411, loss_interctc_layer21=77.081, loss=71.054, backward_time=0.228, grad_norm=63.737, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.579e-05, train_time=1.496 +[gpua006:0/64] 2024-02-13 13:06:33,120 (trainer:756) INFO: 37epoch:train:14701-14800batch: iter_time=8.127e-05, forward_time=0.142, loss_ctc=79.567, loss_interctc_layer6=85.391, loss_interctc_layer12=70.640, loss_interctc_layer15=64.727, loss_interctc_layer21=82.532, loss=76.571, backward_time=0.209, grad_norm=89.588, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.578e-05, train_time=1.163 +[gpua006:0/64] 2024-02-13 13:08:49,649 (trainer:756) INFO: 37epoch:train:14801-14900batch: iter_time=8.611e-05, forward_time=0.143, loss_ctc=69.996, loss_interctc_layer6=75.958, loss_interctc_layer12=62.531, loss_interctc_layer15=57.072, loss_interctc_layer21=72.538, loss=67.619, backward_time=0.209, grad_norm=68.178, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.578e-05, train_time=1.366 +[gpua006:0/64] 2024-02-13 13:11:16,718 (trainer:756) INFO: 37epoch:train:14901-15000batch: iter_time=1.031e-04, forward_time=0.269, loss_ctc=67.838, loss_interctc_layer6=76.071, loss_interctc_layer12=63.396, loss_interctc_layer15=58.357, loss_interctc_layer21=70.307, loss=67.194, backward_time=0.236, grad_norm=63.865, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=6.577e-05, train_time=1.470 +[gpua006:0/64] 2024-02-13 13:42:19,013 (trainer:355) INFO: 37epoch results: [train] iter_time=0.230, forward_time=0.158, loss_ctc=76.345, loss_interctc_layer6=81.510, loss_interctc_layer12=67.501, loss_interctc_layer15=61.872, loss_interctc_layer21=79.135, loss=73.273, backward_time=0.215, grad_norm=84.775, clip=100.000, loss_scale=1.804e+31, optim_step_time=0.137, optim0_lr0=6.622e-05, train_time=1.504, time=6 hours, 16 minutes and 25.06 seconds, total_count=555000, gpu_max_cached_mem_GB=33.436, [valid] loss_ctc=42.243, cer_ctc=0.193, loss_interctc_layer6=48.624, cer_interctc_layer6=0.215, loss_interctc_layer12=35.538, cer_interctc_layer12=0.146, loss_interctc_layer15=31.279, cer_interctc_layer15=0.123, loss_interctc_layer21=44.818, cer_interctc_layer21=0.206, loss=40.500, time=30 minutes and 38.23 seconds, total_count=172827, gpu_max_cached_mem_GB=33.436 +[gpua006:0/64] 2024-02-13 13:42:38,904 (trainer:410) INFO: The best model has been updated: valid.total_count +[gpua006:0/64] 2024-02-13 13:42:38,945 (trainer:289) INFO: 38/45epoch started. Estimated time to finish: 2 days, 7 hours and 11 minutes +[gpua006:0/64] 2024-02-13 13:42:38,961 (multiple_iter_factory:32) INFO: Building 0th iter-factory... +[gpua006:0/64] 2024-02-13 13:42:57,106 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4') +[gpua006:0/64] 2024-02-13 13:43:00,528 (abs_task:1660) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"} + text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"} + text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"} + text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"} + preprocess: ) +[gpua006:0/64] 2024-02-13 13:43:00,528 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, +[gpua006:0/64] 2024-02-13 13:43:00,531 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257 +[gpua006:0/64] 2024-02-13 13:49:32,703 (trainer:756) INFO: 38epoch:train:1-100batch: iter_time=2.936, forward_time=0.173, loss_ctc=80.929, loss_interctc_layer6=87.954, loss_interctc_layer12=72.658, loss_interctc_layer15=66.502, loss_interctc_layer21=84.181, loss=78.445, backward_time=0.215, grad_norm=88.924, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.137, optim0_lr0=6.576e-05, train_time=4.137 +[gpua006:0/64] 2024-02-13 13:51:08,487 (trainer:756) INFO: 38epoch:train:101-200batch: iter_time=8.198e-05, forward_time=0.142, loss_ctc=66.358, loss_interctc_layer6=80.974, loss_interctc_layer12=66.883, loss_interctc_layer15=61.219, loss_interctc_layer21=68.572, loss=68.801, backward_time=0.209, grad_norm=75.943, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.576e-05, train_time=0.958 +[gpua006:0/64] 2024-02-13 13:52:54,440 (trainer:756) INFO: 38epoch:train:201-300batch: iter_time=9.081e-05, forward_time=0.144, loss_ctc=76.030, loss_interctc_layer6=91.236, loss_interctc_layer12=75.568, loss_interctc_layer15=69.159, loss_interctc_layer21=78.858, loss=78.170, backward_time=0.208, grad_norm=74.914, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.575e-05, train_time=1.059 +[gpua006:0/64] 2024-02-13 13:54:51,356 (trainer:756) INFO: 38epoch:train:301-400batch: iter_time=8.495e-05, forward_time=0.142, loss_ctc=65.845, loss_interctc_layer6=77.222, loss_interctc_layer12=64.073, loss_interctc_layer15=58.754, loss_interctc_layer21=68.316, loss=66.842, backward_time=0.211, grad_norm=67.061, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.575e-05, train_time=1.169 +[gpua006:0/64] 2024-02-13 13:56:33,547 (trainer:756) INFO: 38epoch:train:401-500batch: iter_time=8.289e-05, forward_time=0.143, loss_ctc=67.524, loss_interctc_layer6=82.208, loss_interctc_layer12=68.092, loss_interctc_layer15=62.365, loss_interctc_layer21=69.459, loss=69.930, backward_time=0.213, grad_norm=76.633, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.574e-05, train_time=1.021 +[gpua006:0/64] 2024-02-13 13:58:49,667 (trainer:756) INFO: 38epoch:train:501-600batch: iter_time=0.003, forward_time=0.170, loss_ctc=71.636, loss_interctc_layer6=81.895, loss_interctc_layer12=67.702, loss_interctc_layer15=61.905, loss_interctc_layer21=74.175, loss=71.463, backward_time=0.239, grad_norm=66.083, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.573e-05, train_time=1.362 +[gpua006:0/64] 2024-02-13 14:01:04,483 (trainer:756) INFO: 38epoch:train:601-700batch: iter_time=1.031e-04, forward_time=0.201, loss_ctc=67.914, loss_interctc_layer6=77.364, loss_interctc_layer12=64.191, loss_interctc_layer15=59.064, loss_interctc_layer21=70.315, loss=67.769, backward_time=0.223, grad_norm=72.279, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=6.573e-05, train_time=1.348 +[gpua006:0/64] 2024-02-13 14:03:20,630 (trainer:756) INFO: 38epoch:train:701-800batch: iter_time=9.019e-05, forward_time=0.141, loss_ctc=78.502, loss_interctc_layer6=84.006, loss_interctc_layer12=69.440, loss_interctc_layer15=63.601, loss_interctc_layer21=81.521, loss=75.414, backward_time=0.205, grad_norm=76.613, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.136, optim0_lr0=6.572e-05, train_time=1.362 +srun: Job step aborted: Waiting up to 32 seconds for job step to finish.