diff --git "a/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.29.log" "b/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.29.log"
new file mode 100644--- /dev/null
+++ "b/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.29.log"
@@ -0,0 +1,8849 @@
+# Running on gpua002.delta.ncsa.illinois.edu
+# Started at Thu Jan 18 08:48:18 CST 2024
+# SLURMD_NODENAME=gpua002
+# SLURM_CLUSTER_NAME=delta
+# SLURM_CONF=/var/spool/slurmd/conf-cache/slurm.conf
+# SLURM_CPUS_ON_NODE=64
+# SLURM_CPUS_PER_TASK=64
+# SLURM_EXPORT_ENV=PATH
+# SLURM_GET_USER_ENV=1
+# SLURM_GPUS_ON_NODE=4
+# SLURM_GTIDS=0
+# SLURM_JOBID=2867687
+# SLURM_JOB_ACCOUNT=bbjs-delta-gpu
+# SLURM_JOB_CPUS_PER_NODE='64(x16)'
+# SLURM_JOB_END_TIME=1705762082
+# SLURM_JOB_GID=202
+# SLURM_JOB_GPUS=0,1,2,3
+# SLURM_JOB_ID=2867687
+# SLURM_JOB_NAME=exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/train.log
+# SLURM_JOB_NODELIST='gpua[002,013,016-017,024,034-036,051,053,055,058,079-080,083,087]'
+# SLURM_JOB_NUM_NODES=16
+# SLURM_JOB_PARTITION=gpuA100x4
+# SLURM_JOB_QOS=bbjs-delta-gpu
+# SLURM_JOB_START_TIME=1705589282
+# SLURM_JOB_UID=68077
+# SLURM_JOB_USER=peng6
+# SLURM_LOCALID=0
+# SLURM_MEM_PER_NODE=240000
+# SLURM_NNODES=16
+# SLURM_NODEID=0
+# SLURM_NODELIST='gpua[002,013,016-017,024,034-036,051,053,055,058,079-080,083,087]'
+# SLURM_NODE_ALIASES='(null)'
+# SLURM_OPEN_MODE=a
+# SLURM_PRIO_PROCESS=0
+# SLURM_PROCID=0
+# SLURM_SUBMIT_DIR=/scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1
+# SLURM_SUBMIT_HOST=dt-login01.delta.ncsa.illinois.edu
+# SLURM_TASKS_PER_NODE='1(x16)'
+# SLURM_TASK_PID=51542
+# SLURM_TOPOLOGY_ADDR=ss00.ss05.gpua002
+# SLURM_TOPOLOGY_ADDR_PATTERN=switch.switch.node
+# SLURM_WORKING_CLUSTER=delta:dt-sched:6817:9984:109
+# srun --export=ALL python3 -m espnet2.bin.s2t_train --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25 
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+[gpua002:0/64] 2024-01-18 08:48:29,210 (distributed_c10d:319) INFO: Added key: store_based_barrier_key:1 to store for rank: 0
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+[gpua002:0/64] 2024-01-18 08:48:39,248 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=28, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:48:49,285 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:48:59,299 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:49:09,315 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:49:19,327 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:49:29,359 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:49:39,409 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:49:49,467 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:49:59,496 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:50:09,528 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:50:19,538 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:50:29,584 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:50:39,633 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:50:49,643 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:50:59,697 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:51:09,737 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:51:19,781 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:51:29,829 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:51:39,856 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:51:49,905 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:51:59,930 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:52:09,983 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/bin/python3 /scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py --use_preprocessor true --bpemodel data/token_list/bpe_unigram50000/bpe.model --token_type bpe --token_list data/token_list/bpe_unigram50000/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_v3/wav.scp,speech,kaldi_ark --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/speech_shape --resume true --fold_length 80000 --output_dir exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000 --config conf/train_s2t_multitask-ctc_ebf27_conv2d8_size1024.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/wav.scp,speech,kaldi_ark --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/speech_shape --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method fraw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+ile:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method fraw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+ile:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+raw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method fraw_bpe50000/splits12/text.prev,text_prev,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_prev_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text.ctc,text_ctc,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_ctc_shape.bpe --fold_length 150 --train_data_path_and_name_and_type exp/s2t_stats_raw_bpe50000/splits12/text,text,text --train_shape_file exp/s2t_stats_raw_bpe50000/splits12/text_shape.bpe --multiple_iterator true --valid_data_path_and_name_and_type dump/raw/dev_v3/text.prev,text_prev,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_prev_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text.ctc,text_ctc,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_ctc_shape.bpe --valid_data_path_and_name_and_type dump/raw/dev_v3/text,text,text --valid_shape_file exp/s2t_stats_raw_bpe50000/valid/text_shape.bpe --ngpu 4 --multiprocessing_distributed true --dist_launcher slurm --dist_init_method file:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+ile:///scratch/bbjs/peng6/espnet-owsm-ctc/egs2/owsm_v3.1_ctc/s2t1/exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/.dist_init_bf1159c2-d6f2-40e7-be9c-047791bedd25
+[gpua002:0/64] 2024-01-18 08:52:20,027 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=32, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:52:42,180 (distributed_c10d:337) INFO: Waiting in store based barrier to initialize process group for rank: 0, key: store_based_barrier_key:1 (world_size=64, worker_count=64, timeout=0:30:00)
+[gpua002:0/64] 2024-01-18 08:52:42,180 (distributed_c10d:353) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 64 nodes.
+[gpua002:0/64] 2024-01-18 08:52:42,308 (s2t:420) INFO: Vocabulary size: 50002
+[gpua002:0/64] 2024-01-18 08:52:53,646 (abs_task:1270) INFO: pytorch.version=1.13.1, cuda.available=True, cudnn.version=8500, cudnn.benchmark=False, cudnn.deterministic=True
+[gpua002:0/64] 2024-01-18 08:52:53,657 (abs_task:1271) INFO: Model structure:
+ESPnetS2TCTCModel(
+  (frontend): DefaultFrontend(
+    (stft): Stft(n_fft=512, win_length=400, hop_length=160, center=True, normalized=False, onesided=True)
+    (frontend): Frontend()
+    (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False)
+  )
+  (specaug): SpecAug(
+    (freq_mask): MaskAlongAxis(mask_width_range=[0, 27], num_mask=2, axis=freq)
+    (time_mask): MaskAlongAxisVariableMaxWidth(mask_width_ratio_range=[0.0, 0.05], num_mask=10, axis=time)
+  )
+  (normalize): GlobalMVN(stats_file=exp/s2t_stats_raw_bpe50000/train/feats_stats.npz, norm_means=True, norm_vars=True)
+  (encoder): EBranchformerCTCEncoder(
+    (embed): Conv2dSubsampling8(
+      (conv): Sequential(
+        (0): Conv2d(1, 1024, kernel_size=(3, 3), stride=(2, 2))
+        (1): ReLU()
+        (2): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2))
+        (3): ReLU()
+        (4): Conv2d(1024, 1024, kernel_size=(3, 3), stride=(2, 2))
+        (5): ReLU()
+      )
+      (out): Linear(in_features=9216, out_features=1024, bias=True)
+      (pos_enc): PositionalEncoding(
+        (dropout): Dropout(p=0.1, inplace=False)
+      )
+    )
+    (encoders): MultiSequential(
+      (0): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (1): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (2): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (3): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (4): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (5): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (6): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (7): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (8): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (9): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (10): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (11): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (12): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (13): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (14): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (15): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (16): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (17): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (18): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (19): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (20): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (21): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (22): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (23): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (24): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (25): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+      (26): EBranchformerEncoderLayer(
+        (attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (cgmlp): ConvolutionalGatingMLP(
+          (channel_proj1): Sequential(
+            (0): Linear(in_features=1024, out_features=4096, bias=True)
+            (1): GELU(approximate='none')
+          )
+          (csgu): ConvolutionalSpatialGatingUnit(
+            (norm): LayerNorm((2048,), eps=1e-12, elementwise_affine=True)
+            (conv): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+            (act): Identity()
+            (dropout): Dropout(p=0.1, inplace=False)
+          )
+          (channel_proj2): Linear(in_features=2048, out_features=1024, bias=True)
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (feed_forward_macaron): PositionwiseFeedForward(
+          (w_1): Linear(in_features=1024, out_features=4096, bias=True)
+          (w_2): Linear(in_features=4096, out_features=1024, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): Swish()
+        )
+        (norm_ff): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_ff_macaron): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mha): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_mlp): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (norm_final): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (cross_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_k): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_v): Linear(in_features=1024, out_features=1024, bias=True)
+          (linear_out): Linear(in_features=1024, out_features=1024, bias=True)
+          (dropout): Identity()
+        )
+        (norm_cross_attn): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+        (depthwise_conv_fusion): Conv1d(2048, 2048, kernel_size=(31,), stride=(1,), padding=(15,), groups=2048)
+        (merge_proj): Linear(in_features=2048, out_features=1024, bias=True)
+      )
+    )
+    (after_norm): LayerNorm((1024,), eps=1e-12, elementwise_affine=True)
+    (conditioning_layer): Linear(in_features=50002, out_features=1024, bias=True)
+  )
+  (prompt_encoder): TransformerEncoder(
+    (encoders): MultiSequential(
+      (0): EncoderLayer(
+        (self_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=512, out_features=512, bias=True)
+          (linear_k): Linear(in_features=512, out_features=512, bias=True)
+          (linear_v): Linear(in_features=512, out_features=512, bias=True)
+          (linear_out): Linear(in_features=512, out_features=512, bias=True)
+          (dropout): Identity()
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=512, out_features=2048, bias=True)
+          (w_2): Linear(in_features=2048, out_features=512, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): ReLU()
+        )
+        (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+      )
+      (1): EncoderLayer(
+        (self_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=512, out_features=512, bias=True)
+          (linear_k): Linear(in_features=512, out_features=512, bias=True)
+          (linear_v): Linear(in_features=512, out_features=512, bias=True)
+          (linear_out): Linear(in_features=512, out_features=512, bias=True)
+          (dropout): Identity()
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=512, out_features=2048, bias=True)
+          (w_2): Linear(in_features=2048, out_features=512, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): ReLU()
+        )
+        (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+      )
+      (2): EncoderLayer(
+        (self_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=512, out_features=512, bias=True)
+          (linear_k): Linear(in_features=512, out_features=512, bias=True)
+          (linear_v): Linear(in_features=512, out_features=512, bias=True)
+          (linear_out): Linear(in_features=512, out_features=512, bias=True)
+          (dropout): Identity()
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=512, out_features=2048, bias=True)
+          (w_2): Linear(in_features=2048, out_features=512, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): ReLU()
+        )
+        (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+      )
+      (3): EncoderLayer(
+        (self_attn): MultiHeadedAttention(
+          (linear_q): Linear(in_features=512, out_features=512, bias=True)
+          (linear_k): Linear(in_features=512, out_features=512, bias=True)
+          (linear_v): Linear(in_features=512, out_features=512, bias=True)
+          (linear_out): Linear(in_features=512, out_features=512, bias=True)
+          (dropout): Identity()
+        )
+        (feed_forward): PositionwiseFeedForward(
+          (w_1): Linear(in_features=512, out_features=2048, bias=True)
+          (w_2): Linear(in_features=2048, out_features=512, bias=True)
+          (dropout): Dropout(p=0.1, inplace=False)
+          (activation): ReLU()
+        )
+        (norm1): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (norm2): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+        (dropout): Dropout(p=0.1, inplace=False)
+      )
+    )
+    (after_norm): LayerNorm((512,), eps=1e-12, elementwise_affine=True)
+  )
+  (embed): Embedding(50002, 512)
+  (pos_enc): PositionalEncoding(
+    (dropout): Dropout(p=0.0, inplace=False)
+  )
+  (embed_proj): Linear(in_features=512, out_features=1024, bias=True)
+  (prompt_proj): Linear(in_features=512, out_features=1024, bias=True)
+  (ctc): CTC(
+    (ctc_lo): Linear(in_features=1024, out_features=50002, bias=True)
+    (ctc_loss): CTCLoss()
+  )
+)
+
+Model summary:
+    Class Name: ESPnetS2TCTCModel
+    Total Number of model parameters: 1.01 B
+    Number of trainable parameters: 1.01 B (100.0%)
+    Size: 4.02 GB
+    Type: torch.float32
+[gpua002:0/64] 2024-01-18 08:52:53,658 (abs_task:1274) INFO: Optimizer:
+AdamW (
+Parameter Group 0
+    amsgrad: False
+    betas: [0.9, 0.98]
+    capturable: False
+    eps: 1e-06
+    foreach: None
+    initial_lr: 0.0002
+    lr: 1.6666666666666667e-09
+    maximize: False
+    weight_decay: 0.0
+)
+[gpua002:0/64] 2024-01-18 08:52:53,658 (abs_task:1275) INFO: Scheduler: PiecewiseLinearWarmupLR(warmup_steps_list=[0, 30000, 60000], warmup_lr_list=[0.0, 5e-05, 0.0002])
+[gpua002:0/64] 2024-01-18 08:52:53,672 (abs_task:1284) INFO: Saving the configuration in exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/config.yaml
+[gpua002:0/64] 2024-01-18 08:52:59,236 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 08:53:00,158 (abs_task:1660) INFO: [valid] dataset:
+ESPnetDataset(
+  speech: {"path": "dump/raw/dev_v3/wav.scp", "type": "kaldi_ark"}
+  text_prev: {"path": "dump/raw/dev_v3/text.prev", "type": "text"}
+  text_ctc: {"path": "dump/raw/dev_v3/text.ctc", "type": "text"}
+  text: {"path": "dump/raw/dev_v3/text", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7ff4e2f79f90>)
+[gpua002:0/64] 2024-01-18 08:53:00,158 (abs_task:1661) INFO: [valid] Batch sampler: UnsortedBatchSampler(N-batch=4671, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/valid/speech_shape, 
+[gpua002:0/64] 2024-01-18 08:53:00,159 (abs_task:1662) INFO: [valid] mini-batch sizes summary: N-batch=4671, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 08:53:35,252 (trainer:167) INFO: The training was resumed using exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/checkpoint.pth
+gpua002:51609:51609 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.2<0>
+gpua002:51609:51609 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua002:51609:51609 [0] NCCL INFO cudaDriverVersion 12020
+NCCL version 2.14.3+cuda11.7
+[gpua002:0/64] 2024-01-18 08:53:40,986 (trainer:298) INFO: 10/45epoch started
+[gpua002:0/64] 2024-01-18 08:53:41,027 (multiple_iter_factory:32) INFO: Building 0th iter-factory...
+[gpua002:0/64] 2024-01-18 08:54:00,138 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 08:54:03,611 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7ff30fcf0cd0>)
+[gpua002:0/64] 2024-01-18 08:54:03,611 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, 
+[gpua002:0/64] 2024-01-18 08:54:03,615 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+gpua017:2815425:2815425 [2] NCCL INFO cudaDriverVersion 12020
+gpua017:2815425:2815425 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.17<0>
+gpua017:2815425:2815425 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua017:2815425:2815496 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.17<0>
+gpua017:2815425:2815496 [2] NCCL INFO Using network IB
+gpua017:2815425:2815496 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua017:2815425:2815496 [2] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13
+gpua017:2815425:2815496 [2] NCCL INFO Channel 00/0 : 14[85000] -> 15[c7000] via P2P/IPC/read
+gpua017:2815425:2815496 [2] NCCL INFO Channel 01/0 : 14[85000] -> 15[c7000] via P2P/IPC/read
+gpua017:2815425:2815496 [2] NCCL INFO Connected all rings
+gpua017:2815425:2815496 [2] NCCL INFO Channel 00/0 : 14[85000] -> 13[46000] via P2P/IPC/read
+gpua017:2815425:2815496 [2] NCCL INFO Channel 01/0 : 14[85000] -> 13[46000] via P2P/IPC/read
+gpua017:2815425:2815496 [2] NCCL INFO Connected all trees
+gpua017:2815425:2815496 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua017:2815425:2815496 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua017:2815425:2815496 [2] NCCL INFO comm 0x55bc91f10020 rank 14 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua017:2815426:2815426 [3] NCCL INFO cudaDriverVersion 12020
+gpua017:2815426:2815426 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.17<0>
+gpua017:2815426:2815426 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua017:2815426:2815498 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.17<0>
+gpua017:2815426:2815498 [3] NCCL INFO Using network IB
+gpua017:2815426:2815498 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua017:2815426:2815498 [3] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] -1/-1/-1->15->14
+gpua017:2815426:2815498 [3] NCCL INFO Channel 00/0 : 15[c7000] -> 16[7000] [send] via NET/IB/0
+gpua017:2815426:2815498 [3] NCCL INFO Channel 01/0 : 15[c7000] -> 16[7000] [send] via NET/IB/0
+gpua017:2815426:2815498 [3] NCCL INFO Connected all rings
+gpua017:2815426:2815498 [3] NCCL INFO Channel 00/0 : 15[c7000] -> 14[85000] via P2P/IPC/read
+gpua017:2815426:2815498 [3] NCCL INFO Channel 01/0 : 15[c7000] -> 14[85000] via P2P/IPC/read
+gpua036:985763:985763 [1] NCCL INFO cudaDriverVersion 12020
+gpua036:985763:985763 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.36<0>
+gpua036:985763:985763 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua036:985763:985825 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.36<0>
+gpua036:985763:985825 [1] NCCL INFO Using network IB
+gpua036:985763:985825 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua036:985763:985825 [1] NCCL INFO Trees [0] 30/-1/-1->29->28 [1] 30/44/-1->29->28
+gpua036:985763:985825 [1] NCCL INFO Channel 00/0 : 29[46000] -> 30[85000] via P2P/IPC/read
+gpua036:985763:985825 [1] NCCL INFO Channel 01/0 : 29[46000] -> 30[85000] via P2P/IPC/read
+gpua036:985763:985825 [1] NCCL INFO Connected all rings
+gpua036:985763:985825 [1] NCCL INFO Channel 01/0 : 29[46000] -> 44[7000] [send] via NET/IB/0
+gpua036:985763:985825 [1] NCCL INFO Channel 01/0 : 44[7000] -> 29[46000] [receive] via NET/IB/0
+gpua013:395174:395174 [2] NCCL INFO cudaDriverVersion 12020
+gpua013:395174:395174 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.13<0>
+gpua013:395174:395174 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua013:395174:395239 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.13<0>
+gpua013:395174:395239 [2] NCCL INFO Using network IB
+gpua013:395174:395239 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua013:395174:395239 [2] NCCL INFO Trees [0] 7/-1/-1->6->5 [1] 7/-1/-1->6->5
+gpua013:395174:395239 [2] NCCL INFO Channel 00/0 : 6[85000] -> 7[c7000] via P2P/IPC/read
+gpua013:395174:395239 [2] NCCL INFO Channel 01/0 : 6[85000] -> 7[c7000] via P2P/IPC/read
+gpua013:395174:395239 [2] NCCL INFO Connected all rings
+gpua013:395174:395239 [2] NCCL INFO Channel 00/0 : 6[85000] -> 5[46000] via P2P/IPC/read
+gpua013:395174:395239 [2] NCCL INFO Channel 01/0 : 6[85000] -> 5[46000] via P2P/IPC/read
+gpua013:395174:395239 [2] NCCL INFO Connected all trees
+gpua024:349659:349659 [1] NCCL INFO cudaDriverVersion 12020
+gpua024:349659:349659 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.24<0>
+gpua024:349659:349659 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua024:349659:349748 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.24<0>
+gpua024:349659:349748 [1] NCCL INFO Using network IB
+gpua024:349659:349748 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua024:349659:349748 [1] NCCL INFO Trees [0] 18/8/-1->17->16 [1] 18/-1/-1->17->16
+gpua024:349659:349748 [1] NCCL INFO Channel 00/0 : 17[46000] -> 18[85000] via P2P/IPC/read
+gpua024:349659:349748 [1] NCCL INFO Channel 01/0 : 17[46000] -> 18[85000] via P2P/IPC/read
+gpua024:349659:349748 [1] NCCL INFO Connected all rings
+gpua024:349659:349748 [1] NCCL INFO Channel 00/0 : 8[7000] -> 17[46000] [receive] via NET/IB/0
+gpua024:349659:349748 [1] NCCL INFO Channel 00/0 : 17[46000] -> 8[7000] [send] via NET/IB/0
+gpua051:387978:387978 [2] NCCL INFO cudaDriverVersion 12020
+gpua051:387978:387978 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.51<0>
+gpua051:387978:387978 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua051:387978:388071 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.51<0>
+gpua051:387978:388071 [2] NCCL INFO Using network IB
+gpua051:387978:388071 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua051:387978:388071 [2] NCCL INFO Trees [0] 35/-1/-1->34->33 [1] 35/-1/-1->34->33
+gpua051:387978:388071 [2] NCCL INFO Channel 00/0 : 34[85000] -> 35[c7000] via P2P/IPC/read
+gpua051:387978:388071 [2] NCCL INFO Channel 01/0 : 34[85000] -> 35[c7000] via P2P/IPC/read
+gpua051:387978:388071 [2] NCCL INFO Connected all rings
+gpua051:387978:388071 [2] NCCL INFO Channel 00/0 : 34[85000] -> 33[46000] via P2P/IPC/read
+gpua051:387978:388071 [2] NCCL INFO Channel 01/0 : 34[85000] -> 33[46000] via P2P/IPC/read
+gpua079:2406229:2406229 [3] NCCL INFO cudaDriverVersion 12020
+gpua079:2406229:2406229 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0>
+gpua079:2406229:2406229 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua079:2406229:2406304 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.79<0>
+gpua079:2406229:2406304 [3] NCCL INFO Using network IB
+gpua079:2406229:2406304 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua079:2406229:2406304 [3] NCCL INFO Trees [0] -1/-1/-1->51->50 [1] -1/-1/-1->51->50
+gpua079:2406229:2406304 [3] NCCL INFO Channel 00/0 : 51[c7000] -> 52[7000] [send] via NET/IB/0
+gpua079:2406229:2406304 [3] NCCL INFO Channel 01/0 : 51[c7000] -> 52[7000] [send] via NET/IB/0
+gpua079:2406229:2406304 [3] NCCL INFO Connected all rings
+gpua079:2406229:2406304 [3] NCCL INFO Channel 00/0 : 51[c7000] -> 50[85000] via P2P/IPC/read
+gpua079:2406229:2406304 [3] NCCL INFO Channel 01/0 : 51[c7000] -> 50[85000] via P2P/IPC/read
+gpua058:102160:102160 [1] NCCL INFO cudaDriverVersion 12020
+gpua058:102160:102160 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.58<0>
+gpua058:102160:102160 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua058:102160:102252 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.58<0>
+gpua058:102160:102252 [1] NCCL INFO Using network IB
+gpua058:102160:102252 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua058:102160:102252 [1] NCCL INFO Trees [0] 46/-1/-1->45->44 [1] 46/52/-1->45->44
+gpua058:102160:102252 [1] NCCL INFO Channel 00/0 : 45[46000] -> 46[85000] via P2P/IPC/read
+gpua058:102160:102252 [1] NCCL INFO Channel 01/0 : 45[46000] -> 46[85000] via P2P/IPC/read
+gpua058:102160:102252 [1] NCCL INFO Connected all rings
+gpua058:102160:102252 [1] NCCL INFO Channel 01/0 : 45[46000] -> 52[7000] [send] via NET/IB/0
+gpua058:102160:102252 [1] NCCL INFO Channel 01/0 : 52[7000] -> 45[46000] [receive] via NET/IB/0
+gpua053:179587:179587 [3] NCCL INFO cudaDriverVersion 12020
+gpua053:179587:179587 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.53<0>
+gpua053:179587:179587 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua053:179587:179684 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.53<0>
+gpua053:179587:179684 [3] NCCL INFO Using network IB
+gpua053:179587:179684 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua053:179587:179684 [3] NCCL INFO Trees [0] -1/-1/-1->39->38 [1] -1/-1/-1->39->38
+gpua053:179587:179684 [3] NCCL INFO Channel 00/0 : 39[c7000] -> 40[7000] [send] via NET/IB/0
+gpua053:179587:179684 [3] NCCL INFO Channel 01/0 : 39[c7000] -> 40[7000] [send] via NET/IB/0
+gpua053:179587:179684 [3] NCCL INFO Connected all rings
+gpua053:179587:179684 [3] NCCL INFO Channel 00/0 : 39[c7000] -> 38[85000] via P2P/IPC/read
+gpua053:179587:179684 [3] NCCL INFO Channel 01/0 : 39[c7000] -> 38[85000] via P2P/IPC/read
+gpua055:2316455:2316455 [2] NCCL INFO cudaDriverVersion 12020
+gpua055:2316455:2316455 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0>
+gpua055:2316455:2316455 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua055:2316455:2316520 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.55<0>
+gpua055:2316455:2316520 [2] NCCL INFO Using network IB
+gpua055:2316455:2316520 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua055:2316455:2316520 [2] NCCL INFO Trees [0] 43/-1/-1->42->41 [1] 43/-1/-1->42->41
+gpua055:2316455:2316520 [2] NCCL INFO Channel 00/0 : 42[85000] -> 43[c7000] via P2P/IPC/read
+gpua055:2316455:2316520 [2] NCCL INFO Channel 01/0 : 42[85000] -> 43[c7000] via P2P/IPC/read
+gpua055:2316455:2316520 [2] NCCL INFO Connected all rings
+gpua055:2316455:2316520 [2] NCCL INFO Channel 00/0 : 42[85000] -> 41[46000] via P2P/IPC/read
+gpua055:2316455:2316520 [2] NCCL INFO Channel 01/0 : 42[85000] -> 41[46000] via P2P/IPC/read
+gpua083:196706:196706 [3] NCCL INFO cudaDriverVersion 12020
+gpua083:196706:196706 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.83<0>
+gpua083:196706:196706 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua083:196706:196773 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.83<0>
+gpua083:196706:196773 [3] NCCL INFO Using network IB
+gpua083:196706:196773 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua083:196706:196773 [3] NCCL INFO Trees [0] -1/-1/-1->59->58 [1] -1/-1/-1->59->58
+gpua083:196706:196773 [3] NCCL INFO Channel 00/0 : 59[c7000] -> 60[7000] [send] via NET/IB/0
+gpua083:196706:196773 [3] NCCL INFO Channel 01/0 : 59[c7000] -> 60[7000] [send] via NET/IB/0
+gpua083:196706:196773 [3] NCCL INFO Connected all rings
+gpua083:196706:196773 [3] NCCL INFO Channel 00/0 : 59[c7000] -> 58[85000] via P2P/IPC/read
+gpua083:196706:196773 [3] NCCL INFO Channel 01/0 : 59[c7000] -> 58[85000] via P2P/IPC/read
+gpua080:1427106:1427106 [3] NCCL INFO cudaDriverVersion 12020
+gpua080:1427106:1427106 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0>
+gpua080:1427106:1427106 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua080:1427106:1427167 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.80<0>
+gpua080:1427106:1427167 [3] NCCL INFO Using network IB
+gpua080:1427106:1427167 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua080:1427106:1427167 [3] NCCL INFO Trees [0] -1/-1/-1->55->54 [1] -1/-1/-1->55->54
+gpua080:1427106:1427167 [3] NCCL INFO Channel 00/0 : 55[c7000] -> 56[7000] [send] via NET/IB/0
+gpua080:1427106:1427167 [3] NCCL INFO Channel 01/0 : 55[c7000] -> 56[7000] [send] via NET/IB/0
+gpua080:1427106:1427167 [3] NCCL INFO Connected all rings
+gpua080:1427106:1427167 [3] NCCL INFO Channel 00/0 : 55[c7000] -> 54[85000] via P2P/IPC/read
+gpua080:1427106:1427167 [3] NCCL INFO Channel 01/0 : 55[c7000] -> 54[85000] via P2P/IPC/read
+gpua087:102966:102966 [2] NCCL INFO cudaDriverVersion 12020
+gpua087:102966:102966 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.87<0>
+gpua087:102966:102966 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua087:102966:103028 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.87<0>
+gpua087:102966:103028 [2] NCCL INFO Using network IB
+gpua087:102966:103028 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua087:102966:103028 [2] NCCL INFO Trees [0] 63/-1/-1->62->61 [1] 63/-1/-1->62->61
+gpua087:102966:103028 [2] NCCL INFO Channel 00/0 : 62[85000] -> 63[c7000] via P2P/IPC/read
+gpua087:102966:103028 [2] NCCL INFO Channel 01/0 : 62[85000] -> 63[c7000] via P2P/IPC/read
+gpua087:102966:103028 [2] NCCL INFO Connected all rings
+gpua087:102966:103028 [2] NCCL INFO Channel 00/0 : 62[85000] -> 61[46000] via P2P/IPC/read
+gpua087:102966:103028 [2] NCCL INFO Channel 01/0 : 62[85000] -> 61[46000] via P2P/IPC/read
+gpua017:2815426:2815498 [3] NCCL INFO Connected all trees
+gpua017:2815426:2815498 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua017:2815426:2815498 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua017:2815426:2815498 [3] NCCL INFO comm 0x55e3633b10e0 rank 15 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua036:985763:985825 [1] NCCL INFO Channel 00/0 : 29[46000] -> 28[7000] via P2P/IPC/read
+gpua036:985763:985825 [1] NCCL INFO Channel 01/0 : 29[46000] -> 28[7000] via P2P/IPC/read
+gpua036:985763:985825 [1] NCCL INFO Connected all trees
+gpua036:985763:985825 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua036:985763:985825 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua036:985763:985825 [1] NCCL INFO comm 0x5558a13b1a50 rank 29 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua013:395174:395239 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua013:395174:395239 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua013:395174:395239 [2] NCCL INFO comm 0x55a365b5aa60 rank 6 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua024:349659:349748 [1] NCCL INFO Channel 00/0 : 17[46000] -> 16[7000] via P2P/IPC/read
+gpua024:349659:349748 [1] NCCL INFO Channel 01/0 : 17[46000] -> 16[7000] via P2P/IPC/read
+gpua024:349659:349748 [1] NCCL INFO Connected all trees
+gpua024:349659:349748 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua024:349659:349748 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua024:349659:349748 [1] NCCL INFO comm 0x5602f3ebe4e0 rank 17 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua051:387978:388071 [2] NCCL INFO Connected all trees
+gpua051:387978:388071 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua051:387978:388071 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua051:387978:388071 [2] NCCL INFO comm 0x55b2b8bf5dc0 rank 34 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua079:2406229:2406304 [3] NCCL INFO Connected all trees
+gpua079:2406229:2406304 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua079:2406229:2406304 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua079:2406229:2406304 [3] NCCL INFO comm 0x56029a9c6950 rank 51 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua058:102160:102252 [1] NCCL INFO Channel 00/0 : 45[46000] -> 44[7000] via P2P/IPC/read
+gpua058:102160:102252 [1] NCCL INFO Channel 01/0 : 45[46000] -> 44[7000] via P2P/IPC/read
+gpua058:102160:102252 [1] NCCL INFO Connected all trees
+gpua058:102160:102252 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua058:102160:102252 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua058:102160:102252 [1] NCCL INFO comm 0x5608e2f758d0 rank 45 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua053:179587:179684 [3] NCCL INFO Connected all trees
+gpua053:179587:179684 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua053:179587:179684 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua053:179587:179684 [3] NCCL INFO comm 0x564395cc1f10 rank 39 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua055:2316455:2316520 [2] NCCL INFO Connected all trees
+gpua055:2316455:2316520 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua055:2316455:2316520 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua055:2316455:2316520 [2] NCCL INFO comm 0x5568d932cac0 rank 42 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua083:196706:196773 [3] NCCL INFO Connected all trees
+gpua083:196706:196773 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua083:196706:196773 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua083:196706:196773 [3] NCCL INFO comm 0x55e11249eda0 rank 59 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua080:1427106:1427167 [3] NCCL INFO Connected all trees
+gpua080:1427106:1427167 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua080:1427106:1427167 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua080:1427106:1427167 [3] NCCL INFO comm 0x555942f648e0 rank 55 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua080:1427103:1427103 [0] NCCL INFO cudaDriverVersion 12020
+gpua080:1427103:1427103 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0>
+gpua080:1427103:1427103 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua080:1427103:1427168 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.80<0>
+gpua080:1427103:1427168 [0] NCCL INFO Using network IB
+gpua080:1427103:1427168 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua080:1427103:1427168 [0] NCCL INFO Trees [0] 53/-1/-1->52->57 [1] 53/48/-1->52->45
+gpua087:102966:103028 [2] NCCL INFO Connected all trees
+gpua087:102966:103028 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua087:102966:103028 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua087:102966:103028 [2] NCCL INFO comm 0x555c44b6efc0 rank 62 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua017:2815423:2815423 [0] NCCL INFO cudaDriverVersion 12020
+gpua017:2815423:2815423 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.17<0>
+gpua017:2815423:2815423 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua017:2815423:2815499 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.17<0>
+gpua017:2815423:2815499 [0] NCCL INFO Using network IB
+gpua017:2815423:2815499 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua017:2815423:2815499 [0] NCCL INFO Trees [0] 13/-1/-1->12->8 [1] 13/4/-1->12->28
+gpua017:2815423:2815499 [0] NCCL INFO Channel 00/0 : 11[c7000] -> 12[7000] [receive] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Channel 01/0 : 11[c7000] -> 12[7000] [receive] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Channel 00/0 : 12[7000] -> 13[46000] via P2P/IPC/read
+gpua017:2815423:2815499 [0] NCCL INFO Channel 01/0 : 12[7000] -> 13[46000] via P2P/IPC/read
+gpua017:2815423:2815499 [0] NCCL INFO Connected all rings
+gpua036:985765:985765 [3] NCCL INFO cudaDriverVersion 12020
+gpua036:985765:985765 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.36<0>
+gpua036:985765:985765 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua036:985765:985827 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.36<0>
+gpua036:985765:985827 [3] NCCL INFO Using network IB
+gpua036:985765:985827 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua036:985765:985827 [3] NCCL INFO Trees [0] -1/-1/-1->31->30 [1] -1/-1/-1->31->30
+gpua036:985765:985827 [3] NCCL INFO Channel 00/0 : 31[c7000] -> 32[7000] [send] via NET/IB/0
+gpua036:985765:985827 [3] NCCL INFO Channel 01/0 : 31[c7000] -> 32[7000] [send] via NET/IB/0
+gpua036:985765:985827 [3] NCCL INFO Connected all rings
+gpua036:985765:985827 [3] NCCL INFO Channel 00/0 : 31[c7000] -> 30[85000] via P2P/IPC/read
+gpua036:985765:985827 [3] NCCL INFO Channel 01/0 : 31[c7000] -> 30[85000] via P2P/IPC/read
+gpua013:395175:395175 [3] NCCL INFO cudaDriverVersion 12020
+gpua013:395175:395175 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.13<0>
+gpua013:395175:395175 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua013:395175:395241 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.13<0>
+gpua013:395175:395241 [3] NCCL INFO Using network IB
+gpua013:395175:395241 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua013:395175:395241 [3] NCCL INFO Trees [0] -1/-1/-1->7->6 [1] -1/-1/-1->7->6
+gpua013:395175:395241 [3] NCCL INFO Channel 00/0 : 7[c7000] -> 8[7000] [send] via NET/IB/0
+gpua013:395175:395241 [3] NCCL INFO Channel 01/0 : 7[c7000] -> 8[7000] [send] via NET/IB/0
+gpua013:395175:395241 [3] NCCL INFO Connected all rings
+gpua013:395175:395241 [3] NCCL INFO Channel 00/0 : 7[c7000] -> 6[85000] via P2P/IPC/read
+gpua013:395175:395241 [3] NCCL INFO Channel 01/0 : 7[c7000] -> 6[85000] via P2P/IPC/read
+gpua013:395175:395241 [3] NCCL INFO Connected all trees
+gpua051:387979:387979 [3] NCCL INFO cudaDriverVersion 12020
+gpua051:387979:387979 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.51<0>
+gpua051:387979:387979 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua051:387979:388072 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.51<0>
+gpua051:387979:388072 [3] NCCL INFO Using network IB
+gpua051:387979:388072 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua051:387979:388072 [3] NCCL INFO Trees [0] -1/-1/-1->35->34 [1] -1/-1/-1->35->34
+gpua051:387979:388072 [3] NCCL INFO Channel 00/0 : 35[c7000] -> 36[7000] [send] via NET/IB/0
+gpua051:387979:388072 [3] NCCL INFO Channel 01/0 : 35[c7000] -> 36[7000] [send] via NET/IB/0
+gpua051:387979:388072 [3] NCCL INFO Connected all rings
+gpua051:387979:388072 [3] NCCL INFO Channel 00/0 : 35[c7000] -> 34[85000] via P2P/IPC/read
+gpua051:387979:388072 [3] NCCL INFO Channel 01/0 : 35[c7000] -> 34[85000] via P2P/IPC/read
+gpua079:2406228:2406228 [2] NCCL INFO cudaDriverVersion 12020
+gpua079:2406228:2406228 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0>
+gpua079:2406228:2406228 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua079:2406228:2406306 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.79<0>
+gpua079:2406228:2406306 [2] NCCL INFO Using network IB
+gpua079:2406228:2406306 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua079:2406228:2406306 [2] NCCL INFO Trees [0] 51/-1/-1->50->49 [1] 51/-1/-1->50->49
+gpua079:2406228:2406306 [2] NCCL INFO Channel 00/0 : 50[85000] -> 51[c7000] via P2P/IPC/read
+gpua079:2406228:2406306 [2] NCCL INFO Channel 01/0 : 50[85000] -> 51[c7000] via P2P/IPC/read
+gpua079:2406228:2406306 [2] NCCL INFO Connected all rings
+gpua079:2406228:2406306 [2] NCCL INFO Channel 00/0 : 50[85000] -> 49[46000] via P2P/IPC/read
+gpua079:2406228:2406306 [2] NCCL INFO Channel 01/0 : 50[85000] -> 49[46000] via P2P/IPC/read
+gpua058:102162:102162 [3] NCCL INFO cudaDriverVersion 12020
+gpua058:102162:102162 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.58<0>
+gpua058:102162:102162 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua058:102162:102254 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.58<0>
+gpua058:102162:102254 [3] NCCL INFO Using network IB
+gpua058:102162:102254 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua058:102162:102254 [3] NCCL INFO Trees [0] -1/-1/-1->47->46 [1] -1/-1/-1->47->46
+gpua058:102162:102254 [3] NCCL INFO Channel 00/0 : 47[c7000] -> 48[7000] [send] via NET/IB/0
+gpua058:102162:102254 [3] NCCL INFO Channel 01/0 : 47[c7000] -> 48[7000] [send] via NET/IB/0
+gpua058:102162:102254 [3] NCCL INFO Connected all rings
+gpua058:102162:102254 [3] NCCL INFO Channel 00/0 : 47[c7000] -> 46[85000] via P2P/IPC/read
+gpua058:102162:102254 [3] NCCL INFO Channel 01/0 : 47[c7000] -> 46[85000] via P2P/IPC/read
+gpua053:179586:179586 [2] NCCL INFO cudaDriverVersion 12020
+gpua053:179586:179586 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.53<0>
+gpua053:179586:179586 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua053:179586:179683 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.53<0>
+gpua053:179586:179683 [2] NCCL INFO Using network IB
+gpua053:179586:179683 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua053:179586:179683 [2] NCCL INFO Trees [0] 39/-1/-1->38->37 [1] 39/-1/-1->38->37
+gpua053:179586:179683 [2] NCCL INFO Channel 00/0 : 38[85000] -> 39[c7000] via P2P/IPC/read
+gpua053:179586:179683 [2] NCCL INFO Channel 01/0 : 38[85000] -> 39[c7000] via P2P/IPC/read
+gpua053:179586:179683 [2] NCCL INFO Connected all rings
+gpua053:179586:179683 [2] NCCL INFO Channel 00/0 : 38[85000] -> 37[46000] via P2P/IPC/read
+gpua053:179586:179683 [2] NCCL INFO Channel 01/0 : 38[85000] -> 37[46000] via P2P/IPC/read
+gpua055:2316456:2316456 [3] NCCL INFO cudaDriverVersion 12020
+gpua055:2316456:2316456 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0>
+gpua055:2316456:2316456 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua055:2316456:2316517 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.55<0>
+gpua055:2316456:2316517 [3] NCCL INFO Using network IB
+gpua055:2316456:2316517 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua055:2316456:2316517 [3] NCCL INFO Trees [0] -1/-1/-1->43->42 [1] -1/-1/-1->43->42
+gpua055:2316456:2316517 [3] NCCL INFO Channel 00/0 : 43[c7000] -> 44[7000] [send] via NET/IB/0
+gpua055:2316456:2316517 [3] NCCL INFO Channel 01/0 : 43[c7000] -> 44[7000] [send] via NET/IB/0
+gpua055:2316456:2316517 [3] NCCL INFO Connected all rings
+gpua055:2316456:2316517 [3] NCCL INFO Channel 00/0 : 43[c7000] -> 42[85000] via P2P/IPC/read
+gpua055:2316456:2316517 [3] NCCL INFO Channel 01/0 : 43[c7000] -> 42[85000] via P2P/IPC/read
+gpua083:196705:196705 [2] NCCL INFO cudaDriverVersion 12020
+gpua083:196705:196705 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.83<0>
+gpua083:196705:196705 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua083:196705:196775 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.83<0>
+gpua083:196705:196775 [2] NCCL INFO Using network IB
+gpua083:196705:196775 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua083:196705:196775 [2] NCCL INFO Trees [0] 59/-1/-1->58->57 [1] 59/-1/-1->58->57
+gpua083:196705:196775 [2] NCCL INFO Channel 00/0 : 58[85000] -> 59[c7000] via P2P/IPC/read
+gpua083:196705:196775 [2] NCCL INFO Channel 01/0 : 58[85000] -> 59[c7000] via P2P/IPC/read
+gpua083:196705:196775 [2] NCCL INFO Connected all rings
+gpua083:196705:196775 [2] NCCL INFO Channel 00/0 : 58[85000] -> 57[46000] via P2P/IPC/read
+gpua083:196705:196775 [2] NCCL INFO Channel 01/0 : 58[85000] -> 57[46000] via P2P/IPC/read
+gpua080:1427103:1427168 [0] NCCL INFO Channel 00/0 : 51[c7000] -> 52[7000] [receive] via NET/IB/0
+gpua080:1427103:1427168 [0] NCCL INFO Channel 01/0 : 51[c7000] -> 52[7000] [receive] via NET/IB/0
+gpua080:1427103:1427168 [0] NCCL INFO Channel 00/0 : 52[7000] -> 53[46000] via P2P/IPC/read
+gpua080:1427103:1427168 [0] NCCL INFO Channel 01/0 : 52[7000] -> 53[46000] via P2P/IPC/read
+gpua080:1427103:1427168 [0] NCCL INFO Connected all rings
+gpua080:1427103:1427168 [0] NCCL INFO Channel 01/0 : 48[7000] -> 52[7000] [receive] via NET/IB/0
+gpua080:1427103:1427168 [0] NCCL INFO Channel 00/0 : 52[7000] -> 57[46000] [send] via NET/IB/0
+gpua080:1427103:1427168 [0] NCCL INFO Channel 01/0 : 45[46000] -> 52[7000] [receive] via NET/IB/0
+gpua080:1427103:1427168 [0] NCCL INFO Channel 01/0 : 52[7000] -> 45[46000] [send] via NET/IB/0
+gpua080:1427103:1427168 [0] NCCL INFO Channel 00/0 : 57[46000] -> 52[7000] [receive] via NET/IB/0
+gpua080:1427103:1427168 [0] NCCL INFO Channel 01/0 : 52[7000] -> 48[7000] [send] via NET/IB/0
+gpua087:102964:102964 [0] NCCL INFO cudaDriverVersion 12020
+gpua087:102964:102964 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.87<0>
+gpua087:102964:102964 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua087:102964:103029 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.87<0>
+gpua087:102964:103029 [0] NCCL INFO Using network IB
+gpua087:102964:103029 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua087:102964:103029 [0] NCCL INFO Trees [0] 61/-1/-1->60->56 [1] 61/28/-1->60->-1
+gpua087:102964:103029 [0] NCCL INFO Channel 00/0 : 59[c7000] -> 60[7000] [receive] via NET/IB/0
+gpua087:102964:103029 [0] NCCL INFO Channel 01/0 : 59[c7000] -> 60[7000] [receive] via NET/IB/0
+gpua087:102964:103029 [0] NCCL INFO Channel 00/0 : 60[7000] -> 61[46000] via P2P/IPC/read
+gpua087:102964:103029 [0] NCCL INFO Channel 01/0 : 60[7000] -> 61[46000] via P2P/IPC/read
+gpua087:102964:103029 [0] NCCL INFO Connected all rings
+gpua017:2815423:2815499 [0] NCCL INFO Channel 00/0 : 8[7000] -> 12[7000] [receive] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Channel 01/0 : 4[7000] -> 12[7000] [receive] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Channel 01/0 : 12[7000] -> 28[7000] [send] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Channel 01/0 : 28[7000] -> 12[7000] [receive] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Channel 01/0 : 12[7000] -> 4[7000] [send] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Channel 00/0 : 12[7000] -> 8[7000] [send] via NET/IB/0
+gpua017:2815423:2815499 [0] NCCL INFO Connected all trees
+gpua017:2815423:2815499 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua017:2815423:2815499 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua017:2815423:2815499 [0] NCCL INFO comm 0x558c31a2ef20 rank 12 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua036:985765:985827 [3] NCCL INFO Connected all trees
+gpua036:985765:985827 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua036:985765:985827 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua036:985765:985827 [3] NCCL INFO comm 0x560c55567fe0 rank 31 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua013:395175:395241 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua013:395175:395241 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua013:395175:395241 [3] NCCL INFO comm 0x55debbd55590 rank 7 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua051:387979:388072 [3] NCCL INFO Connected all trees
+gpua051:387979:388072 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua051:387979:388072 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua051:387979:388072 [3] NCCL INFO comm 0x55dcff877f00 rank 35 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua079:2406228:2406306 [2] NCCL INFO Connected all trees
+gpua079:2406228:2406306 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua079:2406228:2406306 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua079:2406228:2406306 [2] NCCL INFO comm 0x55aebc056f20 rank 50 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua079:2406226:2406226 [0] NCCL INFO cudaDriverVersion 12020
+gpua079:2406226:2406226 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0>
+gpua079:2406226:2406226 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua079:2406226:2406303 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.79<0>
+gpua079:2406226:2406303 [0] NCCL INFO Using network IB
+gpua079:2406226:2406303 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua079:2406226:2406303 [0] NCCL INFO Trees [0] 49/56/-1->48->32 [1] 49/-1/-1->48->52
+gpua058:102162:102254 [3] NCCL INFO Connected all trees
+gpua058:102162:102254 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua058:102162:102254 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua058:102162:102254 [3] NCCL INFO comm 0x55a590e20350 rank 47 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua058:102159:102159 [0] NCCL INFO cudaDriverVersion 12020
+gpua058:102159:102159 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.58<0>
+gpua058:102159:102159 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua058:102159:102253 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.58<0>
+gpua058:102159:102253 [0] NCCL INFO Using network IB
+gpua058:102159:102253 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua058:102159:102253 [0] NCCL INFO Trees [0] 45/-1/-1->44->40 [1] 45/36/-1->44->29
+gpua058:102159:102253 [0] NCCL INFO Channel 00/0 : 43[c7000] -> 44[7000] [receive] via NET/IB/0
+gpua053:179586:179683 [2] NCCL INFO Connected all trees
+gpua053:179586:179683 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua053:179586:179683 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua053:179586:179683 [2] NCCL INFO comm 0x559e082b31e0 rank 38 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua055:2316456:2316517 [3] NCCL INFO Connected all trees
+gpua055:2316456:2316517 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua055:2316456:2316517 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua055:2316456:2316517 [3] NCCL INFO comm 0x563e974bd250 rank 43 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua083:196705:196775 [2] NCCL INFO Connected all trees
+gpua083:196705:196775 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua083:196705:196775 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua083:196705:196775 [2] NCCL INFO comm 0x564924364b60 rank 58 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua080:1427103:1427168 [0] NCCL INFO Connected all trees
+gpua080:1427103:1427168 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua080:1427103:1427168 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua080:1427103:1427168 [0] NCCL INFO comm 0x5603ba03ce60 rank 52 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua087:102964:103029 [0] NCCL INFO Channel 00/0 : 56[7000] -> 60[7000] [receive] via NET/IB/0
+gpua087:102964:103029 [0] NCCL INFO Channel 01/0 : 28[7000] -> 60[7000] [receive] via NET/IB/0
+gpua087:102964:103029 [0] NCCL INFO Channel 01/0 : 60[7000] -> 28[7000] [send] via NET/IB/0
+gpua087:102964:103029 [0] NCCL INFO Channel 00/0 : 60[7000] -> 56[7000] [send] via NET/IB/0
+gpua087:102964:103029 [0] NCCL INFO Connected all trees
+gpua087:102964:103029 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua087:102964:103029 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua087:102964:103029 [0] NCCL INFO comm 0x5565fd0a0580 rank 60 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua017:2815424:2815424 [1] NCCL INFO cudaDriverVersion 12020
+gpua017:2815424:2815424 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.17<0>
+gpua017:2815424:2815424 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua017:2815424:2815497 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.17<0>
+gpua017:2815424:2815497 [1] NCCL INFO Using network IB
+gpua017:2815424:2815497 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua017:2815424:2815497 [1] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/20/-1->13->12
+gpua017:2815424:2815497 [1] NCCL INFO Channel 00/0 : 13[46000] -> 14[85000] via P2P/IPC/read
+gpua017:2815424:2815497 [1] NCCL INFO Channel 01/0 : 13[46000] -> 14[85000] via P2P/IPC/read
+gpua017:2815424:2815497 [1] NCCL INFO Connected all rings
+gpua017:2815424:2815497 [1] NCCL INFO Channel 01/0 : 13[46000] -> 20[7000] [send] via NET/IB/0
+gpua017:2815424:2815497 [1] NCCL INFO Channel 01/0 : 20[7000] -> 13[46000] [receive] via NET/IB/0
+gpua036:985762:985762 [0] NCCL INFO cudaDriverVersion 12020
+gpua036:985762:985762 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.36<0>
+gpua036:985762:985762 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua036:985762:985824 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.36<0>
+gpua036:985762:985824 [0] NCCL INFO Using network IB
+gpua036:985762:985824 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua036:985762:985824 [0] NCCL INFO Trees [0] 29/-1/-1->28->24 [1] 29/12/-1->28->60
+gpua036:985762:985824 [0] NCCL INFO Channel 00/0 : 27[c7000] -> 28[7000] [receive] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Channel 01/0 : 27[c7000] -> 28[7000] [receive] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Channel 00/0 : 28[7000] -> 29[46000] via P2P/IPC/read
+gpua036:985762:985824 [0] NCCL INFO Channel 01/0 : 28[7000] -> 29[46000] via P2P/IPC/read
+gpua036:985762:985824 [0] NCCL INFO Connected all rings
+gpua013:395173:395173 [1] NCCL INFO cudaDriverVersion 12020
+gpua013:395173:395173 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.13<0>
+gpua013:395173:395173 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua013:395173:395240 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.13<0>
+gpua013:395173:395240 [1] NCCL INFO Using network IB
+gpua013:395173:395240 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua013:395173:395240 [1] NCCL INFO Trees [0] 6/-1/-1->5->4 [1] 6/8/-1->5->4
+gpua013:395173:395240 [1] NCCL INFO Channel 00/0 : 5[46000] -> 6[85000] via P2P/IPC/read
+gpua013:395173:395240 [1] NCCL INFO Channel 01/0 : 5[46000] -> 6[85000] via P2P/IPC/read
+gpua013:395173:395240 [1] NCCL INFO Connected all rings
+gpua013:395173:395240 [1] NCCL INFO Channel 01/0 : 5[46000] -> 8[7000] [send] via NET/IB/0
+gpua013:395173:395240 [1] NCCL INFO Channel 01/0 : 8[7000] -> 5[46000] [receive] via NET/IB/0
+gpua051:387976:387976 [0] NCCL INFO cudaDriverVersion 12020
+gpua051:387976:387976 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.51<0>
+gpua051:387976:387976 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua051:387976:388073 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.51<0>
+gpua051:387976:388073 [0] NCCL INFO Using network IB
+gpua051:387976:388073 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua051:387976:388073 [0] NCCL INFO Trees [0] 33/48/-1->32->0 [1] 33/-1/-1->32->36
+gpua051:387976:388073 [0] NCCL INFO Channel 00/0 : 31[c7000] -> 32[7000] [receive] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Channel 01/0 : 31[c7000] -> 32[7000] [receive] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Channel 00/0 : 32[7000] -> 33[46000] via P2P/IPC/read
+gpua051:387976:388073 [0] NCCL INFO Channel 01/0 : 32[7000] -> 33[46000] via P2P/IPC/read
+gpua051:387976:388073 [0] NCCL INFO Connected all rings
+gpua079:2406226:2406303 [0] NCCL INFO Channel 00/0 : 47[c7000] -> 48[7000] [receive] via NET/IB/0
+gpua079:2406226:2406303 [0] NCCL INFO Channel 01/0 : 47[c7000] -> 48[7000] [receive] via NET/IB/0
+gpua079:2406226:2406303 [0] NCCL INFO Channel 00/0 : 48[7000] -> 49[46000] via P2P/IPC/read
+gpua079:2406226:2406303 [0] NCCL INFO Channel 01/0 : 48[7000] -> 49[46000] via P2P/IPC/read
+gpua079:2406226:2406303 [0] NCCL INFO Connected all rings
+gpua079:2406226:2406303 [0] NCCL INFO Channel 01/0 : 48[7000] -> 52[7000] [send] via NET/IB/0
+gpua079:2406226:2406303 [0] NCCL INFO Channel 00/0 : 48[7000] -> 56[7000] [send] via NET/IB/0
+gpua079:2406226:2406303 [0] NCCL INFO Channel 00/0 : 32[7000] -> 48[7000] [receive] via NET/IB/0
+gpua079:2406226:2406303 [0] NCCL INFO Channel 00/0 : 48[7000] -> 32[7000] [send] via NET/IB/0
+gpua079:2406226:2406303 [0] NCCL INFO Channel 00/0 : 56[7000] -> 48[7000] [receive] via NET/IB/0
+gpua079:2406226:2406303 [0] NCCL INFO Channel 01/0 : 52[7000] -> 48[7000] [receive] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Channel 01/0 : 43[c7000] -> 44[7000] [receive] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Channel 00/0 : 44[7000] -> 45[46000] via P2P/IPC/read
+gpua058:102159:102253 [0] NCCL INFO Channel 01/0 : 44[7000] -> 45[46000] via P2P/IPC/read
+gpua058:102159:102253 [0] NCCL INFO Connected all rings
+gpua058:102159:102253 [0] NCCL INFO Channel 00/0 : 40[7000] -> 44[7000] [receive] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Channel 01/0 : 36[7000] -> 44[7000] [receive] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Channel 01/0 : 29[46000] -> 44[7000] [receive] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Channel 01/0 : 44[7000] -> 29[46000] [send] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Channel 01/0 : 44[7000] -> 36[7000] [send] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Channel 00/0 : 44[7000] -> 40[7000] [send] via NET/IB/0
+gpua058:102159:102253 [0] NCCL INFO Connected all trees
+gpua053:179585:179585 [1] NCCL INFO cudaDriverVersion 12020
+gpua053:179585:179585 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.53<0>
+gpua053:179585:179585 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua053:179585:179686 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.53<0>
+gpua053:179585:179686 [1] NCCL INFO Using network IB
+gpua053:179585:179686 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua053:179585:179686 [1] NCCL INFO Trees [0] 38/-1/-1->37->36 [1] 38/40/-1->37->36
+gpua053:179585:179686 [1] NCCL INFO Channel 00/0 : 37[46000] -> 38[85000] via P2P/IPC/read
+gpua053:179585:179686 [1] NCCL INFO Channel 01/0 : 37[46000] -> 38[85000] via P2P/IPC/read
+gpua053:179585:179686 [1] NCCL INFO Connected all rings
+gpua053:179585:179686 [1] NCCL INFO Channel 01/0 : 37[46000] -> 40[7000] [send] via NET/IB/0
+gpua053:179585:179686 [1] NCCL INFO Channel 01/0 : 40[7000] -> 37[46000] [receive] via NET/IB/0
+gpua055:2316453:2316453 [0] NCCL INFO cudaDriverVersion 12020
+gpua055:2316453:2316453 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0>
+gpua055:2316453:2316453 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua055:2316453:2316518 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.55<0>
+gpua055:2316453:2316518 [0] NCCL INFO Using network IB
+gpua055:2316453:2316518 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua055:2316453:2316518 [0] NCCL INFO Trees [0] 41/44/-1->40->49 [1] 41/-1/-1->40->37
+gpua055:2316453:2316518 [0] NCCL INFO Channel 00/0 : 39[c7000] -> 40[7000] [receive] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Channel 01/0 : 39[c7000] -> 40[7000] [receive] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Channel 00/0 : 40[7000] -> 41[46000] via P2P/IPC/read
+gpua055:2316453:2316518 [0] NCCL INFO Channel 01/0 : 40[7000] -> 41[46000] via P2P/IPC/read
+gpua055:2316453:2316518 [0] NCCL INFO Connected all rings
+gpua083:196703:196703 [0] NCCL INFO cudaDriverVersion 12020
+gpua083:196703:196703 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.83<0>
+gpua083:196703:196703 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua083:196703:196776 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.83<0>
+gpua083:196703:196776 [0] NCCL INFO Using network IB
+gpua083:196703:196776 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua083:196703:196776 [0] NCCL INFO Trees [0] 57/60/-1->56->48 [1] 57/-1/-1->56->53
+gpua083:196703:196776 [0] NCCL INFO Channel 00/0 : 55[c7000] -> 56[7000] [receive] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Channel 01/0 : 55[c7000] -> 56[7000] [receive] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Channel 00/0 : 56[7000] -> 57[46000] via P2P/IPC/read
+gpua083:196703:196776 [0] NCCL INFO Channel 01/0 : 56[7000] -> 57[46000] via P2P/IPC/read
+gpua083:196703:196776 [0] NCCL INFO Connected all rings
+gpua080:1427104:1427104 [1] NCCL INFO cudaDriverVersion 12020
+gpua080:1427104:1427104 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0>
+gpua080:1427104:1427104 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua080:1427104:1427169 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.80<0>
+gpua080:1427104:1427169 [1] NCCL INFO Using network IB
+gpua080:1427104:1427169 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua080:1427104:1427169 [1] NCCL INFO Trees [0] 54/-1/-1->53->52 [1] 54/56/-1->53->52
+gpua080:1427104:1427169 [1] NCCL INFO Channel 00/0 : 53[46000] -> 54[85000] via P2P/IPC/read
+gpua080:1427104:1427169 [1] NCCL INFO Channel 01/0 : 53[46000] -> 54[85000] via P2P/IPC/read
+gpua080:1427104:1427169 [1] NCCL INFO Connected all rings
+gpua080:1427104:1427169 [1] NCCL INFO Channel 01/0 : 53[46000] -> 56[7000] [send] via NET/IB/0
+gpua080:1427104:1427169 [1] NCCL INFO Channel 01/0 : 56[7000] -> 53[46000] [receive] via NET/IB/0
+gpua087:102967:102967 [3] NCCL INFO cudaDriverVersion 12020
+gpua087:102967:102967 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.87<0>
+gpua087:102967:102967 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua087:102967:103026 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.87<0>
+gpua087:102967:103026 [3] NCCL INFO Using network IB
+gpua087:102967:103026 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua087:102967:103026 [3] NCCL INFO Trees [0] -1/-1/-1->63->62 [1] -1/-1/-1->63->62
+gpua087:102967:103026 [3] NCCL INFO Channel 00/0 : 63[c7000] -> 0[7000] [send] via NET/IB/0
+gpua087:102967:103026 [3] NCCL INFO Channel 01/0 : 63[c7000] -> 0[7000] [send] via NET/IB/0
+gpua087:102967:103026 [3] NCCL INFO Connected all rings
+gpua087:102967:103026 [3] NCCL INFO Channel 00/0 : 63[c7000] -> 62[85000] via P2P/IPC/read
+gpua087:102967:103026 [3] NCCL INFO Channel 01/0 : 63[c7000] -> 62[85000] via P2P/IPC/read
+gpua017:2815424:2815497 [1] NCCL INFO Channel 00/0 : 13[46000] -> 12[7000] via P2P/IPC/read
+gpua017:2815424:2815497 [1] NCCL INFO Channel 01/0 : 13[46000] -> 12[7000] via P2P/IPC/read
+gpua017:2815424:2815497 [1] NCCL INFO Connected all trees
+gpua017:2815424:2815497 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua017:2815424:2815497 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua017:2815424:2815497 [1] NCCL INFO comm 0x555efacf0500 rank 13 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua036:985762:985824 [0] NCCL INFO Channel 00/0 : 24[7000] -> 28[7000] [receive] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Channel 01/0 : 12[7000] -> 28[7000] [receive] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Channel 01/0 : 60[7000] -> 28[7000] [receive] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Channel 01/0 : 28[7000] -> 60[7000] [send] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Channel 01/0 : 28[7000] -> 12[7000] [send] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Channel 00/0 : 28[7000] -> 24[7000] [send] via NET/IB/0
+gpua036:985762:985824 [0] NCCL INFO Connected all trees
+gpua036:985762:985824 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua036:985762:985824 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua036:985762:985824 [0] NCCL INFO comm 0x55c2a75b5130 rank 28 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua013:395173:395240 [1] NCCL INFO Channel 00/0 : 5[46000] -> 4[7000] via P2P/IPC/read
+gpua013:395173:395240 [1] NCCL INFO Channel 01/0 : 5[46000] -> 4[7000] via P2P/IPC/read
+gpua013:395173:395240 [1] NCCL INFO Connected all trees
+gpua013:395173:395240 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua013:395173:395240 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua013:395173:395240 [1] NCCL INFO comm 0x55c396d17fe0 rank 5 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua051:387976:388073 [0] NCCL INFO Channel 01/0 : 32[7000] -> 36[7000] [send] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Channel 00/0 : 32[7000] -> 48[7000] [send] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Channel 00/0 : 0[7000] -> 32[7000] [receive] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Channel 00/0 : 32[7000] -> 0[7000] [send] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Channel 00/0 : 48[7000] -> 32[7000] [receive] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Channel 01/0 : 36[7000] -> 32[7000] [receive] via NET/IB/0
+gpua051:387976:388073 [0] NCCL INFO Connected all trees
+gpua051:387976:388073 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua051:387976:388073 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua051:387976:388073 [0] NCCL INFO comm 0x555713d9a190 rank 32 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua079:2406226:2406303 [0] NCCL INFO Connected all trees
+gpua079:2406226:2406303 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua079:2406226:2406303 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua079:2406226:2406303 [0] NCCL INFO comm 0x55cf75509b10 rank 48 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua058:102159:102253 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua058:102159:102253 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua058:102159:102253 [0] NCCL INFO comm 0x5642f3bcd790 rank 44 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua053:179585:179686 [1] NCCL INFO Channel 00/0 : 37[46000] -> 36[7000] via P2P/IPC/read
+gpua053:179585:179686 [1] NCCL INFO Channel 01/0 : 37[46000] -> 36[7000] via P2P/IPC/read
+gpua053:179585:179686 [1] NCCL INFO Connected all trees
+gpua053:179585:179686 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua053:179585:179686 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua053:179585:179686 [1] NCCL INFO comm 0x55baf655d6f0 rank 37 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua055:2316453:2316518 [0] NCCL INFO Channel 01/0 : 37[46000] -> 40[7000] [receive] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Channel 00/0 : 40[7000] -> 44[7000] [send] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Channel 00/0 : 40[7000] -> 49[46000] [send] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Channel 00/0 : 49[46000] -> 40[7000] [receive] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Channel 00/0 : 44[7000] -> 40[7000] [receive] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Channel 01/0 : 40[7000] -> 37[46000] [send] via NET/IB/0
+gpua055:2316453:2316518 [0] NCCL INFO Connected all trees
+gpua055:2316453:2316518 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua055:2316453:2316518 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua055:2316453:2316518 [0] NCCL INFO comm 0x56406b6e9aa0 rank 40 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua083:196703:196776 [0] NCCL INFO Channel 01/0 : 53[46000] -> 56[7000] [receive] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Channel 00/0 : 56[7000] -> 60[7000] [send] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Channel 00/0 : 48[7000] -> 56[7000] [receive] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Channel 00/0 : 56[7000] -> 48[7000] [send] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Channel 00/0 : 60[7000] -> 56[7000] [receive] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Channel 01/0 : 56[7000] -> 53[46000] [send] via NET/IB/0
+gpua083:196703:196776 [0] NCCL INFO Connected all trees
+gpua083:196703:196776 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua083:196703:196776 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua083:196703:196776 [0] NCCL INFO comm 0x55f08d116e60 rank 56 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua080:1427104:1427169 [1] NCCL INFO Channel 00/0 : 53[46000] -> 52[7000] via P2P/IPC/read
+gpua080:1427104:1427169 [1] NCCL INFO Channel 01/0 : 53[46000] -> 52[7000] via P2P/IPC/read
+gpua080:1427104:1427169 [1] NCCL INFO Connected all trees
+gpua080:1427104:1427169 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua080:1427104:1427169 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua080:1427104:1427169 [1] NCCL INFO comm 0x55604e9d43e0 rank 53 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua087:102967:103026 [3] NCCL INFO Connected all trees
+gpua087:102967:103026 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua087:102967:103026 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua087:102967:103026 [3] NCCL INFO comm 0x555c31f87ca0 rank 63 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua036:985764:985764 [2] NCCL INFO cudaDriverVersion 12020
+gpua036:985764:985764 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.36<0>
+gpua036:985764:985764 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua036:985764:985826 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.36<0>
+gpua036:985764:985826 [2] NCCL INFO Using network IB
+gpua036:985764:985826 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua036:985764:985826 [2] NCCL INFO Trees [0] 31/-1/-1->30->29 [1] 31/-1/-1->30->29
+gpua036:985764:985826 [2] NCCL INFO Channel 00/0 : 30[85000] -> 31[c7000] via P2P/IPC/read
+gpua036:985764:985826 [2] NCCL INFO Channel 01/0 : 30[85000] -> 31[c7000] via P2P/IPC/read
+gpua036:985764:985826 [2] NCCL INFO Connected all rings
+gpua036:985764:985826 [2] NCCL INFO Channel 00/0 : 30[85000] -> 29[46000] via P2P/IPC/read
+gpua036:985764:985826 [2] NCCL INFO Channel 01/0 : 30[85000] -> 29[46000] via P2P/IPC/read
+gpua051:387977:387977 [1] NCCL INFO cudaDriverVersion 12020
+gpua051:387977:387977 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.51<0>
+gpua051:387977:387977 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua051:387977:388070 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.51<0>
+gpua051:387977:388070 [1] NCCL INFO Using network IB
+gpua051:387977:388070 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua051:387977:388070 [1] NCCL INFO Trees [0] 34/16/-1->33->32 [1] 34/-1/-1->33->32
+gpua051:387977:388070 [1] NCCL INFO Channel 00/0 : 33[46000] -> 34[85000] via P2P/IPC/read
+gpua051:387977:388070 [1] NCCL INFO Channel 01/0 : 33[46000] -> 34[85000] via P2P/IPC/read
+gpua051:387977:388070 [1] NCCL INFO Connected all rings
+gpua051:387977:388070 [1] NCCL INFO Channel 00/0 : 16[7000] -> 33[46000] [receive] via NET/IB/0
+gpua051:387977:388070 [1] NCCL INFO Channel 00/0 : 33[46000] -> 16[7000] [send] via NET/IB/0
+gpua079:2406227:2406227 [1] NCCL INFO cudaDriverVersion 12020
+gpua079:2406227:2406227 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.79<0>
+gpua079:2406227:2406227 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua079:2406227:2406305 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.79<0>
+gpua079:2406227:2406305 [1] NCCL INFO Using network IB
+gpua079:2406227:2406305 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua079:2406227:2406305 [1] NCCL INFO Trees [0] 50/40/-1->49->48 [1] 50/-1/-1->49->48
+gpua079:2406227:2406305 [1] NCCL INFO Channel 00/0 : 49[46000] -> 50[85000] via P2P/IPC/read
+gpua079:2406227:2406305 [1] NCCL INFO Channel 01/0 : 49[46000] -> 50[85000] via P2P/IPC/read
+gpua079:2406227:2406305 [1] NCCL INFO Connected all rings
+gpua079:2406227:2406305 [1] NCCL INFO Channel 00/0 : 40[7000] -> 49[46000] [receive] via NET/IB/0
+gpua079:2406227:2406305 [1] NCCL INFO Channel 00/0 : 49[46000] -> 40[7000] [send] via NET/IB/0
+gpua058:102161:102161 [2] NCCL INFO cudaDriverVersion 12020
+gpua058:102161:102161 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.58<0>
+gpua058:102161:102161 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua058:102161:102251 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.58<0>
+gpua058:102161:102251 [2] NCCL INFO Using network IB
+gpua058:102161:102251 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua058:102161:102251 [2] NCCL INFO Trees [0] 47/-1/-1->46->45 [1] 47/-1/-1->46->45
+gpua058:102161:102251 [2] NCCL INFO Channel 00/0 : 46[85000] -> 47[c7000] via P2P/IPC/read
+gpua058:102161:102251 [2] NCCL INFO Channel 01/0 : 46[85000] -> 47[c7000] via P2P/IPC/read
+gpua058:102161:102251 [2] NCCL INFO Connected all rings
+gpua058:102161:102251 [2] NCCL INFO Channel 00/0 : 46[85000] -> 45[46000] via P2P/IPC/read
+gpua058:102161:102251 [2] NCCL INFO Channel 01/0 : 46[85000] -> 45[46000] via P2P/IPC/read
+gpua053:179584:179584 [0] NCCL INFO cudaDriverVersion 12020
+gpua053:179584:179584 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.53<0>
+gpua053:179584:179584 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua053:179584:179685 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.53<0>
+gpua053:179584:179685 [0] NCCL INFO Using network IB
+gpua053:179584:179685 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua053:179584:179685 [0] NCCL INFO Trees [0] 37/-1/-1->36->41 [1] 37/32/-1->36->44
+gpua053:179584:179685 [0] NCCL INFO Channel 00/0 : 35[c7000] -> 36[7000] [receive] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Channel 01/0 : 35[c7000] -> 36[7000] [receive] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Channel 00/0 : 36[7000] -> 37[46000] via P2P/IPC/read
+gpua053:179584:179685 [0] NCCL INFO Channel 01/0 : 36[7000] -> 37[46000] via P2P/IPC/read
+gpua053:179584:179685 [0] NCCL INFO Connected all rings
+gpua055:2316454:2316454 [1] NCCL INFO cudaDriverVersion 12020
+gpua055:2316454:2316454 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.55<0>
+gpua055:2316454:2316454 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua055:2316454:2316519 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.55<0>
+gpua055:2316454:2316519 [1] NCCL INFO Using network IB
+gpua055:2316454:2316519 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua055:2316454:2316519 [1] NCCL INFO Trees [0] 42/36/-1->41->40 [1] 42/-1/-1->41->40
+gpua055:2316454:2316519 [1] NCCL INFO Channel 00/0 : 41[46000] -> 42[85000] via P2P/IPC/read
+gpua055:2316454:2316519 [1] NCCL INFO Channel 01/0 : 41[46000] -> 42[85000] via P2P/IPC/read
+gpua055:2316454:2316519 [1] NCCL INFO Connected all rings
+gpua055:2316454:2316519 [1] NCCL INFO Channel 00/0 : 36[7000] -> 41[46000] [receive] via NET/IB/0
+gpua055:2316454:2316519 [1] NCCL INFO Channel 00/0 : 41[46000] -> 36[7000] [send] via NET/IB/0
+gpua083:196704:196704 [1] NCCL INFO cudaDriverVersion 12020
+gpua083:196704:196704 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.83<0>
+gpua083:196704:196704 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua083:196704:196774 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.83<0>
+gpua083:196704:196774 [1] NCCL INFO Using network IB
+gpua083:196704:196774 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua083:196704:196774 [1] NCCL INFO Trees [0] 58/52/-1->57->56 [1] 58/-1/-1->57->56
+gpua083:196704:196774 [1] NCCL INFO Channel 00/0 : 57[46000] -> 58[85000] via P2P/IPC/read
+gpua083:196704:196774 [1] NCCL INFO Channel 01/0 : 57[46000] -> 58[85000] via P2P/IPC/read
+gpua083:196704:196774 [1] NCCL INFO Connected all rings
+gpua083:196704:196774 [1] NCCL INFO Channel 00/0 : 52[7000] -> 57[46000] [receive] via NET/IB/0
+gpua083:196704:196774 [1] NCCL INFO Channel 00/0 : 57[46000] -> 52[7000] [send] via NET/IB/0
+gpua080:1427105:1427105 [2] NCCL INFO cudaDriverVersion 12020
+gpua080:1427105:1427105 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.80<0>
+gpua080:1427105:1427105 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua080:1427105:1427166 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.80<0>
+gpua080:1427105:1427166 [2] NCCL INFO Using network IB
+gpua080:1427105:1427166 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua080:1427105:1427166 [2] NCCL INFO Trees [0] 55/-1/-1->54->53 [1] 55/-1/-1->54->53
+gpua080:1427105:1427166 [2] NCCL INFO Channel 00/0 : 54[85000] -> 55[c7000] via P2P/IPC/read
+gpua080:1427105:1427166 [2] NCCL INFO Channel 01/0 : 54[85000] -> 55[c7000] via P2P/IPC/read
+gpua080:1427105:1427166 [2] NCCL INFO Connected all rings
+gpua080:1427105:1427166 [2] NCCL INFO Channel 00/0 : 54[85000] -> 53[46000] via P2P/IPC/read
+gpua080:1427105:1427166 [2] NCCL INFO Channel 01/0 : 54[85000] -> 53[46000] via P2P/IPC/read
+gpua087:102965:102965 [1] NCCL INFO cudaDriverVersion 12020
+gpua087:102965:102965 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.87<0>
+gpua087:102965:102965 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua087:102965:103027 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.87<0>
+gpua087:102965:103027 [1] NCCL INFO Using network IB
+gpua087:102965:103027 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua087:102965:103027 [1] NCCL INFO Trees [0] 62/-1/-1->61->60 [1] 62/-1/-1->61->60
+gpua087:102965:103027 [1] NCCL INFO Channel 00/0 : 61[46000] -> 62[85000] via P2P/IPC/read
+gpua087:102965:103027 [1] NCCL INFO Channel 01/0 : 61[46000] -> 62[85000] via P2P/IPC/read
+gpua087:102965:103027 [1] NCCL INFO Connected all rings
+gpua087:102965:103027 [1] NCCL INFO Channel 00/0 : 61[46000] -> 60[7000] via P2P/IPC/read
+gpua087:102965:103027 [1] NCCL INFO Channel 01/0 : 61[46000] -> 60[7000] via P2P/IPC/read
+gpua036:985764:985826 [2] NCCL INFO Connected all trees
+gpua036:985764:985826 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua036:985764:985826 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua036:985764:985826 [2] NCCL INFO comm 0x55c4a4c8da20 rank 30 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua051:387977:388070 [1] NCCL INFO Channel 00/0 : 33[46000] -> 32[7000] via P2P/IPC/read
+gpua051:387977:388070 [1] NCCL INFO Channel 01/0 : 33[46000] -> 32[7000] via P2P/IPC/read
+gpua051:387977:388070 [1] NCCL INFO Connected all trees
+gpua051:387977:388070 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua051:387977:388070 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua051:387977:388070 [1] NCCL INFO comm 0x561d489334e0 rank 33 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua079:2406227:2406305 [1] NCCL INFO Channel 00/0 : 49[46000] -> 48[7000] via P2P/IPC/read
+gpua079:2406227:2406305 [1] NCCL INFO Channel 01/0 : 49[46000] -> 48[7000] via P2P/IPC/read
+gpua079:2406227:2406305 [1] NCCL INFO Connected all trees
+gpua079:2406227:2406305 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua079:2406227:2406305 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua079:2406227:2406305 [1] NCCL INFO comm 0x5643b77bfd50 rank 49 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua058:102161:102251 [2] NCCL INFO Connected all trees
+gpua058:102161:102251 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua058:102161:102251 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua058:102161:102251 [2] NCCL INFO comm 0x55705da34420 rank 46 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua053:179584:179685 [0] NCCL INFO Channel 01/0 : 32[7000] -> 36[7000] [receive] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Channel 00/0 : 36[7000] -> 41[46000] [send] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Channel 01/0 : 36[7000] -> 44[7000] [send] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Channel 01/0 : 44[7000] -> 36[7000] [receive] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Channel 00/0 : 41[46000] -> 36[7000] [receive] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Channel 01/0 : 36[7000] -> 32[7000] [send] via NET/IB/0
+gpua053:179584:179685 [0] NCCL INFO Connected all trees
+gpua053:179584:179685 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua053:179584:179685 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua053:179584:179685 [0] NCCL INFO comm 0x55d72416a360 rank 36 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua055:2316454:2316519 [1] NCCL INFO Channel 00/0 : 41[46000] -> 40[7000] via P2P/IPC/read
+gpua055:2316454:2316519 [1] NCCL INFO Channel 01/0 : 41[46000] -> 40[7000] via P2P/IPC/read
+gpua055:2316454:2316519 [1] NCCL INFO Connected all trees
+gpua055:2316454:2316519 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua055:2316454:2316519 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua055:2316454:2316519 [1] NCCL INFO comm 0x55cdce03aee0 rank 41 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua083:196704:196774 [1] NCCL INFO Channel 00/0 : 57[46000] -> 56[7000] via P2P/IPC/read
+gpua083:196704:196774 [1] NCCL INFO Channel 01/0 : 57[46000] -> 56[7000] via P2P/IPC/read
+gpua083:196704:196774 [1] NCCL INFO Connected all trees
+gpua083:196704:196774 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua083:196704:196774 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua083:196704:196774 [1] NCCL INFO comm 0x55ce729cb670 rank 57 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua080:1427105:1427166 [2] NCCL INFO Connected all trees
+gpua080:1427105:1427166 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua080:1427105:1427166 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua080:1427105:1427166 [2] NCCL INFO comm 0x561931f3d1d0 rank 54 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua087:102965:103027 [1] NCCL INFO Connected all trees
+gpua087:102965:103027 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua087:102965:103027 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua087:102965:103027 [1] NCCL INFO comm 0x55a57cf7b4f0 rank 61 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua002:51611:51611 [2] NCCL INFO cudaDriverVersion 12020
+gpua002:51611:51611 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.2<0>
+gpua002:51611:51611 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua002:51611:51706 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.2<0>
+gpua002:51611:51706 [2] NCCL INFO Using network IB
+gpua002:51611:51706 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua002:51611:51706 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1
+gpua002:51611:51706 [2] NCCL INFO Channel 00/0 : 2[85000] -> 3[c7000] via P2P/IPC/read
+gpua002:51611:51706 [2] NCCL INFO Channel 01/0 : 2[85000] -> 3[c7000] via P2P/IPC/read
+gpua002:51611:51706 [2] NCCL INFO Connected all rings
+gpua002:51611:51706 [2] NCCL INFO Channel 00/0 : 2[85000] -> 1[46000] via P2P/IPC/read
+gpua002:51611:51706 [2] NCCL INFO Channel 01/0 : 2[85000] -> 1[46000] via P2P/IPC/read
+gpua002:51611:51706 [2] NCCL INFO Connected all trees
+gpua016:311570:311570 [3] NCCL INFO cudaDriverVersion 12020
+gpua016:311570:311570 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0>
+gpua016:311570:311570 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua016:311570:311681 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.16<0>
+gpua016:311570:311681 [3] NCCL INFO Using network IB
+gpua016:311570:311681 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua016:311570:311681 [3] NCCL INFO Trees [0] -1/-1/-1->11->10 [1] -1/-1/-1->11->10
+gpua016:311570:311681 [3] NCCL INFO Channel 00/0 : 11[c7000] -> 12[7000] [send] via NET/IB/0
+gpua016:311570:311681 [3] NCCL INFO Channel 01/0 : 11[c7000] -> 12[7000] [send] via NET/IB/0
+gpua016:311570:311681 [3] NCCL INFO Connected all rings
+gpua016:311570:311681 [3] NCCL INFO Channel 00/0 : 11[c7000] -> 10[85000] via P2P/IPC/read
+gpua016:311570:311681 [3] NCCL INFO Channel 01/0 : 11[c7000] -> 10[85000] via P2P/IPC/read
+gpua002:51611:51706 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua002:51611:51706 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua002:51611:51706 [2] NCCL INFO comm 0x55acdf676190 rank 2 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua024:349660:349660 [2] NCCL INFO cudaDriverVersion 12020
+gpua024:349660:349660 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.24<0>
+gpua024:349660:349660 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua024:349660:349749 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.24<0>
+gpua024:349660:349749 [2] NCCL INFO Using network IB
+gpua024:349660:349749 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua024:349660:349749 [2] NCCL INFO Trees [0] 19/-1/-1->18->17 [1] 19/-1/-1->18->17
+gpua024:349660:349749 [2] NCCL INFO Channel 00/0 : 18[85000] -> 19[c7000] via P2P/IPC/read
+gpua024:349660:349749 [2] NCCL INFO Channel 01/0 : 18[85000] -> 19[c7000] via P2P/IPC/read
+gpua024:349660:349749 [2] NCCL INFO Connected all rings
+gpua024:349660:349749 [2] NCCL INFO Channel 00/0 : 18[85000] -> 17[46000] via P2P/IPC/read
+gpua024:349660:349749 [2] NCCL INFO Channel 01/0 : 18[85000] -> 17[46000] via P2P/IPC/read
+gpua034:186777:186777 [1] NCCL INFO cudaDriverVersion 12020
+gpua034:186777:186777 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.34<0>
+gpua034:186777:186777 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua034:186777:186865 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.34<0>
+gpua034:186777:186865 [1] NCCL INFO Using network IB
+gpua034:186777:186865 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua034:186777:186865 [1] NCCL INFO Trees [0] 22/-1/-1->21->20 [1] 22/24/-1->21->20
+gpua034:186777:186865 [1] NCCL INFO Channel 00/0 : 21[46000] -> 22[85000] via P2P/IPC/read
+gpua034:186777:186865 [1] NCCL INFO Channel 01/0 : 21[46000] -> 22[85000] via P2P/IPC/read
+gpua034:186777:186865 [1] NCCL INFO Connected all rings
+gpua034:186777:186865 [1] NCCL INFO Channel 01/0 : 21[46000] -> 24[7000] [send] via NET/IB/0
+gpua034:186777:186865 [1] NCCL INFO Channel 01/0 : 24[7000] -> 21[46000] [receive] via NET/IB/0
+gpua035:260522:260522 [3] NCCL INFO cudaDriverVersion 12020
+gpua035:260522:260522 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.35<0>
+gpua035:260522:260522 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua035:260522:260623 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.35<0>
+gpua035:260522:260623 [3] NCCL INFO Using network IB
+gpua035:260522:260623 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua035:260522:260623 [3] NCCL INFO Trees [0] -1/-1/-1->27->26 [1] -1/-1/-1->27->26
+gpua035:260522:260623 [3] NCCL INFO Channel 00/0 : 27[c7000] -> 28[7000] [send] via NET/IB/0
+gpua035:260522:260623 [3] NCCL INFO Channel 01/0 : 27[c7000] -> 28[7000] [send] via NET/IB/0
+gpua035:260522:260623 [3] NCCL INFO Connected all rings
+gpua035:260522:260623 [3] NCCL INFO Channel 00/0 : 27[c7000] -> 26[85000] via P2P/IPC/read
+gpua035:260522:260623 [3] NCCL INFO Channel 01/0 : 27[c7000] -> 26[85000] via P2P/IPC/read
+gpua016:311570:311681 [3] NCCL INFO Connected all trees
+gpua016:311570:311681 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua016:311570:311681 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua016:311570:311681 [3] NCCL INFO comm 0x55c00019cee0 rank 11 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua016:311568:311568 [1] NCCL INFO cudaDriverVersion 12020
+gpua016:311568:311568 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0>
+gpua016:311568:311568 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua016:311568:311682 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.16<0>
+gpua016:311568:311682 [1] NCCL INFO Using network IB
+gpua016:311568:311682 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua016:311568:311682 [1] NCCL INFO Trees [0] 10/4/-1->9->8 [1] 10/-1/-1->9->8
+gpua016:311568:311682 [1] NCCL INFO Channel 00/0 : 9[46000] -> 10[85000] via P2P/IPC/read
+gpua013:395172:395172 [0] NCCL INFO cudaDriverVersion 12020
+gpua013:395172:395172 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.13<0>
+gpua013:395172:395172 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua013:395172:395238 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.13<0>
+gpua013:395172:395238 [0] NCCL INFO Using network IB
+gpua013:395172:395238 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua013:395172:395238 [0] NCCL INFO Trees [0] 5/-1/-1->4->9 [1] 5/0/-1->4->12
+gpua013:395172:395238 [0] NCCL INFO Channel 00/0 : 3[c7000] -> 4[7000] [receive] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Channel 01/0 : 3[c7000] -> 4[7000] [receive] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Channel 00/0 : 4[7000] -> 5[46000] via P2P/IPC/read
+gpua013:395172:395238 [0] NCCL INFO Channel 01/0 : 4[7000] -> 5[46000] via P2P/IPC/read
+gpua013:395172:395238 [0] NCCL INFO Connected all rings
+gpua024:349660:349749 [2] NCCL INFO Connected all trees
+gpua024:349660:349749 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua024:349660:349749 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua024:349660:349749 [2] NCCL INFO comm 0x5581f3f66310 rank 18 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua034:186777:186865 [1] NCCL INFO Channel 00/0 : 21[46000] -> 20[7000] via P2P/IPC/read
+gpua034:186777:186865 [1] NCCL INFO Channel 01/0 : 21[46000] -> 20[7000] via P2P/IPC/read
+gpua034:186777:186865 [1] NCCL INFO Connected all trees
+gpua034:186777:186865 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua034:186777:186865 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua034:186777:186865 [1] NCCL INFO comm 0x55e7d5b957e0 rank 21 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua035:260522:260623 [3] NCCL INFO Connected all trees
+gpua035:260522:260623 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua035:260522:260623 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua035:260522:260623 [3] NCCL INFO comm 0x561a303144d0 rank 27 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua035:260521:260521 [2] NCCL INFO cudaDriverVersion 12020
+gpua035:260521:260521 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.35<0>
+gpua035:260521:260521 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua035:260521:260621 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.35<0>
+gpua035:260521:260621 [2] NCCL INFO Using network IB
+gpua035:260521:260621 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua035:260521:260621 [2] NCCL INFO Trees [0] 27/-1/-1->26->25 [1] 27/-1/-1->26->25
+gpua035:260521:260621 [2] NCCL INFO Channel 00/0 : 26[85000] -> 27[c7000] via P2P/IPC/read
+gpua016:311568:311682 [1] NCCL INFO Channel 01/0 : 9[46000] -> 10[85000] via P2P/IPC/read
+gpua016:311568:311682 [1] NCCL INFO Connected all rings
+gpua016:311568:311682 [1] NCCL INFO Channel 00/0 : 4[7000] -> 9[46000] [receive] via NET/IB/0
+gpua016:311568:311682 [1] NCCL INFO Channel 00/0 : 9[46000] -> 4[7000] [send] via NET/IB/0
+gpua016:311568:311682 [1] NCCL INFO Channel 00/0 : 9[46000] -> 8[7000] via P2P/IPC/read
+gpua016:311568:311682 [1] NCCL INFO Channel 01/0 : 9[46000] -> 8[7000] via P2P/IPC/read
+gpua016:311568:311682 [1] NCCL INFO Connected all trees
+gpua016:311568:311682 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua016:311568:311682 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua016:311568:311682 [1] NCCL INFO comm 0x55c0bac9dda0 rank 9 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua016:311567:311567 [0] NCCL INFO cudaDriverVersion 12020
+gpua016:311567:311567 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0>
+gpua013:395172:395238 [0] NCCL INFO Channel 01/0 : 0[7000] -> 4[7000] [receive] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Channel 00/0 : 4[7000] -> 9[46000] [send] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Channel 01/0 : 4[7000] -> 12[7000] [send] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Channel 01/0 : 12[7000] -> 4[7000] [receive] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Channel 00/0 : 9[46000] -> 4[7000] [receive] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Channel 01/0 : 4[7000] -> 0[7000] [send] via NET/IB/0
+gpua013:395172:395238 [0] NCCL INFO Connected all trees
+gpua013:395172:395238 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua013:395172:395238 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua013:395172:395238 [0] NCCL INFO comm 0x55f77ab49490 rank 4 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua034:186778:186778 [2] NCCL INFO cudaDriverVersion 12020
+gpua034:186778:186778 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.34<0>
+gpua034:186778:186778 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua034:186778:186867 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.34<0>
+gpua034:186778:186867 [2] NCCL INFO Using network IB
+gpua034:186778:186867 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua034:186778:186867 [2] NCCL INFO Trees [0] 23/-1/-1->22->21 [1] 23/-1/-1->22->21
+gpua034:186778:186867 [2] NCCL INFO Channel 00/0 : 22[85000] -> 23[c7000] via P2P/IPC/read
+gpua034:186778:186867 [2] NCCL INFO Channel 01/0 : 22[85000] -> 23[c7000] via P2P/IPC/read
+gpua034:186778:186867 [2] NCCL INFO Connected all rings
+gpua034:186778:186867 [2] NCCL INFO Channel 00/0 : 22[85000] -> 21[46000] via P2P/IPC/read
+gpua034:186778:186867 [2] NCCL INFO Channel 01/0 : 22[85000] -> 21[46000] via P2P/IPC/read
+gpua035:260521:260621 [2] NCCL INFO Channel 01/0 : 26[85000] -> 27[c7000] via P2P/IPC/read
+gpua035:260521:260621 [2] NCCL INFO Connected all rings
+gpua035:260521:260621 [2] NCCL INFO Channel 00/0 : 26[85000] -> 25[46000] via P2P/IPC/read
+gpua035:260521:260621 [2] NCCL INFO Channel 01/0 : 26[85000] -> 25[46000] via P2P/IPC/read
+gpua035:260521:260621 [2] NCCL INFO Connected all trees
+gpua035:260521:260621 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua035:260521:260621 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua035:260521:260621 [2] NCCL INFO comm 0x555bf1b64b20 rank 26 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua016:311567:311567 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua016:311567:311679 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.16<0>
+gpua016:311567:311679 [0] NCCL INFO Using network IB
+gpua016:311567:311679 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua016:311567:311679 [0] NCCL INFO Trees [0] 9/12/-1->8->17 [1] 9/-1/-1->8->5
+gpua016:311567:311679 [0] NCCL INFO Channel 00/0 : 7[c7000] -> 8[7000] [receive] via NET/IB/0
+gpua016:311567:311679 [0] NCCL INFO Channel 01/0 : 7[c7000] -> 8[7000] [receive] via NET/IB/0
+gpua016:311567:311679 [0] NCCL INFO Channel 00/0 : 8[7000] -> 9[46000] via P2P/IPC/read
+gpua016:311567:311679 [0] NCCL INFO Channel 01/0 : 8[7000] -> 9[46000] via P2P/IPC/read
+gpua016:311567:311679 [0] NCCL INFO Connected all rings
+gpua016:311567:311679 [0] NCCL INFO Channel 01/0 : 5[46000] -> 8[7000] [receive] via NET/IB/0
+gpua034:186778:186867 [2] NCCL INFO Connected all trees
+gpua034:186778:186867 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua034:186778:186867 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua034:186778:186867 [2] NCCL INFO comm 0x56381768d830 rank 22 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua016:311567:311679 [0] NCCL INFO Channel 00/0 : 8[7000] -> 12[7000] [send] via NET/IB/0
+gpua016:311567:311679 [0] NCCL INFO Channel 00/0 : 8[7000] -> 17[46000] [send] via NET/IB/0
+gpua034:186776:186776 [0] NCCL INFO cudaDriverVersion 12020
+gpua034:186776:186776 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.34<0>
+gpua034:186776:186776 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua034:186776:186864 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.34<0>
+gpua034:186776:186864 [0] NCCL INFO Using network IB
+gpua034:186776:186864 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua034:186776:186864 [0] NCCL INFO Trees [0] 21/-1/-1->20->25 [1] 21/16/-1->20->13
+gpua034:186776:186864 [0] NCCL INFO Channel 00/0 : 19[c7000] -> 20[7000] [receive] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Channel 01/0 : 19[c7000] -> 20[7000] [receive] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Channel 00/0 : 20[7000] -> 21[46000] via P2P/IPC/read
+gpua034:186776:186864 [0] NCCL INFO Channel 01/0 : 20[7000] -> 21[46000] via P2P/IPC/read
+gpua034:186776:186864 [0] NCCL INFO Connected all rings
+gpua016:311567:311679 [0] NCCL INFO Channel 00/0 : 17[46000] -> 8[7000] [receive] via NET/IB/0
+gpua016:311567:311679 [0] NCCL INFO Channel 00/0 : 12[7000] -> 8[7000] [receive] via NET/IB/0
+gpua016:311567:311679 [0] NCCL INFO Channel 01/0 : 8[7000] -> 5[46000] [send] via NET/IB/0
+gpua016:311567:311679 [0] NCCL INFO Connected all trees
+gpua016:311567:311679 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua016:311567:311679 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua016:311567:311679 [0] NCCL INFO comm 0x55da7ca44890 rank 8 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua016:311569:311569 [2] NCCL INFO cudaDriverVersion 12020
+gpua016:311569:311569 [2] NCCL INFO Bootstrap : Using eth1:172.28.23.16<0>
+gpua016:311569:311569 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua016:311569:311680 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.16<0>
+gpua016:311569:311680 [2] NCCL INFO Using network IB
+gpua034:186776:186864 [0] NCCL INFO Channel 01/0 : 16[7000] -> 20[7000] [receive] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Channel 00/0 : 20[7000] -> 25[46000] [send] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Channel 01/0 : 13[46000] -> 20[7000] [receive] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Channel 01/0 : 20[7000] -> 13[46000] [send] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Channel 00/0 : 25[46000] -> 20[7000] [receive] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Channel 01/0 : 20[7000] -> 16[7000] [send] via NET/IB/0
+gpua034:186776:186864 [0] NCCL INFO Connected all trees
+gpua034:186776:186864 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua034:186776:186864 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua034:186776:186864 [0] NCCL INFO comm 0x562076438ba0 rank 20 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua016:311569:311680 [2] NCCL INFO Setting affinity for GPU 2 to ffff0000
+gpua016:311569:311680 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->9
+gpua016:311569:311680 [2] NCCL INFO Channel 00/0 : 10[85000] -> 11[c7000] via P2P/IPC/read
+gpua016:311569:311680 [2] NCCL INFO Channel 01/0 : 10[85000] -> 11[c7000] via P2P/IPC/read
+gpua016:311569:311680 [2] NCCL INFO Connected all rings
+gpua016:311569:311680 [2] NCCL INFO Channel 00/0 : 10[85000] -> 9[46000] via P2P/IPC/read
+gpua016:311569:311680 [2] NCCL INFO Channel 01/0 : 10[85000] -> 9[46000] via P2P/IPC/read
+gpua016:311569:311680 [2] NCCL INFO Connected all trees
+gpua016:311569:311680 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua016:311569:311680 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua016:311569:311680 [2] NCCL INFO comm 0x55726fb33ee0 rank 10 nranks 64 cudaDev 2 busId 85000 - Init COMPLETE
+gpua035:260520:260520 [1] NCCL INFO cudaDriverVersion 12020
+gpua035:260520:260520 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.35<0>
+gpua035:260520:260520 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua035:260520:260622 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.35<0>
+gpua035:260520:260622 [1] NCCL INFO Using network IB
+gpua035:260520:260622 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua035:260520:260622 [1] NCCL INFO Trees [0] 26/20/-1->25->24 [1] 26/-1/-1->25->24
+gpua035:260520:260622 [1] NCCL INFO Channel 00/0 : 25[46000] -> 26[85000] via P2P/IPC/read
+gpua035:260520:260622 [1] NCCL INFO Channel 01/0 : 25[46000] -> 26[85000] via P2P/IPC/read
+gpua035:260520:260622 [1] NCCL INFO Connected all rings
+gpua035:260520:260622 [1] NCCL INFO Channel 00/0 : 20[7000] -> 25[46000] [receive] via NET/IB/0
+gpua035:260520:260622 [1] NCCL INFO Channel 00/0 : 25[46000] -> 20[7000] [send] via NET/IB/0
+gpua035:260520:260622 [1] NCCL INFO Channel 00/0 : 25[46000] -> 24[7000] via P2P/IPC/read
+gpua035:260520:260622 [1] NCCL INFO Channel 01/0 : 25[46000] -> 24[7000] via P2P/IPC/read
+gpua035:260520:260622 [1] NCCL INFO Connected all trees
+gpua035:260520:260622 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua035:260520:260622 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua035:260520:260622 [1] NCCL INFO comm 0x55ad88e815a0 rank 25 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua035:260519:260519 [0] NCCL INFO cudaDriverVersion 12020
+gpua035:260519:260519 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.35<0>
+gpua035:260519:260519 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua035:260519:260620 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.35<0>
+gpua035:260519:260620 [0] NCCL INFO Using network IB
+gpua035:260519:260620 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua035:260519:260620 [0] NCCL INFO Trees [0] 25/28/-1->24->16 [1] 25/-1/-1->24->21
+gpua035:260519:260620 [0] NCCL INFO Channel 00/0 : 23[c7000] -> 24[7000] [receive] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Channel 01/0 : 23[c7000] -> 24[7000] [receive] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Channel 00/0 : 24[7000] -> 25[46000] via P2P/IPC/read
+gpua035:260519:260620 [0] NCCL INFO Channel 01/0 : 24[7000] -> 25[46000] via P2P/IPC/read
+gpua035:260519:260620 [0] NCCL INFO Connected all rings
+gpua035:260519:260620 [0] NCCL INFO Channel 01/0 : 21[46000] -> 24[7000] [receive] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Channel 00/0 : 24[7000] -> 28[7000] [send] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Channel 00/0 : 16[7000] -> 24[7000] [receive] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Channel 00/0 : 24[7000] -> 16[7000] [send] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Channel 00/0 : 28[7000] -> 24[7000] [receive] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Channel 01/0 : 24[7000] -> 21[46000] [send] via NET/IB/0
+gpua035:260519:260620 [0] NCCL INFO Connected all trees
+gpua035:260519:260620 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua035:260519:260620 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua035:260519:260620 [0] NCCL INFO comm 0x55da44f13b30 rank 24 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua002:51612:51612 [3] NCCL INFO cudaDriverVersion 12020
+gpua002:51612:51612 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.2<0>
+gpua002:51612:51612 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua002:51612:51707 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.2<0>
+gpua002:51612:51707 [3] NCCL INFO Using network IB
+gpua002:51612:51707 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua002:51612:51707 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2
+gpua002:51612:51707 [3] NCCL INFO Channel 00/0 : 3[c7000] -> 4[7000] [send] via NET/IB/0
+gpua002:51612:51707 [3] NCCL INFO Channel 01/0 : 3[c7000] -> 4[7000] [send] via NET/IB/0
+gpua002:51612:51707 [3] NCCL INFO Connected all rings
+gpua002:51612:51707 [3] NCCL INFO Channel 00/0 : 3[c7000] -> 2[85000] via P2P/IPC/read
+gpua002:51612:51707 [3] NCCL INFO Channel 01/0 : 3[c7000] -> 2[85000] via P2P/IPC/read
+gpua002:51612:51707 [3] NCCL INFO Connected all trees
+gpua034:186779:186779 [3] NCCL INFO cudaDriverVersion 12020
+gpua034:186779:186779 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.34<0>
+gpua034:186779:186779 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua034:186779:186866 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.34<0>
+gpua034:186779:186866 [3] NCCL INFO Using network IB
+gpua034:186779:186866 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua034:186779:186866 [3] NCCL INFO Trees [0] -1/-1/-1->23->22 [1] -1/-1/-1->23->22
+gpua034:186779:186866 [3] NCCL INFO Channel 00/0 : 23[c7000] -> 24[7000] [send] via NET/IB/0
+gpua034:186779:186866 [3] NCCL INFO Channel 01/0 : 23[c7000] -> 24[7000] [send] via NET/IB/0
+gpua034:186779:186866 [3] NCCL INFO Connected all rings
+gpua034:186779:186866 [3] NCCL INFO Channel 00/0 : 23[c7000] -> 22[85000] via P2P/IPC/read
+gpua034:186779:186866 [3] NCCL INFO Channel 01/0 : 23[c7000] -> 22[85000] via P2P/IPC/read
+gpua002:51612:51707 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua002:51612:51707 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua002:51612:51707 [3] NCCL INFO comm 0x5594523978c0 rank 3 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua034:186779:186866 [3] NCCL INFO Connected all trees
+gpua034:186779:186866 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua034:186779:186866 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua034:186779:186866 [3] NCCL INFO comm 0x5616b81acd10 rank 23 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua002:51610:51610 [1] NCCL INFO cudaDriverVersion 12020
+gpua002:51610:51610 [1] NCCL INFO Bootstrap : Using eth1:172.28.23.2<0>
+gpua002:51610:51610 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua002:51610:51709 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.2<0>
+gpua002:51610:51709 [1] NCCL INFO Using network IB
+gpua002:51610:51709 [1] NCCL INFO Setting affinity for GPU 1 to ffff,00000000
+gpua002:51610:51709 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0
+gpua002:51610:51709 [1] NCCL INFO Channel 00/0 : 1[46000] -> 2[85000] via P2P/IPC/read
+gpua002:51610:51709 [1] NCCL INFO Channel 01/0 : 1[46000] -> 2[85000] via P2P/IPC/read
+gpua002:51610:51709 [1] NCCL INFO Connected all rings
+gpua002:51610:51709 [1] NCCL INFO Channel 00/0 : 1[46000] -> 0[7000] via P2P/IPC/read
+gpua002:51610:51709 [1] NCCL INFO Channel 01/0 : 1[46000] -> 0[7000] via P2P/IPC/read
+gpua002:51610:51709 [1] NCCL INFO Connected all trees
+gpua024:349661:349661 [3] NCCL INFO cudaDriverVersion 12020
+gpua024:349661:349661 [3] NCCL INFO Bootstrap : Using eth1:172.28.23.24<0>
+gpua024:349661:349661 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua024:349661:349750 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.24<0>
+gpua024:349661:349750 [3] NCCL INFO Using network IB
+gpua024:349661:349750 [3] NCCL INFO Setting affinity for GPU 3 to ffff
+gpua024:349661:349750 [3] NCCL INFO Trees [0] -1/-1/-1->19->18 [1] -1/-1/-1->19->18
+gpua024:349661:349750 [3] NCCL INFO Channel 00/0 : 19[c7000] -> 20[7000] [send] via NET/IB/0
+gpua024:349661:349750 [3] NCCL INFO Channel 01/0 : 19[c7000] -> 20[7000] [send] via NET/IB/0
+gpua024:349661:349750 [3] NCCL INFO Connected all rings
+gpua024:349661:349750 [3] NCCL INFO Channel 00/0 : 19[c7000] -> 18[85000] via P2P/IPC/read
+gpua024:349661:349750 [3] NCCL INFO Channel 01/0 : 19[c7000] -> 18[85000] via P2P/IPC/read
+gpua002:51610:51709 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua002:51610:51709 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua002:51610:51709 [1] NCCL INFO comm 0x55dc7087fd20 rank 1 nranks 64 cudaDev 1 busId 46000 - Init COMPLETE
+gpua024:349661:349750 [3] NCCL INFO Connected all trees
+gpua024:349661:349750 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua024:349661:349750 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua024:349661:349750 [3] NCCL INFO comm 0x55e3a5740670 rank 19 nranks 64 cudaDev 3 busId c7000 - Init COMPLETE
+gpua024:349658:349658 [0] NCCL INFO cudaDriverVersion 12020
+gpua024:349658:349658 [0] NCCL INFO Bootstrap : Using eth1:172.28.23.24<0>
+gpua024:349658:349658 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation
+gpua024:349658:349747 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.24<0>
+gpua024:349658:349747 [0] NCCL INFO Using network IB
+gpua024:349658:349747 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua024:349658:349747 [0] NCCL INFO Trees [0] 17/24/-1->16->33 [1] 17/-1/-1->16->20
+gpua024:349658:349747 [0] NCCL INFO Channel 00/0 : 15[c7000] -> 16[7000] [receive] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Channel 01/0 : 15[c7000] -> 16[7000] [receive] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Channel 00/0 : 16[7000] -> 17[46000] via P2P/IPC/read
+gpua024:349658:349747 [0] NCCL INFO Channel 01/0 : 16[7000] -> 17[46000] via P2P/IPC/read
+gpua024:349658:349747 [0] NCCL INFO Connected all rings
+gpua024:349658:349747 [0] NCCL INFO Channel 01/0 : 16[7000] -> 20[7000] [send] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Channel 00/0 : 16[7000] -> 24[7000] [send] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Channel 00/0 : 16[7000] -> 33[46000] [send] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Channel 00/0 : 33[46000] -> 16[7000] [receive] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Channel 00/0 : 24[7000] -> 16[7000] [receive] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Channel 01/0 : 20[7000] -> 16[7000] [receive] via NET/IB/0
+gpua024:349658:349747 [0] NCCL INFO Connected all trees
+gpua024:349658:349747 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua024:349658:349747 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua024:349658:349747 [0] NCCL INFO comm 0x5637d5b1cf10 rank 16 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+gpua002:51609:51705 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [RO]; OOB eth1:172.28.23.2<0>
+gpua002:51609:51705 [0] NCCL INFO Using network IB
+gpua002:51609:51705 [0] NCCL INFO Setting affinity for GPU 0 to ffff0000,00000000
+gpua002:51609:51705 [0] NCCL INFO Channel 00/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
+gpua002:51609:51705 [0] NCCL INFO Channel 01/02 :    0   1   2   3   4   5   6   7   8   9  10  11  12  13  14  15  16  17  18  19
+gpua002:51609:51705 [0] NCCL INFO Trees [0] 1/32/-1->0->-1 [1] 1/-1/-1->0->4
+gpua002:51609:51705 [0] NCCL INFO Channel 00/0 : 63[c7000] -> 0[7000] [receive] via NET/IB/0
+gpua002:51609:51705 [0] NCCL INFO Channel 01/0 : 63[c7000] -> 0[7000] [receive] via NET/IB/0
+gpua002:51609:51705 [0] NCCL INFO Channel 00/0 : 0[7000] -> 1[46000] via P2P/IPC/read
+gpua002:51609:51705 [0] NCCL INFO Channel 01/0 : 0[7000] -> 1[46000] via P2P/IPC/read
+gpua002:51609:51705 [0] NCCL INFO Connected all rings
+gpua002:51609:51705 [0] NCCL INFO Channel 01/0 : 0[7000] -> 4[7000] [send] via NET/IB/0
+gpua002:51609:51705 [0] NCCL INFO Channel 00/0 : 32[7000] -> 0[7000] [receive] via NET/IB/0
+gpua002:51609:51705 [0] NCCL INFO Channel 00/0 : 0[7000] -> 32[7000] [send] via NET/IB/0
+gpua002:51609:51705 [0] NCCL INFO Channel 01/0 : 4[7000] -> 0[7000] [receive] via NET/IB/0
+gpua002:51609:51705 [0] NCCL INFO Connected all trees
+gpua002:51609:51705 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 512 | 512
+gpua002:51609:51705 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
+gpua002:51609:51705 [0] NCCL INFO comm 0x5654a5ed42e0 rank 0 nranks 64 cudaDev 0 busId 7000 - Init COMPLETE
+[gpua002:0/64] 2024-01-18 09:02:48,604 (distributed:1027) INFO: Reducer buckets have been rebuilt in this iteration.
+[gpua002:0/64] 2024-01-18 09:05:41,671 (trainer:753) INFO: 10epoch:train:1-100batch: iter_time=2.270, forward_time=0.241, loss_ctc=98.331, loss_interctc_layer6=100.418, loss_interctc_layer12=85.955, loss_interctc_layer15=80.034, loss_interctc_layer21=99.934, loss=92.934, backward_time=0.416, grad_norm=65.974, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.140, optim0_lr0=1.333e-04, train_time=7.205
+[gpua002:0/64] 2024-01-18 09:09:19,814 (trainer:753) INFO: 10epoch:train:101-200batch: iter_time=1.110e-04, forward_time=0.140, loss_ctc=76.288, loss_interctc_layer6=86.201, loss_interctc_layer12=73.480, loss_interctc_layer15=68.054, loss_interctc_layer21=78.009, loss=76.406, backward_time=0.454, grad_norm=109.694, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.136, optim0_lr0=1.333e-04, train_time=2.182
+[gpua002:0/64] 2024-01-18 09:13:05,536 (trainer:753) INFO: 10epoch:train:201-300batch: iter_time=9.399e-05, forward_time=0.152, loss_ctc=121.595, loss_interctc_layer6=110.290, loss_interctc_layer12=94.964, loss_interctc_layer15=88.793, loss_interctc_layer21=124.036, loss=107.935, backward_time=0.503, grad_norm=71.895, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.136, optim0_lr0=1.332e-04, train_time=2.257
+[gpua002:0/64] 2024-01-18 09:16:59,105 (trainer:753) INFO: 10epoch:train:301-400batch: iter_time=9.777e-05, forward_time=0.200, loss_ctc=109.121, loss_interctc_layer6=111.944, loss_interctc_layer12=95.710, loss_interctc_layer15=89.065, loss_interctc_layer21=111.135, loss=103.395, backward_time=0.552, grad_norm=67.281, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.140, optim0_lr0=1.332e-04, train_time=2.335
+[gpua002:0/64] 2024-01-18 09:20:30,669 (trainer:753) INFO: 10epoch:train:401-500batch: iter_time=1.002e-04, forward_time=0.174, loss_ctc=77.964, loss_interctc_layer6=84.434, loss_interctc_layer12=71.922, loss_interctc_layer15=66.740, loss_interctc_layer21=79.454, loss=76.103, backward_time=0.488, grad_norm=74.126, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.138, optim0_lr0=1.331e-04, train_time=2.116
+[gpua002:0/64] 2024-01-18 09:24:25,535 (trainer:753) INFO: 10epoch:train:501-600batch: iter_time=9.848e-05, forward_time=0.147, loss_ctc=93.136, loss_interctc_layer6=96.106, loss_interctc_layer12=81.477, loss_interctc_layer15=75.543, loss_interctc_layer21=94.979, loss=88.248, backward_time=0.459, grad_norm=62.946, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.136, optim0_lr0=1.331e-04, train_time=2.345
+[gpua002:0/64] 2024-01-18 09:28:31,062 (trainer:753) INFO: 10epoch:train:601-700batch: iter_time=1.007e-04, forward_time=0.228, loss_ctc=91.665, loss_interctc_layer6=99.179, loss_interctc_layer12=84.312, loss_interctc_layer15=78.444, loss_interctc_layer21=93.353, loss=89.391, backward_time=0.603, grad_norm=64.359, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.142, optim0_lr0=1.330e-04, train_time=2.457
+[gpua002:0/64] 2024-01-18 09:32:41,587 (trainer:753) INFO: 10epoch:train:701-800batch: iter_time=1.015e-04, forward_time=0.177, loss_ctc=89.784, loss_interctc_layer6=93.778, loss_interctc_layer12=79.393, loss_interctc_layer15=73.810, loss_interctc_layer21=91.691, loss=85.691, backward_time=0.548, grad_norm=72.675, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.141, optim0_lr0=1.330e-04, train_time=2.506
+[gpua002:0/64] 2024-01-18 09:36:19,128 (trainer:753) INFO: 10epoch:train:801-900batch: iter_time=9.845e-05, forward_time=0.140, loss_ctc=94.441, loss_interctc_layer6=94.437, loss_interctc_layer12=81.053, loss_interctc_layer15=75.643, loss_interctc_layer21=96.224, loss=88.360, backward_time=0.440, grad_norm=69.264, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.136, optim0_lr0=1.329e-04, train_time=2.175
+[gpua002:0/64] 2024-01-18 09:40:29,412 (trainer:753) INFO: 10epoch:train:901-1000batch: iter_time=9.528e-05, forward_time=0.258, loss_ctc=84.186, loss_interctc_layer6=95.888, loss_interctc_layer12=81.814, loss_interctc_layer15=75.747, loss_interctc_layer21=85.529, loss=84.633, backward_time=0.573, grad_norm=74.742, clip=100.000, loss_scale=9.671e+24, optim_step_time=0.142, optim0_lr0=1.329e-04, train_time=2.502
+[gpua002:0/64] 2024-01-18 09:44:20,001 (trainer:753) INFO: 10epoch:train:1001-1100batch: iter_time=9.241e-05, forward_time=0.181, loss_ctc=111.905, loss_interctc_layer6=116.779, loss_interctc_layer12=101.392, loss_interctc_layer15=95.878, loss_interctc_layer21=113.063, loss=107.804, backward_time=0.485, grad_norm=91.842, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.137, optim0_lr0=1.328e-04, train_time=2.306
+[gpua002:0/64] 2024-01-18 09:48:14,220 (trainer:753) INFO: 10epoch:train:1101-1200batch: iter_time=9.246e-05, forward_time=0.249, loss_ctc=83.406, loss_interctc_layer6=93.386, loss_interctc_layer12=79.728, loss_interctc_layer15=74.355, loss_interctc_layer21=83.822, loss=82.939, backward_time=0.561, grad_norm=62.319, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.141, optim0_lr0=1.328e-04, train_time=2.341
+[gpua002:0/64] 2024-01-18 09:50:29,106 (multiple_iter_factory:32) INFO: Building 1th iter-factory...
+[gpua002:0/64] 2024-01-18 09:50:48,577 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 09:50:52,179 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea80d00250>)
+[gpua002:0/64] 2024-01-18 09:50:52,179 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, 
+[gpua002:0/64] 2024-01-18 09:50:52,182 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 10:07:25,358 (trainer:753) INFO: 10epoch:train:1201-1300batch: iter_time=5.141, forward_time=0.449, loss_ctc=93.836, loss_interctc_layer6=92.404, loss_interctc_layer12=78.168, loss_interctc_layer15=72.272, loss_interctc_layer21=95.796, loss=86.495, backward_time=0.455, grad_norm=64.234, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.144, optim0_lr0=1.327e-04, train_time=11.510
+[gpua002:0/64] 2024-01-18 10:10:41,566 (trainer:753) INFO: 10epoch:train:1301-1400batch: iter_time=8.395e-05, forward_time=0.217, loss_ctc=100.332, loss_interctc_layer6=98.367, loss_interctc_layer12=83.829, loss_interctc_layer15=77.750, loss_interctc_layer21=102.560, loss=92.568, backward_time=0.556, grad_norm=72.182, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.145, optim0_lr0=1.327e-04, train_time=1.964
+[gpua002:0/64] 2024-01-18 10:14:22,542 (trainer:753) INFO: 10epoch:train:1401-1500batch: iter_time=9.160e-05, forward_time=0.158, loss_ctc=92.338, loss_interctc_layer6=93.921, loss_interctc_layer12=80.557, loss_interctc_layer15=75.201, loss_interctc_layer21=94.323, loss=87.268, backward_time=0.569, grad_norm=66.868, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.137, optim0_lr0=1.326e-04, train_time=2.207
+[gpua002:0/64] 2024-01-18 10:18:45,238 (trainer:753) INFO: 10epoch:train:1501-1600batch: iter_time=2.447e-04, forward_time=0.262, loss_ctc=128.692, loss_interctc_layer6=113.209, loss_interctc_layer12=96.063, loss_interctc_layer15=89.384, loss_interctc_layer21=131.758, loss=111.821, backward_time=0.558, grad_norm=66.152, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.140, optim0_lr0=1.326e-04, train_time=2.628
+[gpua002:0/64] 2024-01-18 10:22:00,556 (trainer:753) INFO: 10epoch:train:1601-1700batch: iter_time=9.580e-05, forward_time=0.243, loss_ctc=88.731, loss_interctc_layer6=91.807, loss_interctc_layer12=77.970, loss_interctc_layer15=72.063, loss_interctc_layer21=90.577, loss=84.230, backward_time=0.432, grad_norm=59.771, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.143, optim0_lr0=1.325e-04, train_time=1.953
+[gpua002:0/64] 2024-01-18 10:24:44,151 (trainer:753) INFO: 10epoch:train:1701-1800batch: iter_time=1.008e-04, forward_time=0.144, loss_ctc=98.109, loss_interctc_layer6=91.305, loss_interctc_layer12=77.594, loss_interctc_layer15=71.851, loss_interctc_layer21=100.507, loss=87.873, backward_time=0.417, grad_norm=73.521, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.137, optim0_lr0=1.325e-04, train_time=1.635
+[gpua002:0/64] 2024-01-18 10:28:01,185 (trainer:753) INFO: 10epoch:train:1801-1900batch: iter_time=1.029e-04, forward_time=0.257, loss_ctc=94.375, loss_interctc_layer6=93.771, loss_interctc_layer12=78.904, loss_interctc_layer15=72.691, loss_interctc_layer21=96.496, loss=87.247, backward_time=0.510, grad_norm=73.872, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.143, optim0_lr0=1.324e-04, train_time=1.970
+[gpua002:0/64] 2024-01-18 10:30:51,655 (trainer:753) INFO: 10epoch:train:1901-2000batch: iter_time=8.851e-05, forward_time=0.143, loss_ctc=93.393, loss_interctc_layer6=95.814, loss_interctc_layer12=81.241, loss_interctc_layer15=75.334, loss_interctc_layer21=95.279, loss=88.212, backward_time=0.382, grad_norm=70.081, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.136, optim0_lr0=1.324e-04, train_time=1.705
+[gpua002:0/64] 2024-01-18 10:35:24,440 (trainer:753) INFO: 10epoch:train:2001-2100batch: iter_time=3.186e-04, forward_time=0.266, loss_ctc=100.531, loss_interctc_layer6=93.709, loss_interctc_layer12=79.307, loss_interctc_layer15=73.814, loss_interctc_layer21=102.673, loss=90.007, backward_time=0.590, grad_norm=112.767, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.142, optim0_lr0=1.323e-04, train_time=2.727
+[gpua002:0/64] 2024-01-18 10:41:21,390 (trainer:753) INFO: 10epoch:train:2101-2200batch: iter_time=1.012e-04, forward_time=0.383, loss_ctc=90.612, loss_interctc_layer6=95.458, loss_interctc_layer12=80.999, loss_interctc_layer15=75.005, loss_interctc_layer21=92.573, loss=86.929, backward_time=0.820, grad_norm=68.151, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.148, optim0_lr0=1.323e-04, train_time=3.569
+[gpua002:0/64] 2024-01-18 10:46:27,072 (trainer:753) INFO: 10epoch:train:2201-2300batch: iter_time=4.333e-04, forward_time=0.255, loss_ctc=96.738, loss_interctc_layer6=107.029, loss_interctc_layer12=91.659, loss_interctc_layer15=85.539, loss_interctc_layer21=98.472, loss=95.888, backward_time=0.725, grad_norm=80.202, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.141, optim0_lr0=1.322e-04, train_time=3.057
+[gpua002:0/64] 2024-01-18 10:50:45,419 (trainer:753) INFO: 10epoch:train:2301-2400batch: iter_time=3.970e-04, forward_time=0.261, loss_ctc=86.362, loss_interctc_layer6=94.858, loss_interctc_layer12=81.399, loss_interctc_layer15=75.951, loss_interctc_layer21=88.321, loss=85.378, backward_time=0.610, grad_norm=82.702, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.140, optim0_lr0=1.322e-04, train_time=2.584
+[gpua002:0/64] 2024-01-18 10:53:47,221 (trainer:753) INFO: 10epoch:train:2401-2500batch: iter_time=9.869e-05, forward_time=0.148, loss_ctc=91.144, loss_interctc_layer6=93.448, loss_interctc_layer12=79.713, loss_interctc_layer15=74.175, loss_interctc_layer21=92.809, loss=86.258, backward_time=0.466, grad_norm=71.437, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.137, optim0_lr0=1.321e-04, train_time=1.818
+[gpua002:0/64] 2024-01-18 10:54:07,296 (multiple_iter_factory:32) INFO: Building 2th iter-factory...
+[gpua002:0/64] 2024-01-18 10:54:26,525 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 10:54:30,185 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea37dcc6a0>)
+[gpua002:0/64] 2024-01-18 10:54:30,185 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, 
+[gpua002:0/64] 2024-01-18 10:54:30,189 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 11:08:47,958 (trainer:753) INFO: 10epoch:train:2501-2600batch: iter_time=3.465, forward_time=0.215, loss_ctc=101.589, loss_interctc_layer6=98.485, loss_interctc_layer12=83.805, loss_interctc_layer15=77.760, loss_interctc_layer21=103.886, loss=93.105, backward_time=0.320, grad_norm=83.846, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.141, optim0_lr0=1.321e-04, train_time=9.008
+[gpua002:0/64] 2024-01-18 11:11:50,145 (trainer:753) INFO: 10epoch:train:2601-2700batch: iter_time=9.697e-05, forward_time=0.170, loss_ctc=78.507, loss_interctc_layer6=85.415, loss_interctc_layer12=72.538, loss_interctc_layer15=67.237, loss_interctc_layer21=80.138, loss=76.767, backward_time=0.421, grad_norm=68.810, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.146, optim0_lr0=1.320e-04, train_time=1.822
+[gpua002:0/64] 2024-01-18 11:15:22,688 (trainer:753) INFO: 10epoch:train:2701-2800batch: iter_time=1.013e-04, forward_time=0.274, loss_ctc=126.751, loss_interctc_layer6=107.375, loss_interctc_layer12=91.683, loss_interctc_layer15=85.136, loss_interctc_layer21=129.440, loss=108.077, backward_time=0.474, grad_norm=68.127, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.141, optim0_lr0=1.320e-04, train_time=2.123
+[gpua002:0/64] 2024-01-18 11:18:16,883 (trainer:753) INFO: 10epoch:train:2801-2900batch: iter_time=1.010e-04, forward_time=0.145, loss_ctc=110.296, loss_interctc_layer6=109.997, loss_interctc_layer12=93.330, loss_interctc_layer15=86.478, loss_interctc_layer21=112.738, loss=102.567, backward_time=0.348, grad_norm=79.108, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.136, optim0_lr0=1.319e-04, train_time=1.743
+[gpua002:0/64] 2024-01-18 11:22:45,397 (trainer:753) INFO: 10epoch:train:2901-3000batch: iter_time=0.001, forward_time=0.154, loss_ctc=80.900, loss_interctc_layer6=82.719, loss_interctc_layer12=69.806, loss_interctc_layer15=64.651, loss_interctc_layer21=82.966, loss=76.208, backward_time=0.546, grad_norm=100.429, clip=100.000, loss_scale=1.934e+25, optim_step_time=0.136, optim0_lr0=1.319e-04, train_time=2.685
+[gpua002:0/64] 2024-01-18 11:26:32,596 (trainer:753) INFO: 10epoch:train:3001-3100batch: iter_time=1.073e-04, forward_time=0.142, loss_ctc=99.720, loss_interctc_layer6=95.598, loss_interctc_layer12=80.624, loss_interctc_layer15=74.387, loss_interctc_layer21=102.048, loss=90.475, backward_time=0.530, grad_norm=62.231, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.137, optim0_lr0=1.319e-04, train_time=2.272
+[gpua002:0/64] 2024-01-18 11:29:19,158 (trainer:753) INFO: 10epoch:train:3101-3200batch: iter_time=1.069e-04, forward_time=0.160, loss_ctc=93.595, loss_interctc_layer6=96.821, loss_interctc_layer12=81.673, loss_interctc_layer15=75.735, loss_interctc_layer21=95.232, loss=88.611, backward_time=0.449, grad_norm=68.156, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.137, optim0_lr0=1.318e-04, train_time=1.665
+[gpua002:0/64] 2024-01-18 11:33:16,341 (trainer:753) INFO: 10epoch:train:3201-3300batch: iter_time=1.083e-04, forward_time=0.295, loss_ctc=93.670, loss_interctc_layer6=93.001, loss_interctc_layer12=78.791, loss_interctc_layer15=72.732, loss_interctc_layer21=95.671, loss=86.773, backward_time=0.605, grad_norm=59.784, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.146, optim0_lr0=1.318e-04, train_time=2.371
+[gpua002:0/64] 2024-01-18 11:36:52,661 (trainer:753) INFO: 10epoch:train:3301-3400batch: iter_time=1.130e-04, forward_time=0.142, loss_ctc=94.966, loss_interctc_layer6=92.840, loss_interctc_layer12=78.866, loss_interctc_layer15=73.141, loss_interctc_layer21=97.298, loss=87.423, backward_time=0.404, grad_norm=53.843, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.317e-04, train_time=2.163
+[gpua002:0/64] 2024-01-18 11:40:23,540 (trainer:753) INFO: 10epoch:train:3401-3500batch: iter_time=7.516e-04, forward_time=0.152, loss_ctc=87.488, loss_interctc_layer6=94.957, loss_interctc_layer12=80.012, loss_interctc_layer15=73.820, loss_interctc_layer21=89.476, loss=85.151, backward_time=0.377, grad_norm=64.279, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.317e-04, train_time=2.108
+[gpua002:0/64] 2024-01-18 11:42:37,237 (trainer:753) INFO: 10epoch:train:3501-3600batch: iter_time=1.091e-04, forward_time=0.147, loss_ctc=104.848, loss_interctc_layer6=111.691, loss_interctc_layer12=95.459, loss_interctc_layer15=89.496, loss_interctc_layer21=106.894, loss=101.678, backward_time=0.304, grad_norm=104.732, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.316e-04, train_time=1.337
+[gpua002:0/64] 2024-01-18 11:46:06,052 (trainer:753) INFO: 10epoch:train:3601-3700batch: iter_time=1.040e-04, forward_time=0.142, loss_ctc=83.460, loss_interctc_layer6=92.000, loss_interctc_layer12=78.695, loss_interctc_layer15=73.105, loss_interctc_layer21=84.856, loss=82.423, backward_time=0.581, grad_norm=114.441, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.316e-04, train_time=2.088
+[gpua002:0/64] 2024-01-18 11:47:52,843 (multiple_iter_factory:32) INFO: Building 3th iter-factory...
+[gpua002:0/64] 2024-01-18 11:48:12,169 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 11:48:15,833 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7ff30931a560>)
+[gpua002:0/64] 2024-01-18 11:48:15,833 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, 
+[gpua002:0/64] 2024-01-18 11:48:15,836 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 11:58:11,555 (trainer:753) INFO: 10epoch:train:3701-3800batch: iter_time=3.416, forward_time=0.299, loss_ctc=91.378, loss_interctc_layer6=91.684, loss_interctc_layer12=77.297, loss_interctc_layer15=71.201, loss_interctc_layer21=93.484, loss=85.009, backward_time=0.356, grad_norm=68.199, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.143, optim0_lr0=1.315e-04, train_time=7.254
+[gpua002:0/64] 2024-01-18 12:00:25,566 (trainer:753) INFO: 10epoch:train:3801-3900batch: iter_time=9.544e-05, forward_time=0.142, loss_ctc=98.771, loss_interctc_layer6=98.144, loss_interctc_layer12=83.796, loss_interctc_layer15=77.706, loss_interctc_layer21=101.040, loss=91.891, backward_time=0.311, grad_norm=65.395, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.315e-04, train_time=1.340
+[gpua002:0/64] 2024-01-18 12:03:11,100 (trainer:753) INFO: 10epoch:train:3901-4000batch: iter_time=0.005, forward_time=0.196, loss_ctc=90.607, loss_interctc_layer6=92.746, loss_interctc_layer12=79.164, loss_interctc_layer15=73.552, loss_interctc_layer21=92.694, loss=85.753, backward_time=0.381, grad_norm=106.868, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.141, optim0_lr0=1.314e-04, train_time=1.655
+[gpua002:0/64] 2024-01-18 12:06:11,797 (trainer:753) INFO: 10epoch:train:4001-4100batch: iter_time=9.378e-05, forward_time=0.227, loss_ctc=125.297, loss_interctc_layer6=111.731, loss_interctc_layer12=94.303, loss_interctc_layer15=87.219, loss_interctc_layer21=128.309, loss=109.372, backward_time=0.419, grad_norm=71.552, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.138, optim0_lr0=1.314e-04, train_time=1.805
+[gpua002:0/64] 2024-01-18 12:08:36,935 (trainer:753) INFO: 10epoch:train:4101-4200batch: iter_time=8.751e-05, forward_time=0.141, loss_ctc=87.728, loss_interctc_layer6=91.388, loss_interctc_layer12=77.254, loss_interctc_layer15=71.476, loss_interctc_layer21=89.573, loss=83.484, backward_time=0.321, grad_norm=66.328, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.313e-04, train_time=1.453
+[gpua002:0/64] 2024-01-18 12:11:37,432 (trainer:753) INFO: 10epoch:train:4201-4300batch: iter_time=8.813e-05, forward_time=0.175, loss_ctc=95.403, loss_interctc_layer6=90.427, loss_interctc_layer12=76.519, loss_interctc_layer15=70.561, loss_interctc_layer21=97.979, loss=86.178, backward_time=0.420, grad_norm=71.442, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.144, optim0_lr0=1.313e-04, train_time=1.805
+[gpua002:0/64] 2024-01-18 12:14:24,125 (trainer:753) INFO: 10epoch:train:4301-4400batch: iter_time=9.011e-05, forward_time=0.143, loss_ctc=93.193, loss_interctc_layer6=92.108, loss_interctc_layer12=77.363, loss_interctc_layer15=71.285, loss_interctc_layer21=95.193, loss=85.828, backward_time=0.341, grad_norm=55.006, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.312e-04, train_time=1.666
+[gpua002:0/64] 2024-01-18 12:18:13,123 (trainer:753) INFO: 10epoch:train:4401-4500batch: iter_time=8.976e-05, forward_time=0.161, loss_ctc=90.931, loss_interctc_layer6=94.776, loss_interctc_layer12=80.135, loss_interctc_layer15=74.221, loss_interctc_layer21=92.918, loss=86.596, backward_time=0.556, grad_norm=60.959, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.137, optim0_lr0=1.312e-04, train_time=2.290
+[gpua002:0/64] 2024-01-18 12:20:51,333 (trainer:753) INFO: 10epoch:train:4501-4600batch: iter_time=8.773e-05, forward_time=0.142, loss_ctc=98.815, loss_interctc_layer6=93.043, loss_interctc_layer12=78.722, loss_interctc_layer15=72.838, loss_interctc_layer21=101.310, loss=88.946, backward_time=0.357, grad_norm=75.445, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.311e-04, train_time=1.579
+[gpua002:0/64] 2024-01-18 12:23:19,757 (trainer:753) INFO: 10epoch:train:4601-4700batch: iter_time=8.852e-05, forward_time=0.142, loss_ctc=90.182, loss_interctc_layer6=94.999, loss_interctc_layer12=80.378, loss_interctc_layer15=74.420, loss_interctc_layer21=92.192, loss=86.434, backward_time=0.318, grad_norm=62.071, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.311e-04, train_time=1.487
+[gpua002:0/64] 2024-01-18 12:26:26,861 (trainer:753) INFO: 10epoch:train:4701-4800batch: iter_time=8.786e-05, forward_time=0.205, loss_ctc=94.653, loss_interctc_layer6=104.955, loss_interctc_layer12=89.502, loss_interctc_layer15=83.360, loss_interctc_layer21=97.408, loss=93.976, backward_time=0.366, grad_norm=75.777, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.141, optim0_lr0=1.310e-04, train_time=1.871
+[gpua002:0/64] 2024-01-18 12:31:56,228 (trainer:753) INFO: 10epoch:train:4801-4900batch: iter_time=0.002, forward_time=0.202, loss_ctc=85.058, loss_interctc_layer6=93.433, loss_interctc_layer12=80.031, loss_interctc_layer15=74.225, loss_interctc_layer21=87.034, loss=83.956, backward_time=0.535, grad_norm=70.661, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.144, optim0_lr0=1.310e-04, train_time=3.291
+[gpua002:0/64] 2024-01-18 12:36:25,844 (trainer:753) INFO: 10epoch:train:4901-5000batch: iter_time=8.645e-05, forward_time=0.142, loss_ctc=88.757, loss_interctc_layer6=91.971, loss_interctc_layer12=78.240, loss_interctc_layer15=72.442, loss_interctc_layer21=90.579, loss=84.398, backward_time=0.560, grad_norm=76.866, clip=100.000, loss_scale=3.869e+25, optim_step_time=0.136, optim0_lr0=1.310e-04, train_time=2.698
+[gpua002:0/64] 2024-01-18 12:36:45,936 (multiple_iter_factory:32) INFO: Building 4th iter-factory...
+[gpua002:0/64] 2024-01-18 12:37:06,751 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 12:37:10,894 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe6f1e650f0>)
+[gpua002:0/64] 2024-01-18 12:37:10,894 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, 
+[gpua002:0/64] 2024-01-18 12:37:10,897 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 12:55:10,267 (trainer:753) INFO: 10epoch:train:5001-5100batch: iter_time=5.750, forward_time=0.220, loss_ctc=101.028, loss_interctc_layer6=97.913, loss_interctc_layer12=83.056, loss_interctc_layer15=77.130, loss_interctc_layer21=102.973, loss=92.420, backward_time=0.395, grad_norm=60.981, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.138, optim0_lr0=1.309e-04, train_time=11.242
+[gpua002:0/64] 2024-01-18 12:58:29,401 (trainer:753) INFO: 10epoch:train:5101-5200batch: iter_time=1.008e-04, forward_time=0.340, loss_ctc=76.896, loss_interctc_layer6=84.129, loss_interctc_layer12=71.052, loss_interctc_layer15=65.770, loss_interctc_layer21=78.429, loss=75.255, backward_time=0.466, grad_norm=60.060, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.150, optim0_lr0=1.309e-04, train_time=1.990
+[gpua002:0/64] 2024-01-18 13:03:30,095 (trainer:753) INFO: 10epoch:train:5201-5300batch: iter_time=8.678e-05, forward_time=0.570, loss_ctc=124.147, loss_interctc_layer6=106.345, loss_interctc_layer12=90.031, loss_interctc_layer15=83.504, loss_interctc_layer21=127.442, loss=106.294, backward_time=0.519, grad_norm=66.097, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.153, optim0_lr0=1.308e-04, train_time=3.009
+[gpua002:0/64] 2024-01-18 13:07:54,010 (trainer:753) INFO: 10epoch:train:5301-5400batch: iter_time=9.469e-05, forward_time=0.303, loss_ctc=108.483, loss_interctc_layer6=108.611, loss_interctc_layer12=91.990, loss_interctc_layer15=85.200, loss_interctc_layer21=110.877, loss=101.032, backward_time=0.517, grad_norm=66.892, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.142, optim0_lr0=1.308e-04, train_time=2.636
+[gpua002:0/64] 2024-01-18 13:13:10,508 (trainer:753) INFO: 10epoch:train:5401-5500batch: iter_time=8.562e-05, forward_time=0.603, loss_ctc=80.992, loss_interctc_layer6=82.762, loss_interctc_layer12=69.768, loss_interctc_layer15=64.598, loss_interctc_layer21=82.755, loss=76.175, backward_time=0.596, grad_norm=60.207, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.168, optim0_lr0=1.307e-04, train_time=3.166
+[gpua002:0/64] 2024-01-18 13:16:53,481 (trainer:753) INFO: 10epoch:train:5501-5600batch: iter_time=8.541e-05, forward_time=0.533, loss_ctc=97.457, loss_interctc_layer6=94.333, loss_interctc_layer12=79.291, loss_interctc_layer15=73.321, loss_interctc_layer21=99.465, loss=88.773, backward_time=0.458, grad_norm=81.371, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.166, optim0_lr0=1.307e-04, train_time=2.228
+[gpua002:0/64] 2024-01-18 13:21:13,743 (trainer:753) INFO: 10epoch:train:5601-5700batch: iter_time=9.549e-04, forward_time=0.426, loss_ctc=93.140, loss_interctc_layer6=97.025, loss_interctc_layer12=81.775, loss_interctc_layer15=75.834, loss_interctc_layer21=94.999, loss=88.554, backward_time=0.472, grad_norm=74.148, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.161, optim0_lr0=1.306e-04, train_time=2.600
+[gpua002:0/64] 2024-01-18 13:25:45,037 (trainer:753) INFO: 10epoch:train:5701-5800batch: iter_time=9.253e-05, forward_time=0.277, loss_ctc=92.827, loss_interctc_layer6=93.283, loss_interctc_layer12=78.443, loss_interctc_layer15=72.690, loss_interctc_layer21=94.833, loss=86.415, backward_time=0.479, grad_norm=62.284, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.149, optim0_lr0=1.306e-04, train_time=2.718
+[gpua002:0/64] 2024-01-18 13:30:00,811 (trainer:753) INFO: 10epoch:train:5801-5900batch: iter_time=0.001, forward_time=0.236, loss_ctc=93.984, loss_interctc_layer6=91.209, loss_interctc_layer12=77.599, loss_interctc_layer15=72.026, loss_interctc_layer21=95.588, loss=86.081, backward_time=0.582, grad_norm=59.342, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.145, optim0_lr0=1.305e-04, train_time=2.555
+[gpua002:0/64] 2024-01-18 13:34:51,088 (trainer:753) INFO: 10epoch:train:5901-6000batch: iter_time=1.895e-04, forward_time=0.489, loss_ctc=85.830, loss_interctc_layer6=93.139, loss_interctc_layer12=79.014, loss_interctc_layer15=72.742, loss_interctc_layer21=87.959, loss=83.737, backward_time=0.561, grad_norm=60.827, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.153, optim0_lr0=1.305e-04, train_time=2.903
+[gpua002:0/64] 2024-01-18 13:39:00,072 (trainer:753) INFO: 10epoch:train:6001-6100batch: iter_time=4.123e-04, forward_time=0.360, loss_ctc=104.128, loss_interctc_layer6=109.592, loss_interctc_layer12=94.022, loss_interctc_layer15=87.867, loss_interctc_layer21=105.309, loss=100.184, backward_time=0.482, grad_norm=121.286, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.151, optim0_lr0=1.304e-04, train_time=2.490
+[gpua002:0/64] 2024-01-18 13:41:45,803 (trainer:753) INFO: 10epoch:train:6101-6200batch: iter_time=8.279e-05, forward_time=0.215, loss_ctc=82.546, loss_interctc_layer6=91.161, loss_interctc_layer12=77.813, loss_interctc_layer15=72.323, loss_interctc_layer21=84.049, loss=81.578, backward_time=0.377, grad_norm=58.156, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.144, optim0_lr0=1.304e-04, train_time=1.658
+[gpua002:0/64] 2024-01-18 13:44:06,007 (multiple_iter_factory:32) INFO: Building 5th iter-factory...
+[gpua002:0/64] 2024-01-18 13:44:25,897 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 13:44:29,756 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea924b9960>)
+[gpua002:0/64] 2024-01-18 13:44:29,756 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, 
+[gpua002:0/64] 2024-01-18 13:44:29,759 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 13:53:37,888 (trainer:753) INFO: 10epoch:train:6201-6300batch: iter_time=2.934, forward_time=0.336, loss_ctc=88.295, loss_interctc_layer6=89.645, loss_interctc_layer12=75.472, loss_interctc_layer15=69.307, loss_interctc_layer21=90.471, loss=82.638, backward_time=0.422, grad_norm=66.913, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.145, optim0_lr0=1.303e-04, train_time=7.121
+[gpua002:0/64] 2024-01-18 13:56:33,299 (trainer:753) INFO: 10epoch:train:6301-6400batch: iter_time=1.040e-04, forward_time=0.143, loss_ctc=92.301, loss_interctc_layer6=96.932, loss_interctc_layer12=82.213, loss_interctc_layer15=76.047, loss_interctc_layer21=94.759, loss=88.450, backward_time=0.333, grad_norm=62.055, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.137, optim0_lr0=1.303e-04, train_time=1.755
+[gpua002:0/64] 2024-01-18 13:59:18,947 (trainer:753) INFO: 10epoch:train:6401-6500batch: iter_time=9.761e-05, forward_time=0.143, loss_ctc=87.637, loss_interctc_layer6=91.140, loss_interctc_layer12=77.308, loss_interctc_layer15=71.787, loss_interctc_layer21=89.275, loss=83.429, backward_time=0.334, grad_norm=67.231, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.137, optim0_lr0=1.303e-04, train_time=1.656
+[gpua002:0/64] 2024-01-18 14:02:19,883 (trainer:753) INFO: 10epoch:train:6501-6600batch: iter_time=1.009e-04, forward_time=0.235, loss_ctc=117.244, loss_interctc_layer6=110.418, loss_interctc_layer12=93.053, loss_interctc_layer15=86.073, loss_interctc_layer21=120.314, loss=105.420, backward_time=0.378, grad_norm=70.783, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.148, optim0_lr0=1.302e-04, train_time=1.809
+[gpua002:0/64] 2024-01-18 14:06:18,523 (trainer:753) INFO: 10epoch:train:6601-6700batch: iter_time=9.846e-05, forward_time=0.278, loss_ctc=83.030, loss_interctc_layer6=89.897, loss_interctc_layer12=75.928, loss_interctc_layer15=70.437, loss_interctc_layer21=84.915, loss=80.842, backward_time=0.514, grad_norm=67.571, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.140, optim0_lr0=1.302e-04, train_time=2.385
+[gpua002:0/64] 2024-01-18 14:10:26,482 (trainer:753) INFO: 10epoch:train:6701-6800batch: iter_time=1.099e-04, forward_time=0.142, loss_ctc=87.359, loss_interctc_layer6=89.806, loss_interctc_layer12=76.178, loss_interctc_layer15=70.112, loss_interctc_layer21=89.837, loss=82.658, backward_time=0.463, grad_norm=100.176, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.137, optim0_lr0=1.301e-04, train_time=2.481
+[gpua002:0/64] 2024-01-18 14:13:21,337 (trainer:753) INFO: 10epoch:train:6801-6900batch: iter_time=1.042e-04, forward_time=0.141, loss_ctc=89.754, loss_interctc_layer6=91.491, loss_interctc_layer12=76.552, loss_interctc_layer15=70.585, loss_interctc_layer21=91.894, loss=84.055, backward_time=0.365, grad_norm=53.679, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.137, optim0_lr0=1.301e-04, train_time=1.748
+[gpua002:0/64] 2024-01-18 14:17:22,216 (trainer:753) INFO: 10epoch:train:6901-7000batch: iter_time=1.016e-04, forward_time=0.211, loss_ctc=87.617, loss_interctc_layer6=95.113, loss_interctc_layer12=80.098, loss_interctc_layer15=74.135, loss_interctc_layer21=89.563, loss=85.305, backward_time=0.523, grad_norm=146.239, clip=100.000, loss_scale=7.737e+25, optim_step_time=0.145, optim0_lr0=1.300e-04, train_time=2.409
+[gpua002:0/64] 2024-01-18 14:20:22,818 (trainer:753) INFO: 10epoch:train:7001-7100batch: iter_time=1.061e-04, forward_time=0.245, loss_ctc=92.431, loss_interctc_layer6=92.291, loss_interctc_layer12=77.977, loss_interctc_layer15=72.438, loss_interctc_layer21=94.497, loss=85.927, backward_time=0.396, grad_norm=63.446, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.145, optim0_lr0=1.300e-04, train_time=1.804
+[gpua002:0/64] 2024-01-18 14:23:02,813 (trainer:753) INFO: 10epoch:train:7101-7200batch: iter_time=1.037e-04, forward_time=0.141, loss_ctc=86.009, loss_interctc_layer6=94.587, loss_interctc_layer12=80.023, loss_interctc_layer15=73.882, loss_interctc_layer21=87.994, loss=84.499, backward_time=0.304, grad_norm=57.047, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.136, optim0_lr0=1.299e-04, train_time=1.601
+[gpua002:0/64] 2024-01-18 14:27:15,380 (trainer:753) INFO: 10epoch:train:7201-7300batch: iter_time=9.886e-05, forward_time=0.144, loss_ctc=93.482, loss_interctc_layer6=103.810, loss_interctc_layer12=88.506, loss_interctc_layer15=82.278, loss_interctc_layer21=94.785, loss=92.572, backward_time=0.422, grad_norm=73.957, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.136, optim0_lr0=1.299e-04, train_time=2.526
+[gpua002:0/64] 2024-01-18 14:30:28,025 (trainer:753) INFO: 10epoch:train:7301-7400batch: iter_time=9.485e-05, forward_time=0.196, loss_ctc=81.393, loss_interctc_layer6=92.787, loss_interctc_layer12=79.639, loss_interctc_layer15=73.478, loss_interctc_layer21=82.775, loss=82.014, backward_time=0.417, grad_norm=66.394, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.144, optim0_lr0=1.298e-04, train_time=1.926
+[gpua002:0/64] 2024-01-18 14:34:20,706 (trainer:753) INFO: 10epoch:train:7401-7500batch: iter_time=8.951e-05, forward_time=0.301, loss_ctc=86.225, loss_interctc_layer6=91.543, loss_interctc_layer12=77.677, loss_interctc_layer15=71.689, loss_interctc_layer21=88.020, loss=83.031, backward_time=0.488, grad_norm=57.928, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.149, optim0_lr0=1.298e-04, train_time=2.322
+[gpua002:0/64] 2024-01-18 14:34:41,007 (multiple_iter_factory:32) INFO: Building 6th iter-factory...
+[gpua002:0/64] 2024-01-18 14:35:00,353 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 14:35:04,350 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe702af7dc0>)
+[gpua002:0/64] 2024-01-18 14:35:04,350 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, 
+[gpua002:0/64] 2024-01-18 14:35:04,353 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 14:51:25,926 (trainer:753) INFO: 10epoch:train:7501-7600batch: iter_time=4.659, forward_time=0.631, loss_ctc=101.688, loss_interctc_layer6=97.189, loss_interctc_layer12=82.198, loss_interctc_layer15=76.032, loss_interctc_layer21=104.190, loss=92.259, backward_time=0.366, grad_norm=96.213, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.164, optim0_lr0=1.298e-04, train_time=10.256
+[gpua002:0/64] 2024-01-18 14:54:18,041 (trainer:753) INFO: 10epoch:train:7601-7700batch: iter_time=8.979e-05, forward_time=0.145, loss_ctc=76.950, loss_interctc_layer6=84.325, loss_interctc_layer12=71.214, loss_interctc_layer15=66.050, loss_interctc_layer21=78.623, loss=75.432, backward_time=0.321, grad_norm=54.920, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.136, optim0_lr0=1.297e-04, train_time=1.722
+[gpua002:0/64] 2024-01-18 14:58:11,276 (trainer:753) INFO: 10epoch:train:7701-7800batch: iter_time=1.029e-04, forward_time=0.142, loss_ctc=122.671, loss_interctc_layer6=104.785, loss_interctc_layer12=89.116, loss_interctc_layer15=82.333, loss_interctc_layer21=125.696, loss=104.920, backward_time=0.465, grad_norm=68.114, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.136, optim0_lr0=1.297e-04, train_time=2.332
+[gpua002:0/64] 2024-01-18 15:02:57,360 (trainer:753) INFO: 10epoch:train:7801-7900batch: iter_time=2.820e-04, forward_time=0.426, loss_ctc=107.144, loss_interctc_layer6=107.202, loss_interctc_layer12=90.651, loss_interctc_layer15=83.710, loss_interctc_layer21=109.522, loss=99.646, backward_time=0.534, grad_norm=74.885, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.156, optim0_lr0=1.296e-04, train_time=2.857
+[gpua002:0/64] 2024-01-18 15:05:50,568 (trainer:753) INFO: 10epoch:train:7901-8000batch: iter_time=8.639e-05, forward_time=0.175, loss_ctc=80.618, loss_interctc_layer6=82.443, loss_interctc_layer12=69.347, loss_interctc_layer15=64.316, loss_interctc_layer21=82.120, loss=75.769, backward_time=0.353, grad_norm=56.950, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.137, optim0_lr0=1.296e-04, train_time=1.735
+[gpua002:0/64] 2024-01-18 15:08:34,565 (trainer:753) INFO: 10epoch:train:8001-8100batch: iter_time=8.590e-05, forward_time=0.142, loss_ctc=97.544, loss_interctc_layer6=93.705, loss_interctc_layer12=78.632, loss_interctc_layer15=72.492, loss_interctc_layer21=99.961, loss=88.467, backward_time=0.333, grad_norm=65.119, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.135, optim0_lr0=1.295e-04, train_time=1.637
+[gpua002:0/64] 2024-01-18 15:12:34,341 (trainer:753) INFO: 10epoch:train:8101-8200batch: iter_time=8.549e-05, forward_time=0.142, loss_ctc=90.341, loss_interctc_layer6=95.128, loss_interctc_layer12=80.153, loss_interctc_layer15=74.110, loss_interctc_layer21=92.292, loss=86.405, backward_time=0.410, grad_norm=55.332, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.135, optim0_lr0=1.295e-04, train_time=2.400
+[gpua002:0/64] 2024-01-18 15:18:02,302 (trainer:753) INFO: 10epoch:train:8201-8300batch: iter_time=9.515e-04, forward_time=0.373, loss_ctc=93.548, loss_interctc_layer6=92.420, loss_interctc_layer12=77.794, loss_interctc_layer15=71.965, loss_interctc_layer21=95.669, loss=86.279, backward_time=0.667, grad_norm=65.438, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.151, optim0_lr0=1.294e-04, train_time=3.278
+[gpua002:0/64] 2024-01-18 15:22:04,304 (trainer:753) INFO: 10epoch:train:8301-8400batch: iter_time=9.417e-05, forward_time=0.193, loss_ctc=92.846, loss_interctc_layer6=91.359, loss_interctc_layer12=77.078, loss_interctc_layer15=71.587, loss_interctc_layer21=94.658, loss=85.506, backward_time=0.493, grad_norm=80.333, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.138, optim0_lr0=1.294e-04, train_time=2.420
+[gpua002:0/64] 2024-01-18 15:24:53,665 (trainer:753) INFO: 10epoch:train:8401-8500batch: iter_time=8.496e-05, forward_time=0.142, loss_ctc=85.250, loss_interctc_layer6=93.323, loss_interctc_layer12=78.792, loss_interctc_layer15=72.729, loss_interctc_layer21=87.211, loss=83.461, backward_time=0.351, grad_norm=87.890, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.135, optim0_lr0=1.293e-04, train_time=1.695
+[gpua002:0/64] 2024-01-18 15:28:14,330 (trainer:753) INFO: 10epoch:train:8501-8600batch: iter_time=9.074e-05, forward_time=0.142, loss_ctc=102.924, loss_interctc_layer6=109.532, loss_interctc_layer12=93.326, loss_interctc_layer15=86.531, loss_interctc_layer21=104.762, loss=99.415, backward_time=0.399, grad_norm=106.276, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.135, optim0_lr0=1.293e-04, train_time=2.005
+[gpua002:0/64] 2024-01-18 15:33:24,798 (trainer:753) INFO: 10epoch:train:8601-8700batch: iter_time=8.945e-05, forward_time=0.389, loss_ctc=81.509, loss_interctc_layer6=91.572, loss_interctc_layer12=78.017, loss_interctc_layer15=72.497, loss_interctc_layer21=83.129, loss=81.345, backward_time=0.747, grad_norm=65.199, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.154, optim0_lr0=1.293e-04, train_time=3.100
+[gpua002:0/64] 2024-01-18 15:35:27,074 (multiple_iter_factory:32) INFO: Building 7th iter-factory...
+[gpua002:0/64] 2024-01-18 15:35:47,892 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 15:35:51,667 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7feaace776a0>)
+[gpua002:0/64] 2024-01-18 15:35:51,667 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, 
+[gpua002:0/64] 2024-01-18 15:35:51,671 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 15:49:31,150 (trainer:753) INFO: 10epoch:train:8701-8800batch: iter_time=4.118, forward_time=0.164, loss_ctc=88.635, loss_interctc_layer6=89.740, loss_interctc_layer12=75.091, loss_interctc_layer15=69.107, loss_interctc_layer21=90.707, loss=82.656, backward_time=0.382, grad_norm=55.364, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.140, optim0_lr0=1.292e-04, train_time=9.669
+[gpua002:0/64] 2024-01-18 15:51:39,024 (trainer:753) INFO: 10epoch:train:8801-8900batch: iter_time=9.808e-05, forward_time=0.142, loss_ctc=90.331, loss_interctc_layer6=95.921, loss_interctc_layer12=80.927, loss_interctc_layer15=74.910, loss_interctc_layer21=92.354, loss=86.888, backward_time=0.301, grad_norm=59.661, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.136, optim0_lr0=1.292e-04, train_time=1.279
+[gpua002:0/64] 2024-01-18 15:55:30,282 (trainer:753) INFO: 10epoch:train:8901-9000batch: iter_time=9.482e-05, forward_time=0.142, loss_ctc=87.644, loss_interctc_layer6=90.974, loss_interctc_layer12=77.269, loss_interctc_layer15=71.543, loss_interctc_layer21=89.586, loss=83.403, backward_time=0.501, grad_norm=64.697, clip=100.000, loss_scale=1.547e+26, optim_step_time=0.136, optim0_lr0=1.291e-04, train_time=2.312
+[gpua002:0/64] 2024-01-18 15:58:22,694 (trainer:753) INFO: 10epoch:train:9001-9100batch: iter_time=9.271e-05, forward_time=0.143, loss_ctc=116.535, loss_interctc_layer6=111.234, loss_interctc_layer12=93.829, loss_interctc_layer15=86.553, loss_interctc_layer21=119.643, loss=105.559, backward_time=0.358, grad_norm=92.408, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.291e-04, train_time=1.724
+[gpua002:0/64] 2024-01-18 16:02:51,974 (trainer:753) INFO: 10epoch:train:9101-9200batch: iter_time=9.287e-05, forward_time=0.266, loss_ctc=81.970, loss_interctc_layer6=89.491, loss_interctc_layer12=75.403, loss_interctc_layer15=69.598, loss_interctc_layer21=83.816, loss=80.056, backward_time=0.591, grad_norm=61.897, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.162, optim0_lr0=1.290e-04, train_time=2.691
+[gpua002:0/64] 2024-01-18 16:06:51,878 (trainer:753) INFO: 10epoch:train:9201-9300batch: iter_time=1.038e-04, forward_time=0.143, loss_ctc=86.992, loss_interctc_layer6=89.316, loss_interctc_layer12=75.484, loss_interctc_layer15=69.631, loss_interctc_layer21=89.249, loss=82.134, backward_time=0.451, grad_norm=63.953, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.290e-04, train_time=2.399
+[gpua002:0/64] 2024-01-18 16:10:55,363 (trainer:753) INFO: 10epoch:train:9301-9400batch: iter_time=1.072e-04, forward_time=0.155, loss_ctc=89.155, loss_interctc_layer6=90.913, loss_interctc_layer12=75.993, loss_interctc_layer15=70.023, loss_interctc_layer21=91.268, loss=83.470, backward_time=0.410, grad_norm=61.540, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.137, optim0_lr0=1.289e-04, train_time=2.436
+[gpua002:0/64] 2024-01-18 16:13:55,215 (trainer:753) INFO: 10epoch:train:9401-9500batch: iter_time=9.613e-05, forward_time=0.151, loss_ctc=85.557, loss_interctc_layer6=93.504, loss_interctc_layer12=78.575, loss_interctc_layer15=72.822, loss_interctc_layer21=87.498, loss=83.591, backward_time=0.391, grad_norm=85.279, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.289e-04, train_time=1.793
+[gpua002:0/64] 2024-01-18 16:17:40,386 (trainer:753) INFO: 10epoch:train:9501-9600batch: iter_time=9.191e-05, forward_time=0.141, loss_ctc=92.446, loss_interctc_layer6=91.756, loss_interctc_layer12=77.292, loss_interctc_layer15=71.416, loss_interctc_layer21=94.449, loss=85.472, backward_time=0.482, grad_norm=63.965, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.135, optim0_lr0=1.289e-04, train_time=2.256
+[gpua002:0/64] 2024-01-18 16:20:56,752 (trainer:753) INFO: 10epoch:train:9601-9700batch: iter_time=1.003e-04, forward_time=0.143, loss_ctc=86.113, loss_interctc_layer6=93.978, loss_interctc_layer12=79.242, loss_interctc_layer15=73.156, loss_interctc_layer21=88.124, loss=84.123, backward_time=0.357, grad_norm=53.981, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.135, optim0_lr0=1.288e-04, train_time=1.964
+[gpua002:0/64] 2024-01-18 16:24:50,348 (trainer:753) INFO: 10epoch:train:9701-9800batch: iter_time=9.319e-05, forward_time=0.142, loss_ctc=91.172, loss_interctc_layer6=103.546, loss_interctc_layer12=87.100, loss_interctc_layer15=81.223, loss_interctc_layer21=93.113, loss=91.231, backward_time=0.568, grad_norm=75.255, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.288e-04, train_time=2.336
+[gpua002:0/64] 2024-01-18 16:28:51,363 (trainer:753) INFO: 10epoch:train:9801-9900batch: iter_time=9.045e-05, forward_time=0.141, loss_ctc=81.781, loss_interctc_layer6=92.610, loss_interctc_layer12=78.339, loss_interctc_layer15=73.299, loss_interctc_layer21=83.282, loss=81.862, backward_time=0.435, grad_norm=72.808, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.287e-04, train_time=2.410
+[gpua002:0/64] 2024-01-18 16:33:01,912 (trainer:753) INFO: 10epoch:train:9901-10000batch: iter_time=8.795e-05, forward_time=0.141, loss_ctc=85.731, loss_interctc_layer6=90.715, loss_interctc_layer12=76.768, loss_interctc_layer15=71.198, loss_interctc_layer21=87.589, loss=82.400, backward_time=0.394, grad_norm=57.380, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.135, optim0_lr0=1.287e-04, train_time=2.505
+[gpua002:0/64] 2024-01-18 16:33:21,942 (multiple_iter_factory:32) INFO: Building 8th iter-factory...
+[gpua002:0/64] 2024-01-18 16:33:42,003 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 16:33:45,740 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea8d0b3e20>)
+[gpua002:0/64] 2024-01-18 16:33:45,740 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, 
+[gpua002:0/64] 2024-01-18 16:33:45,743 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 16:49:42,320 (trainer:753) INFO: 10epoch:train:10001-10100batch: iter_time=8.541, forward_time=0.261, loss_ctc=100.388, loss_interctc_layer6=96.559, loss_interctc_layer12=81.565, loss_interctc_layer15=75.597, loss_interctc_layer21=102.940, loss=91.410, backward_time=0.315, grad_norm=69.236, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.138, optim0_lr0=1.286e-04, train_time=10.004
+[gpua002:0/64] 2024-01-18 16:54:43,013 (trainer:753) INFO: 10epoch:train:10101-10200batch: iter_time=8.703e-05, forward_time=0.182, loss_ctc=75.080, loss_interctc_layer6=83.142, loss_interctc_layer12=69.874, loss_interctc_layer15=64.461, loss_interctc_layer21=76.837, loss=73.879, backward_time=0.543, grad_norm=54.992, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.146, optim0_lr0=1.286e-04, train_time=3.005
+[gpua002:0/64] 2024-01-18 17:04:04,012 (trainer:753) INFO: 10epoch:train:10201-10300batch: iter_time=0.005, forward_time=1.877, loss_ctc=123.353, loss_interctc_layer6=105.052, loss_interctc_layer12=88.821, loss_interctc_layer15=82.291, loss_interctc_layer21=126.481, loss=105.199, backward_time=1.046, grad_norm=73.228, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.308, optim0_lr0=1.285e-04, train_time=5.606
+[gpua002:0/64] 2024-01-18 17:11:20,988 (trainer:753) INFO: 10epoch:train:10301-10400batch: iter_time=0.002, forward_time=1.089, loss_ctc=106.407, loss_interctc_layer6=107.309, loss_interctc_layer12=90.471, loss_interctc_layer15=83.845, loss_interctc_layer21=109.050, loss=99.416, backward_time=0.959, grad_norm=75.345, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.283, optim0_lr0=1.285e-04, train_time=4.373
+[gpua002:0/64] 2024-01-18 17:14:39,867 (trainer:753) INFO: 10epoch:train:10401-10500batch: iter_time=8.637e-05, forward_time=0.142, loss_ctc=80.431, loss_interctc_layer6=82.138, loss_interctc_layer12=69.046, loss_interctc_layer15=63.893, loss_interctc_layer21=81.986, loss=75.499, backward_time=0.384, grad_norm=71.385, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.285e-04, train_time=1.992
+[gpua002:0/64] 2024-01-18 17:18:40,609 (trainer:753) INFO: 10epoch:train:10501-10600batch: iter_time=8.938e-05, forward_time=0.142, loss_ctc=97.490, loss_interctc_layer6=93.867, loss_interctc_layer12=78.706, loss_interctc_layer15=72.661, loss_interctc_layer21=100.012, loss=88.547, backward_time=0.523, grad_norm=65.062, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.284e-04, train_time=2.407
+[gpua002:0/64] 2024-01-18 17:21:44,594 (trainer:753) INFO: 10epoch:train:10601-10700batch: iter_time=8.854e-05, forward_time=0.142, loss_ctc=90.384, loss_interctc_layer6=95.002, loss_interctc_layer12=79.905, loss_interctc_layer15=73.885, loss_interctc_layer21=92.365, loss=86.308, backward_time=0.426, grad_norm=69.633, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.284e-04, train_time=1.840
+[gpua002:0/64] 2024-01-18 17:26:31,418 (trainer:753) INFO: 10epoch:train:10701-10800batch: iter_time=9.409e-05, forward_time=0.144, loss_ctc=92.568, loss_interctc_layer6=91.755, loss_interctc_layer12=77.111, loss_interctc_layer15=71.200, loss_interctc_layer21=94.621, loss=85.451, backward_time=0.514, grad_norm=61.564, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.283e-04, train_time=2.868
+[gpua002:0/64] 2024-01-18 17:32:01,623 (trainer:753) INFO: 10epoch:train:10801-10900batch: iter_time=0.002, forward_time=0.736, loss_ctc=91.755, loss_interctc_layer6=90.484, loss_interctc_layer12=76.394, loss_interctc_layer15=70.812, loss_interctc_layer21=93.652, loss=84.619, backward_time=0.556, grad_norm=56.273, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.153, optim0_lr0=1.283e-04, train_time=3.300
+[gpua002:0/64] 2024-01-18 17:36:37,407 (trainer:753) INFO: 10epoch:train:10901-11000batch: iter_time=9.075e-05, forward_time=0.143, loss_ctc=84.363, loss_interctc_layer6=92.261, loss_interctc_layer12=77.694, loss_interctc_layer15=71.425, loss_interctc_layer21=86.027, loss=82.354, backward_time=0.532, grad_norm=64.916, clip=100.000, loss_scale=3.095e+26, optim_step_time=0.136, optim0_lr0=1.282e-04, train_time=2.756
+[gpua002:0/64] 2024-01-18 17:42:14,255 (trainer:753) INFO: 10epoch:train:11001-11100batch: iter_time=4.543e-04, forward_time=0.362, loss_ctc=102.101, loss_interctc_layer6=108.127, loss_interctc_layer12=92.060, loss_interctc_layer15=85.849, loss_interctc_layer21=104.255, loss=98.478, backward_time=0.501, grad_norm=71.861, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.154, optim0_lr0=1.282e-04, train_time=3.368
+[gpua002:0/64] 2024-01-18 17:46:47,915 (trainer:753) INFO: 10epoch:train:11101-11200batch: iter_time=0.004, forward_time=0.586, loss_ctc=81.686, loss_interctc_layer6=90.836, loss_interctc_layer12=77.417, loss_interctc_layer15=72.070, loss_interctc_layer21=83.364, loss=81.074, backward_time=0.556, grad_norm=54.592, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.200, optim0_lr0=1.281e-04, train_time=2.734
+[gpua002:0/64] 2024-01-18 17:49:46,575 (multiple_iter_factory:32) INFO: Building 9th iter-factory...
+[gpua002:0/64] 2024-01-18 17:50:06,228 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 17:50:10,233 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7feaaee9bac0>)
+[gpua002:0/64] 2024-01-18 17:50:10,233 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, 
+[gpua002:0/64] 2024-01-18 17:50:10,236 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 18:06:39,536 (trainer:753) INFO: 10epoch:train:11201-11300batch: iter_time=4.889, forward_time=0.334, loss_ctc=87.974, loss_interctc_layer6=89.675, loss_interctc_layer12=75.189, loss_interctc_layer15=69.103, loss_interctc_layer21=89.961, loss=82.381, backward_time=0.434, grad_norm=63.826, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.161, optim0_lr0=1.281e-04, train_time=11.923
+[gpua002:0/64] 2024-01-18 18:08:58,031 (trainer:753) INFO: 10epoch:train:11301-11400batch: iter_time=8.061e-05, forward_time=0.144, loss_ctc=89.855, loss_interctc_layer6=95.754, loss_interctc_layer12=80.826, loss_interctc_layer15=74.507, loss_interctc_layer21=92.397, loss=86.668, backward_time=0.312, grad_norm=85.703, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.136, optim0_lr0=1.281e-04, train_time=1.385
+[gpua002:0/64] 2024-01-18 18:13:15,604 (trainer:753) INFO: 10epoch:train:11401-11500batch: iter_time=9.141e-05, forward_time=0.151, loss_ctc=86.874, loss_interctc_layer6=90.226, loss_interctc_layer12=76.232, loss_interctc_layer15=70.728, loss_interctc_layer21=88.756, loss=82.563, backward_time=0.438, grad_norm=68.474, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.138, optim0_lr0=1.280e-04, train_time=2.572
+[gpua002:0/64] 2024-01-18 18:18:07,279 (trainer:753) INFO: 10epoch:train:11501-11600batch: iter_time=8.246e-05, forward_time=0.484, loss_ctc=114.879, loss_interctc_layer6=109.918, loss_interctc_layer12=92.584, loss_interctc_layer15=85.348, loss_interctc_layer21=117.930, loss=104.132, backward_time=0.515, grad_norm=71.977, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.147, optim0_lr0=1.280e-04, train_time=2.920
+[gpua002:0/64] 2024-01-18 18:22:00,003 (trainer:753) INFO: 10epoch:train:11601-11700batch: iter_time=8.091e-05, forward_time=0.141, loss_ctc=81.587, loss_interctc_layer6=89.149, loss_interctc_layer12=75.145, loss_interctc_layer15=69.415, loss_interctc_layer21=83.505, loss=79.760, backward_time=0.483, grad_norm=62.512, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.279e-04, train_time=2.327
+[gpua002:0/64] 2024-01-18 18:26:14,121 (trainer:753) INFO: 10epoch:train:11701-11800batch: iter_time=8.398e-05, forward_time=0.141, loss_ctc=86.324, loss_interctc_layer6=89.197, loss_interctc_layer12=75.209, loss_interctc_layer15=69.088, loss_interctc_layer21=88.316, loss=81.627, backward_time=0.539, grad_norm=55.757, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.279e-04, train_time=2.541
+[gpua002:0/64] 2024-01-18 18:30:14,256 (trainer:753) INFO: 10epoch:train:11801-11900batch: iter_time=8.909e-05, forward_time=0.178, loss_ctc=88.980, loss_interctc_layer6=91.052, loss_interctc_layer12=76.222, loss_interctc_layer15=70.085, loss_interctc_layer21=90.892, loss=83.446, backward_time=0.525, grad_norm=95.880, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.278e-04, train_time=2.400
+[gpua002:0/64] 2024-01-18 18:33:41,206 (trainer:753) INFO: 10epoch:train:11901-12000batch: iter_time=8.083e-05, forward_time=0.141, loss_ctc=84.778, loss_interctc_layer6=93.199, loss_interctc_layer12=78.732, loss_interctc_layer15=72.747, loss_interctc_layer21=86.727, loss=83.237, backward_time=0.437, grad_norm=55.715, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.278e-04, train_time=2.070
+[gpua002:0/64] 2024-01-18 18:37:28,128 (trainer:753) INFO: 10epoch:train:12001-12100batch: iter_time=8.388e-05, forward_time=0.141, loss_ctc=91.180, loss_interctc_layer6=91.478, loss_interctc_layer12=76.900, loss_interctc_layer15=71.084, loss_interctc_layer21=93.284, loss=84.786, backward_time=0.377, grad_norm=214.635, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.278e-04, train_time=2.269
+[gpua002:0/64] 2024-01-18 18:42:57,864 (trainer:753) INFO: 10epoch:train:12101-12200batch: iter_time=8.551e-05, forward_time=0.141, loss_ctc=85.189, loss_interctc_layer6=92.945, loss_interctc_layer12=78.439, loss_interctc_layer15=72.288, loss_interctc_layer21=87.190, loss=83.210, backward_time=0.622, grad_norm=66.788, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.277e-04, train_time=3.297
+[gpua002:0/64] 2024-01-18 18:46:57,198 (trainer:753) INFO: 10epoch:train:12201-12300batch: iter_time=0.010, forward_time=0.292, loss_ctc=89.876, loss_interctc_layer6=101.314, loss_interctc_layer12=86.221, loss_interctc_layer15=79.643, loss_interctc_layer21=92.349, loss=89.881, backward_time=0.415, grad_norm=73.204, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.147, optim0_lr0=1.277e-04, train_time=2.389
+[gpua002:0/64] 2024-01-18 18:49:48,951 (trainer:753) INFO: 10epoch:train:12301-12400batch: iter_time=8.547e-04, forward_time=0.274, loss_ctc=80.181, loss_interctc_layer6=91.684, loss_interctc_layer12=78.093, loss_interctc_layer15=71.975, loss_interctc_layer21=81.963, loss=80.779, backward_time=0.372, grad_norm=58.622, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.140, optim0_lr0=1.276e-04, train_time=1.720
+[gpua002:0/64] 2024-01-18 18:53:41,366 (trainer:753) INFO: 10epoch:train:12401-12500batch: iter_time=8.939e-05, forward_time=0.143, loss_ctc=84.679, loss_interctc_layer6=90.342, loss_interctc_layer12=76.397, loss_interctc_layer15=70.241, loss_interctc_layer21=86.458, loss=81.624, backward_time=0.419, grad_norm=64.690, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.136, optim0_lr0=1.276e-04, train_time=2.325
+[gpua002:0/64] 2024-01-18 18:54:01,451 (multiple_iter_factory:32) INFO: Building 10th iter-factory...
+[gpua002:0/64] 2024-01-18 18:54:21,539 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 18:54:25,351 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7feaace516f0>)
+[gpua002:0/64] 2024-01-18 18:54:25,352 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, 
+[gpua002:0/64] 2024-01-18 18:54:25,355 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 19:08:30,681 (trainer:753) INFO: 10epoch:train:12501-12600batch: iter_time=4.140, forward_time=0.141, loss_ctc=92.777, loss_interctc_layer6=95.732, loss_interctc_layer12=81.388, loss_interctc_layer15=74.895, loss_interctc_layer21=95.009, loss=87.960, backward_time=0.304, grad_norm=82.548, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.275e-04, train_time=8.893
+[gpua002:0/64] 2024-01-18 19:11:34,055 (trainer:753) INFO: 10epoch:train:12601-12700batch: iter_time=8.068e-05, forward_time=0.142, loss_ctc=72.593, loss_interctc_layer6=82.986, loss_interctc_layer12=69.825, loss_interctc_layer15=64.639, loss_interctc_layer21=74.364, loss=72.881, backward_time=0.398, grad_norm=54.526, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.275e-04, train_time=1.834
+[gpua002:0/64] 2024-01-18 19:14:38,704 (trainer:753) INFO: 10epoch:train:12701-12800batch: iter_time=8.070e-05, forward_time=0.141, loss_ctc=113.904, loss_interctc_layer6=103.762, loss_interctc_layer12=87.627, loss_interctc_layer15=81.021, loss_interctc_layer21=116.971, loss=100.657, backward_time=0.349, grad_norm=60.380, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.274e-04, train_time=1.845
+[gpua002:0/64] 2024-01-18 19:18:01,093 (trainer:753) INFO: 10epoch:train:12801-12900batch: iter_time=8.746e-05, forward_time=0.142, loss_ctc=102.013, loss_interctc_layer6=106.359, loss_interctc_layer12=89.675, loss_interctc_layer15=82.901, loss_interctc_layer21=104.167, loss=97.023, backward_time=0.393, grad_norm=69.439, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.135, optim0_lr0=1.274e-04, train_time=2.025
+[gpua002:0/64] 2024-01-18 19:21:11,345 (trainer:753) INFO: 10epoch:train:12901-13000batch: iter_time=8.914e-05, forward_time=0.141, loss_ctc=74.956, loss_interctc_layer6=81.357, loss_interctc_layer12=68.538, loss_interctc_layer15=63.153, loss_interctc_layer21=76.747, loss=72.950, backward_time=0.447, grad_norm=54.460, clip=100.000, loss_scale=6.190e+26, optim_step_time=0.136, optim0_lr0=1.274e-04, train_time=1.902
+[gpua002:0/64] 2024-01-18 19:24:24,401 (trainer:753) INFO: 10epoch:train:13001-13100batch: iter_time=8.571e-05, forward_time=0.142, loss_ctc=88.250, loss_interctc_layer6=92.868, loss_interctc_layer12=77.923, loss_interctc_layer15=71.667, loss_interctc_layer21=90.261, loss=84.194, backward_time=0.399, grad_norm=57.492, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.273e-04, train_time=1.930
+[gpua002:0/64] 2024-01-18 19:27:07,166 (trainer:753) INFO: 10epoch:train:13101-13200batch: iter_time=8.053e-05, forward_time=0.146, loss_ctc=88.294, loss_interctc_layer6=94.726, loss_interctc_layer12=79.722, loss_interctc_layer15=73.509, loss_interctc_layer21=90.348, loss=85.320, backward_time=0.377, grad_norm=73.502, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.138, optim0_lr0=1.273e-04, train_time=1.627
+[gpua002:0/64] 2024-01-18 19:31:02,708 (trainer:753) INFO: 10epoch:train:13201-13300batch: iter_time=8.439e-05, forward_time=0.226, loss_ctc=86.140, loss_interctc_layer6=91.222, loss_interctc_layer12=76.525, loss_interctc_layer15=70.619, loss_interctc_layer21=88.154, loss=82.532, backward_time=0.489, grad_norm=60.001, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.145, optim0_lr0=1.272e-04, train_time=2.355
+[gpua002:0/64] 2024-01-18 19:34:17,918 (trainer:753) INFO: 10epoch:train:13301-13400batch: iter_time=8.510e-05, forward_time=0.140, loss_ctc=88.703, loss_interctc_layer6=89.308, loss_interctc_layer12=75.069, loss_interctc_layer15=69.605, loss_interctc_layer21=90.777, loss=82.692, backward_time=0.370, grad_norm=55.744, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.272e-04, train_time=1.952
+[gpua002:0/64] 2024-01-18 19:37:31,512 (trainer:753) INFO: 10epoch:train:13401-13500batch: iter_time=8.168e-05, forward_time=0.142, loss_ctc=80.461, loss_interctc_layer6=92.133, loss_interctc_layer12=77.589, loss_interctc_layer15=71.734, loss_interctc_layer21=82.343, loss=80.852, backward_time=0.404, grad_norm=67.484, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.271e-04, train_time=1.936
+[gpua002:0/64] 2024-01-18 19:40:18,693 (trainer:753) INFO: 10epoch:train:13501-13600batch: iter_time=8.320e-05, forward_time=0.142, loss_ctc=100.764, loss_interctc_layer6=107.432, loss_interctc_layer12=92.105, loss_interctc_layer15=86.089, loss_interctc_layer21=102.586, loss=97.795, backward_time=0.378, grad_norm=78.322, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.271e-04, train_time=1.672
+[gpua002:0/64] 2024-01-18 19:43:00,941 (trainer:753) INFO: 10epoch:train:13601-13700batch: iter_time=8.341e-05, forward_time=0.141, loss_ctc=78.287, loss_interctc_layer6=89.802, loss_interctc_layer12=76.464, loss_interctc_layer15=71.145, loss_interctc_layer21=79.796, loss=79.099, backward_time=0.398, grad_norm=201.721, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.271e-04, train_time=1.622
+[gpua002:0/64] 2024-01-18 19:44:53,412 (multiple_iter_factory:32) INFO: Building 11th iter-factory...
+[gpua002:0/64] 2024-01-18 19:45:12,759 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 19:45:16,265 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe716f3cf40>)
+[gpua002:0/64] 2024-01-18 19:45:16,265 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, 
+[gpua002:0/64] 2024-01-18 19:45:16,268 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 19:53:02,802 (trainer:753) INFO: 10epoch:train:13701-13800batch: iter_time=4.324, forward_time=0.152, loss_ctc=86.718, loss_interctc_layer6=88.406, loss_interctc_layer12=73.994, loss_interctc_layer15=67.908, loss_interctc_layer21=88.549, loss=81.115, backward_time=0.369, grad_norm=58.864, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.136, optim0_lr0=1.270e-04, train_time=6.018
+[gpua002:0/64] 2024-01-18 19:55:29,807 (trainer:753) INFO: 10epoch:train:13801-13900batch: iter_time=8.094e-05, forward_time=0.142, loss_ctc=88.660, loss_interctc_layer6=94.790, loss_interctc_layer12=80.202, loss_interctc_layer15=74.039, loss_interctc_layer21=90.927, loss=85.724, backward_time=0.295, grad_norm=62.051, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.270e-04, train_time=1.470
+[gpua002:0/64] 2024-01-18 19:58:28,165 (trainer:753) INFO: 10epoch:train:13901-14000batch: iter_time=8.331e-05, forward_time=0.226, loss_ctc=85.844, loss_interctc_layer6=89.689, loss_interctc_layer12=75.948, loss_interctc_layer15=70.132, loss_interctc_layer21=87.461, loss=81.815, backward_time=0.398, grad_norm=105.589, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.145, optim0_lr0=1.269e-04, train_time=1.783
+[gpua002:0/64] 2024-01-18 20:01:39,072 (trainer:753) INFO: 10epoch:train:14001-14100batch: iter_time=8.157e-05, forward_time=0.186, loss_ctc=113.929, loss_interctc_layer6=109.875, loss_interctc_layer12=92.678, loss_interctc_layer15=85.308, loss_interctc_layer21=116.891, loss=103.736, backward_time=0.386, grad_norm=87.210, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.269e-04, train_time=1.907
+[gpua002:0/64] 2024-01-18 20:04:03,187 (trainer:753) INFO: 10epoch:train:14101-14200batch: iter_time=8.041e-05, forward_time=0.141, loss_ctc=80.255, loss_interctc_layer6=88.484, loss_interctc_layer12=74.666, loss_interctc_layer15=68.969, loss_interctc_layer21=82.108, loss=78.897, backward_time=0.346, grad_norm=54.281, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.269e-04, train_time=1.442
+[gpua002:0/64] 2024-01-18 20:06:12,626 (trainer:753) INFO: 10epoch:train:14201-14300batch: iter_time=8.299e-05, forward_time=0.142, loss_ctc=84.499, loss_interctc_layer6=88.483, loss_interctc_layer12=74.272, loss_interctc_layer15=68.530, loss_interctc_layer21=86.957, loss=80.548, backward_time=0.303, grad_norm=62.110, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.268e-04, train_time=1.296
+[gpua002:0/64] 2024-01-18 20:09:01,339 (trainer:753) INFO: 10epoch:train:14301-14400batch: iter_time=8.093e-05, forward_time=0.141, loss_ctc=87.729, loss_interctc_layer6=89.942, loss_interctc_layer12=75.207, loss_interctc_layer15=69.089, loss_interctc_layer21=89.836, loss=82.361, backward_time=0.328, grad_norm=57.731, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.268e-04, train_time=1.687
+[gpua002:0/64] 2024-01-18 20:12:34,585 (trainer:753) INFO: 10epoch:train:14401-14500batch: iter_time=8.399e-05, forward_time=0.142, loss_ctc=84.915, loss_interctc_layer6=93.123, loss_interctc_layer12=78.539, loss_interctc_layer15=72.366, loss_interctc_layer21=86.877, loss=83.164, backward_time=0.450, grad_norm=71.267, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.267e-04, train_time=2.132
+[gpua002:0/64] 2024-01-18 20:16:09,109 (trainer:753) INFO: 10epoch:train:14501-14600batch: iter_time=8.404e-05, forward_time=0.145, loss_ctc=90.239, loss_interctc_layer6=90.383, loss_interctc_layer12=76.032, loss_interctc_layer15=70.216, loss_interctc_layer21=92.396, loss=83.854, backward_time=0.421, grad_norm=63.444, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.141, optim0_lr0=1.267e-04, train_time=2.145
+[gpua002:0/64] 2024-01-18 20:19:14,398 (trainer:753) INFO: 10epoch:train:14601-14700batch: iter_time=8.441e-05, forward_time=0.146, loss_ctc=83.856, loss_interctc_layer6=93.207, loss_interctc_layer12=78.577, loss_interctc_layer15=72.579, loss_interctc_layer21=85.862, loss=82.816, backward_time=0.359, grad_norm=61.082, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.136, optim0_lr0=1.266e-04, train_time=1.853
+[gpua002:0/64] 2024-01-18 20:22:07,579 (trainer:753) INFO: 10epoch:train:14701-14800batch: iter_time=8.684e-05, forward_time=0.142, loss_ctc=90.314, loss_interctc_layer6=102.068, loss_interctc_layer12=86.183, loss_interctc_layer15=80.022, loss_interctc_layer21=92.085, loss=90.135, backward_time=0.369, grad_norm=68.128, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.135, optim0_lr0=1.266e-04, train_time=1.732
+[gpua002:0/64] 2024-01-18 20:24:33,823 (trainer:753) INFO: 10epoch:train:14801-14900batch: iter_time=8.551e-05, forward_time=0.204, loss_ctc=79.030, loss_interctc_layer6=90.568, loss_interctc_layer12=76.888, loss_interctc_layer15=71.084, loss_interctc_layer21=80.982, loss=79.710, backward_time=0.335, grad_norm=59.905, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.138, optim0_lr0=1.266e-04, train_time=1.462
+[gpua002:0/64] 2024-01-18 20:27:07,508 (trainer:753) INFO: 10epoch:train:14901-15000batch: iter_time=8.074e-05, forward_time=0.180, loss_ctc=85.269, loss_interctc_layer6=90.148, loss_interctc_layer12=76.266, loss_interctc_layer15=70.442, loss_interctc_layer21=87.286, loss=81.882, backward_time=0.358, grad_norm=63.162, clip=100.000, loss_scale=1.238e+27, optim_step_time=0.136, optim0_lr0=1.265e-04, train_time=1.535
+[gpua002:0/64] 2024-01-18 21:00:04,285 (trainer:352) INFO: 10epoch results: [train] iter_time=0.358, forward_time=0.231, loss_ctc=92.478, loss_interctc_layer6=95.168, loss_interctc_layer12=80.572, loss_interctc_layer15=74.609, loss_interctc_layer21=94.514, loss=87.468, backward_time=0.455, grad_norm=72.331, clip=100.000, loss_scale=3.282e+26, optim_step_time=0.143, optim0_lr0=1.298e-04, train_time=2.774, time=11 hours, 33 minutes and 50.14 seconds, total_count=150000, gpu_max_cached_mem_GB=34.396, [valid] loss_ctc=61.821, cer_ctc=0.271, loss_interctc_layer6=62.463, cer_interctc_layer6=0.275, loss_interctc_layer12=50.599, cer_interctc_layer12=0.212, loss_interctc_layer15=46.345, cer_interctc_layer15=0.185, loss_interctc_layer21=63.827, cer_interctc_layer21=0.285, loss=57.011, time=32 minutes and 33.04 seconds, total_count=46710, gpu_max_cached_mem_GB=34.396
+[gpua002:0/64] 2024-01-18 21:00:39,410 (trainer:407) INFO: The best model has been updated: valid.cer_ctc, valid.loss_ctc, valid.total_count
+[gpua002:0/64] 2024-01-18 21:00:39,465 (trainer:461) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/5epoch.pth
+[gpua002:0/64] 2024-01-18 21:00:39,465 (trainer:286) INFO: 11/45epoch started. Estimated time to finish: 2 weeks, 3 days and 16 hours
+[gpua002:0/64] 2024-01-18 21:00:39,480 (multiple_iter_factory:32) INFO: Building 0th iter-factory...
+[gpua002:0/64] 2024-01-18 21:00:57,730 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 21:01:01,077 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe717e19660>)
+[gpua002:0/64] 2024-01-18 21:01:01,077 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, 
+[gpua002:0/64] 2024-01-18 21:01:01,080 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 21:13:12,093 (trainer:753) INFO: 11epoch:train:1-100batch: iter_time=3.264, forward_time=0.175, loss_ctc=108.953, loss_interctc_layer6=104.214, loss_interctc_layer12=88.210, loss_interctc_layer15=81.779, loss_interctc_layer21=111.869, loss=99.005, backward_time=0.310, grad_norm=65.007, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.265e-04, train_time=7.526
+[gpua002:0/64] 2024-01-18 21:15:18,498 (trainer:753) INFO: 11epoch:train:101-200batch: iter_time=9.400e-05, forward_time=0.141, loss_ctc=86.312, loss_interctc_layer6=79.025, loss_interctc_layer12=66.919, loss_interctc_layer15=61.940, loss_interctc_layer21=88.238, loss=76.487, backward_time=0.297, grad_norm=72.986, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.264e-04, train_time=1.264
+[gpua002:0/64] 2024-01-18 21:17:51,940 (trainer:753) INFO: 11epoch:train:201-300batch: iter_time=9.395e-05, forward_time=0.142, loss_ctc=79.308, loss_interctc_layer6=83.083, loss_interctc_layer12=70.469, loss_interctc_layer15=65.242, loss_interctc_layer21=80.731, loss=75.767, backward_time=0.309, grad_norm=56.452, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.264e-04, train_time=1.534
+[gpua002:0/64] 2024-01-18 21:20:43,881 (trainer:753) INFO: 11epoch:train:301-400batch: iter_time=9.234e-05, forward_time=0.146, loss_ctc=105.803, loss_interctc_layer6=110.547, loss_interctc_layer12=95.259, loss_interctc_layer15=89.309, loss_interctc_layer21=107.742, loss=101.732, backward_time=0.345, grad_norm=78.150, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.263e-04, train_time=1.718
+[gpua002:0/64] 2024-01-18 21:23:19,091 (trainer:753) INFO: 11epoch:train:401-500batch: iter_time=8.622e-05, forward_time=0.144, loss_ctc=124.048, loss_interctc_layer6=104.254, loss_interctc_layer12=87.742, loss_interctc_layer15=80.989, loss_interctc_layer21=126.738, loss=104.754, backward_time=0.366, grad_norm=65.803, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.263e-04, train_time=1.552
+[gpua002:0/64] 2024-01-18 21:26:08,167 (trainer:753) INFO: 11epoch:train:501-600batch: iter_time=8.579e-05, forward_time=0.143, loss_ctc=79.501, loss_interctc_layer6=90.634, loss_interctc_layer12=75.666, loss_interctc_layer15=70.102, loss_interctc_layer21=81.189, loss=79.419, backward_time=0.342, grad_norm=81.168, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.263e-04, train_time=1.691
+[gpua002:0/64] 2024-01-18 21:28:52,833 (trainer:753) INFO: 11epoch:train:601-700batch: iter_time=9.326e-05, forward_time=0.145, loss_ctc=98.448, loss_interctc_layer6=99.892, loss_interctc_layer12=87.046, loss_interctc_layer15=81.717, loss_interctc_layer21=100.310, loss=93.483, backward_time=0.359, grad_norm=68.469, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.262e-04, train_time=1.646
+[gpua002:0/64] 2024-01-18 21:31:53,220 (trainer:753) INFO: 11epoch:train:701-800batch: iter_time=8.262e-05, forward_time=0.142, loss_ctc=101.694, loss_interctc_layer6=105.485, loss_interctc_layer12=90.001, loss_interctc_layer15=83.761, loss_interctc_layer21=103.192, loss=96.827, backward_time=0.391, grad_norm=73.452, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.262e-04, train_time=1.804
+[gpua002:0/64] 2024-01-18 21:34:36,955 (trainer:753) INFO: 11epoch:train:801-900batch: iter_time=7.691e-05, forward_time=0.143, loss_ctc=121.764, loss_interctc_layer6=108.371, loss_interctc_layer12=91.738, loss_interctc_layer15=85.098, loss_interctc_layer21=124.571, loss=106.309, backward_time=0.357, grad_norm=89.814, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.261e-04, train_time=1.637
+[gpua002:0/64] 2024-01-18 21:37:22,869 (trainer:753) INFO: 11epoch:train:901-1000batch: iter_time=8.471e-05, forward_time=0.142, loss_ctc=117.402, loss_interctc_layer6=106.536, loss_interctc_layer12=90.168, loss_interctc_layer15=84.103, loss_interctc_layer21=120.579, loss=103.758, backward_time=0.337, grad_norm=88.373, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.261e-04, train_time=1.659
+[gpua002:0/64] 2024-01-18 21:40:43,879 (trainer:753) INFO: 11epoch:train:1001-1100batch: iter_time=4.615e-04, forward_time=0.168, loss_ctc=88.685, loss_interctc_layer6=93.258, loss_interctc_layer12=79.095, loss_interctc_layer15=73.681, loss_interctc_layer21=90.665, loss=85.077, backward_time=0.522, grad_norm=64.797, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.261e-04, train_time=2.010
+[gpua002:0/64] 2024-01-18 21:44:11,511 (trainer:753) INFO: 11epoch:train:1101-1200batch: iter_time=9.392e-05, forward_time=0.221, loss_ctc=86.275, loss_interctc_layer6=89.557, loss_interctc_layer12=76.849, loss_interctc_layer15=71.643, loss_interctc_layer21=88.338, loss=82.532, backward_time=0.455, grad_norm=58.253, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.161, optim0_lr0=1.260e-04, train_time=2.076
+[gpua002:0/64] 2024-01-18 21:45:57,476 (multiple_iter_factory:32) INFO: Building 1th iter-factory...
+[gpua002:0/64] 2024-01-18 21:46:16,695 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 21:46:20,246 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe717d6e170>)
+[gpua002:0/64] 2024-01-18 21:46:20,246 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, 
+[gpua002:0/64] 2024-01-18 21:46:20,250 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 21:53:29,694 (trainer:753) INFO: 11epoch:train:1201-1300batch: iter_time=3.020, forward_time=0.147, loss_ctc=108.664, loss_interctc_layer6=112.616, loss_interctc_layer12=94.820, loss_interctc_layer15=87.590, loss_interctc_layer21=110.974, loss=102.933, backward_time=0.368, grad_norm=73.736, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.260e-04, train_time=5.580
+[gpua002:0/64] 2024-01-18 21:55:36,264 (trainer:753) INFO: 11epoch:train:1301-1400batch: iter_time=8.586e-05, forward_time=0.143, loss_ctc=83.898, loss_interctc_layer6=88.242, loss_interctc_layer12=74.539, loss_interctc_layer15=68.855, loss_interctc_layer21=85.697, loss=80.246, backward_time=0.297, grad_norm=53.789, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.259e-04, train_time=1.267
+[gpua002:0/64] 2024-01-18 21:57:43,834 (trainer:753) INFO: 11epoch:train:1401-1500batch: iter_time=8.943e-05, forward_time=0.141, loss_ctc=78.511, loss_interctc_layer6=81.721, loss_interctc_layer12=69.049, loss_interctc_layer15=63.770, loss_interctc_layer21=80.336, loss=74.677, backward_time=0.295, grad_norm=60.326, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.259e-04, train_time=1.275
+[gpua002:0/64] 2024-01-18 22:00:21,649 (trainer:753) INFO: 11epoch:train:1501-1600batch: iter_time=9.077e-05, forward_time=0.144, loss_ctc=79.112, loss_interctc_layer6=87.594, loss_interctc_layer12=74.126, loss_interctc_layer15=68.647, loss_interctc_layer21=80.597, loss=78.015, backward_time=0.335, grad_norm=54.272, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.258e-04, train_time=1.577
+[gpua002:0/64] 2024-01-18 22:02:59,234 (trainer:753) INFO: 11epoch:train:1601-1700batch: iter_time=9.515e-05, forward_time=0.145, loss_ctc=110.680, loss_interctc_layer6=112.588, loss_interctc_layer12=95.908, loss_interctc_layer15=88.963, loss_interctc_layer21=113.192, loss=104.266, backward_time=0.410, grad_norm=80.515, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.258e-04, train_time=1.577
+[gpua002:0/64] 2024-01-18 22:05:50,466 (trainer:753) INFO: 11epoch:train:1701-1800batch: iter_time=8.767e-05, forward_time=0.143, loss_ctc=97.038, loss_interctc_layer6=99.310, loss_interctc_layer12=82.504, loss_interctc_layer15=75.799, loss_interctc_layer21=99.455, loss=90.821, backward_time=0.373, grad_norm=71.286, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.139, optim0_lr0=1.258e-04, train_time=1.712
+[gpua002:0/64] 2024-01-18 22:08:44,225 (trainer:753) INFO: 11epoch:train:1801-1900batch: iter_time=8.845e-05, forward_time=0.141, loss_ctc=89.545, loss_interctc_layer6=93.175, loss_interctc_layer12=78.937, loss_interctc_layer15=73.359, loss_interctc_layer21=91.567, loss=85.316, backward_time=0.388, grad_norm=71.358, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.257e-04, train_time=1.737
+[gpua002:0/64] 2024-01-18 22:12:13,919 (trainer:753) INFO: 11epoch:train:1901-2000batch: iter_time=8.754e-05, forward_time=0.142, loss_ctc=99.272, loss_interctc_layer6=106.182, loss_interctc_layer12=91.234, loss_interctc_layer15=85.294, loss_interctc_layer21=101.032, loss=96.603, backward_time=0.444, grad_norm=67.123, clip=100.000, loss_scale=2.476e+27, optim_step_time=0.138, optim0_lr0=1.257e-04, train_time=2.097
+[gpua002:0/64] 2024-01-18 22:16:01,041 (trainer:753) INFO: 11epoch:train:2001-2100batch: iter_time=8.649e-05, forward_time=0.142, loss_ctc=102.128, loss_interctc_layer6=102.357, loss_interctc_layer12=87.079, loss_interctc_layer15=80.531, loss_interctc_layer21=104.644, loss=95.348, backward_time=0.459, grad_norm=70.814, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.256e-04, train_time=2.271
+[gpua002:0/64] 2024-01-18 22:20:03,532 (trainer:753) INFO: 11epoch:train:2101-2200batch: iter_time=9.214e-05, forward_time=0.265, loss_ctc=112.069, loss_interctc_layer6=109.535, loss_interctc_layer12=92.520, loss_interctc_layer15=85.703, loss_interctc_layer21=114.410, loss=102.847, backward_time=0.477, grad_norm=69.616, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.141, optim0_lr0=1.256e-04, train_time=2.425
+[gpua002:0/64] 2024-01-18 22:22:46,611 (trainer:753) INFO: 11epoch:train:2201-2300batch: iter_time=8.800e-05, forward_time=0.174, loss_ctc=100.265, loss_interctc_layer6=93.917, loss_interctc_layer12=79.503, loss_interctc_layer15=73.519, loss_interctc_layer21=103.009, loss=90.042, backward_time=0.350, grad_norm=58.946, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.142, optim0_lr0=1.256e-04, train_time=1.631
+[gpua002:0/64] 2024-01-18 22:25:56,204 (trainer:753) INFO: 11epoch:train:2301-2400batch: iter_time=9.082e-05, forward_time=0.142, loss_ctc=77.470, loss_interctc_layer6=90.918, loss_interctc_layer12=77.382, loss_interctc_layer15=71.701, loss_interctc_layer21=79.388, loss=79.372, backward_time=0.407, grad_norm=63.709, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.255e-04, train_time=1.892
+[gpua002:0/64] 2024-01-18 22:28:10,602 (trainer:753) INFO: 11epoch:train:2401-2500batch: iter_time=8.949e-05, forward_time=0.148, loss_ctc=96.410, loss_interctc_layer6=103.612, loss_interctc_layer12=87.641, loss_interctc_layer15=81.104, loss_interctc_layer21=98.604, loss=93.474, backward_time=0.305, grad_norm=72.102, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.139, optim0_lr0=1.255e-04, train_time=1.346
+[gpua002:0/64] 2024-01-18 22:28:30,632 (multiple_iter_factory:32) INFO: Building 2th iter-factory...
+[gpua002:0/64] 2024-01-18 22:28:50,012 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 22:28:53,780 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea44deed40>)
+[gpua002:0/64] 2024-01-18 22:28:53,780 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, 
+[gpua002:0/64] 2024-01-18 22:28:53,784 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 22:38:39,933 (trainer:753) INFO: 11epoch:train:2501-2600batch: iter_time=4.755, forward_time=0.148, loss_ctc=99.657, loss_interctc_layer6=104.418, loss_interctc_layer12=88.061, loss_interctc_layer15=81.219, loss_interctc_layer21=101.961, loss=95.063, backward_time=0.319, grad_norm=64.977, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.140, optim0_lr0=1.254e-04, train_time=6.294
+[gpua002:0/64] 2024-01-18 22:41:08,787 (trainer:753) INFO: 11epoch:train:2601-2700batch: iter_time=8.408e-05, forward_time=0.142, loss_ctc=77.854, loss_interctc_layer6=77.981, loss_interctc_layer12=65.713, loss_interctc_layer15=60.680, loss_interctc_layer21=79.628, loss=72.371, backward_time=0.318, grad_norm=53.979, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.139, optim0_lr0=1.254e-04, train_time=1.489
+[gpua002:0/64] 2024-01-18 22:43:40,554 (trainer:753) INFO: 11epoch:train:2701-2800batch: iter_time=8.123e-05, forward_time=0.142, loss_ctc=74.538, loss_interctc_layer6=81.908, loss_interctc_layer12=68.967, loss_interctc_layer15=63.774, loss_interctc_layer21=76.290, loss=73.095, backward_time=0.310, grad_norm=51.406, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.139, optim0_lr0=1.253e-04, train_time=1.517
+[gpua002:0/64] 2024-01-18 22:46:20,779 (trainer:753) INFO: 11epoch:train:2801-2900batch: iter_time=9.776e-05, forward_time=0.143, loss_ctc=96.342, loss_interctc_layer6=107.042, loss_interctc_layer12=90.815, loss_interctc_layer15=84.279, loss_interctc_layer21=98.259, loss=95.347, backward_time=0.380, grad_norm=90.707, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.253e-04, train_time=1.602
+[gpua002:0/64] 2024-01-18 22:48:56,824 (trainer:753) INFO: 11epoch:train:2901-3000batch: iter_time=1.056e-04, forward_time=0.144, loss_ctc=114.388, loss_interctc_layer6=102.821, loss_interctc_layer12=86.159, loss_interctc_layer15=79.404, loss_interctc_layer21=116.917, loss=99.938, backward_time=0.353, grad_norm=86.098, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.253e-04, train_time=1.560
+[gpua002:0/64] 2024-01-18 22:52:01,823 (trainer:753) INFO: 11epoch:train:3001-3100batch: iter_time=9.631e-05, forward_time=0.142, loss_ctc=76.328, loss_interctc_layer6=89.344, loss_interctc_layer12=74.784, loss_interctc_layer15=68.886, loss_interctc_layer21=78.086, loss=77.486, backward_time=0.340, grad_norm=67.481, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.252e-04, train_time=1.850
+[gpua002:0/64] 2024-01-18 22:54:46,792 (trainer:753) INFO: 11epoch:train:3101-3200batch: iter_time=8.330e-05, forward_time=0.142, loss_ctc=89.622, loss_interctc_layer6=96.373, loss_interctc_layer12=82.507, loss_interctc_layer15=76.772, loss_interctc_layer21=91.627, loss=87.380, backward_time=0.351, grad_norm=72.167, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.139, optim0_lr0=1.252e-04, train_time=1.649
+[gpua002:0/64] 2024-01-18 22:57:40,705 (trainer:753) INFO: 11epoch:train:3201-3300batch: iter_time=8.482e-05, forward_time=0.240, loss_ctc=95.327, loss_interctc_layer6=104.426, loss_interctc_layer12=88.413, loss_interctc_layer15=81.928, loss_interctc_layer21=97.465, loss=93.512, backward_time=0.387, grad_norm=69.499, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.143, optim0_lr0=1.251e-04, train_time=1.739
+[gpua002:0/64] 2024-01-18 23:00:27,194 (trainer:753) INFO: 11epoch:train:3301-3400batch: iter_time=1.033e-04, forward_time=0.177, loss_ctc=112.414, loss_interctc_layer6=106.141, loss_interctc_layer12=89.576, loss_interctc_layer15=82.789, loss_interctc_layer21=114.718, loss=101.128, backward_time=0.331, grad_norm=63.978, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.146, optim0_lr0=1.251e-04, train_time=1.665
+[gpua002:0/64] 2024-01-18 23:03:24,755 (trainer:753) INFO: 11epoch:train:3401-3500batch: iter_time=8.899e-05, forward_time=0.144, loss_ctc=109.714, loss_interctc_layer6=105.312, loss_interctc_layer12=89.180, loss_interctc_layer15=82.400, loss_interctc_layer21=112.509, loss=99.823, backward_time=0.346, grad_norm=71.777, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.251e-04, train_time=1.775
+[gpua002:0/64] 2024-01-18 23:05:53,962 (trainer:753) INFO: 11epoch:train:3501-3600batch: iter_time=8.506e-05, forward_time=0.148, loss_ctc=83.339, loss_interctc_layer6=91.081, loss_interctc_layer12=76.936, loss_interctc_layer15=71.069, loss_interctc_layer21=85.125, loss=81.510, backward_time=0.367, grad_norm=57.243, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.139, optim0_lr0=1.250e-04, train_time=1.491
+[gpua002:0/64] 2024-01-18 23:08:53,176 (trainer:753) INFO: 11epoch:train:3601-3700batch: iter_time=8.492e-05, forward_time=0.142, loss_ctc=82.163, loss_interctc_layer6=87.280, loss_interctc_layer12=74.597, loss_interctc_layer15=69.397, loss_interctc_layer21=83.922, loss=79.472, backward_time=0.337, grad_norm=66.843, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.139, optim0_lr0=1.250e-04, train_time=1.792
+[gpua002:0/64] 2024-01-18 23:10:37,198 (multiple_iter_factory:32) INFO: Building 3th iter-factory...
+[gpua002:0/64] 2024-01-18 23:10:56,612 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 23:11:00,169 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea83050ca0>)
+[gpua002:0/64] 2024-01-18 23:11:00,169 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, 
+[gpua002:0/64] 2024-01-18 23:11:00,172 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-18 23:18:46,817 (trainer:753) INFO: 11epoch:train:3701-3800batch: iter_time=2.994, forward_time=0.147, loss_ctc=105.855, loss_interctc_layer6=112.450, loss_interctc_layer12=94.241, loss_interctc_layer15=86.974, loss_interctc_layer21=108.386, loss=101.581, backward_time=0.350, grad_norm=77.125, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.140, optim0_lr0=1.249e-04, train_time=5.936
+[gpua002:0/64] 2024-01-18 23:20:56,615 (trainer:753) INFO: 11epoch:train:3801-3900batch: iter_time=8.835e-05, forward_time=0.144, loss_ctc=81.416, loss_interctc_layer6=86.543, loss_interctc_layer12=73.073, loss_interctc_layer15=67.443, loss_interctc_layer21=83.608, loss=78.417, backward_time=0.298, grad_norm=53.015, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.249e-04, train_time=1.298
+[gpua002:0/64] 2024-01-18 23:23:31,395 (trainer:753) INFO: 11epoch:train:3901-4000batch: iter_time=8.876e-05, forward_time=0.141, loss_ctc=77.296, loss_interctc_layer6=80.976, loss_interctc_layer12=68.134, loss_interctc_layer15=63.037, loss_interctc_layer21=79.230, loss=73.735, backward_time=0.354, grad_norm=49.107, clip=100.000, loss_scale=4.952e+27, optim_step_time=0.138, optim0_lr0=1.249e-04, train_time=1.548
+[gpua002:0/64] 2024-01-18 23:26:40,709 (trainer:753) INFO: 11epoch:train:4001-4100batch: iter_time=9.798e-05, forward_time=0.142, loss_ctc=78.067, loss_interctc_layer6=86.893, loss_interctc_layer12=73.606, loss_interctc_layer15=67.883, loss_interctc_layer21=79.747, loss=77.239, backward_time=0.322, grad_norm=79.970, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.139, optim0_lr0=1.248e-04, train_time=1.893
+[gpua002:0/64] 2024-01-18 23:29:30,616 (trainer:753) INFO: 11epoch:train:4101-4200batch: iter_time=9.777e-05, forward_time=0.143, loss_ctc=107.476, loss_interctc_layer6=109.801, loss_interctc_layer12=93.236, loss_interctc_layer15=86.558, loss_interctc_layer21=109.948, loss=101.404, backward_time=0.354, grad_norm=89.042, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.139, optim0_lr0=1.248e-04, train_time=1.699
+[gpua002:0/64] 2024-01-18 23:32:26,817 (trainer:753) INFO: 11epoch:train:4201-4300batch: iter_time=9.411e-05, forward_time=0.183, loss_ctc=95.572, loss_interctc_layer6=97.487, loss_interctc_layer12=81.257, loss_interctc_layer15=74.413, loss_interctc_layer21=97.600, loss=89.266, backward_time=0.357, grad_norm=61.599, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.159, optim0_lr0=1.247e-04, train_time=1.762
+[gpua002:0/64] 2024-01-18 23:35:22,762 (trainer:753) INFO: 11epoch:train:4301-4400batch: iter_time=8.677e-05, forward_time=0.217, loss_ctc=87.237, loss_interctc_layer6=92.202, loss_interctc_layer12=77.902, loss_interctc_layer15=71.801, loss_interctc_layer21=89.388, loss=83.706, backward_time=0.397, grad_norm=56.812, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.150, optim0_lr0=1.247e-04, train_time=1.759
+[gpua002:0/64] 2024-01-18 23:38:00,980 (trainer:753) INFO: 11epoch:train:4401-4500batch: iter_time=8.811e-05, forward_time=0.158, loss_ctc=95.706, loss_interctc_layer6=103.771, loss_interctc_layer12=88.791, loss_interctc_layer15=82.931, loss_interctc_layer21=98.084, loss=93.856, backward_time=0.337, grad_norm=73.556, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.247e-04, train_time=1.582
+[gpua002:0/64] 2024-01-18 23:40:55,506 (trainer:753) INFO: 11epoch:train:4501-4600batch: iter_time=8.991e-05, forward_time=0.142, loss_ctc=99.009, loss_interctc_layer6=100.149, loss_interctc_layer12=84.621, loss_interctc_layer15=78.197, loss_interctc_layer21=101.275, loss=92.650, backward_time=0.405, grad_norm=65.279, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.246e-04, train_time=1.745
+[gpua002:0/64] 2024-01-18 23:44:44,363 (trainer:753) INFO: 11epoch:train:4601-4700batch: iter_time=9.889e-05, forward_time=0.146, loss_ctc=110.340, loss_interctc_layer6=108.061, loss_interctc_layer12=90.987, loss_interctc_layer15=84.267, loss_interctc_layer21=112.654, loss=101.262, backward_time=0.437, grad_norm=64.469, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.246e-04, train_time=2.288
+[gpua002:0/64] 2024-01-18 23:47:12,812 (trainer:753) INFO: 11epoch:train:4701-4800batch: iter_time=8.381e-05, forward_time=0.142, loss_ctc=99.346, loss_interctc_layer6=93.157, loss_interctc_layer12=78.627, loss_interctc_layer15=72.859, loss_interctc_layer21=102.008, loss=89.199, backward_time=0.300, grad_norm=70.219, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.245e-04, train_time=1.485
+[gpua002:0/64] 2024-01-18 23:51:00,327 (trainer:753) INFO: 11epoch:train:4801-4900batch: iter_time=9.065e-05, forward_time=0.144, loss_ctc=75.678, loss_interctc_layer6=89.538, loss_interctc_layer12=76.007, loss_interctc_layer15=70.578, loss_interctc_layer21=77.131, loss=77.786, backward_time=0.351, grad_norm=61.040, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.139, optim0_lr0=1.245e-04, train_time=2.272
+[gpua002:0/64] 2024-01-18 23:53:25,085 (trainer:753) INFO: 11epoch:train:4901-5000batch: iter_time=8.568e-05, forward_time=0.146, loss_ctc=95.991, loss_interctc_layer6=103.822, loss_interctc_layer12=87.793, loss_interctc_layer15=81.094, loss_interctc_layer21=98.207, loss=93.381, backward_time=0.329, grad_norm=71.609, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.139, optim0_lr0=1.245e-04, train_time=1.449
+[gpua002:0/64] 2024-01-18 23:53:45,148 (multiple_iter_factory:32) INFO: Building 4th iter-factory...
+[gpua002:0/64] 2024-01-18 23:54:04,225 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-18 23:54:07,845 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe736bd7cd0>)
+[gpua002:0/64] 2024-01-18 23:54:07,846 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, 
+[gpua002:0/64] 2024-01-18 23:54:07,849 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 00:03:49,601 (trainer:753) INFO: 11epoch:train:5001-5100batch: iter_time=4.811, forward_time=0.143, loss_ctc=107.244, loss_interctc_layer6=102.033, loss_interctc_layer12=85.766, loss_interctc_layer15=79.262, loss_interctc_layer21=109.613, loss=96.784, backward_time=0.303, grad_norm=99.070, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.244e-04, train_time=6.246
+[gpua002:0/64] 2024-01-19 00:06:00,668 (trainer:753) INFO: 11epoch:train:5101-5200batch: iter_time=8.554e-05, forward_time=0.141, loss_ctc=85.756, loss_interctc_layer6=77.876, loss_interctc_layer12=65.482, loss_interctc_layer15=60.386, loss_interctc_layer21=87.932, loss=75.486, backward_time=0.305, grad_norm=63.682, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.244e-04, train_time=1.310
+[gpua002:0/64] 2024-01-19 00:08:33,858 (trainer:753) INFO: 11epoch:train:5201-5300batch: iter_time=9.195e-05, forward_time=0.187, loss_ctc=77.421, loss_interctc_layer6=81.257, loss_interctc_layer12=68.617, loss_interctc_layer15=63.286, loss_interctc_layer21=79.394, loss=73.995, backward_time=0.324, grad_norm=72.030, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.147, optim0_lr0=1.243e-04, train_time=1.532
+[gpua002:0/64] 2024-01-19 00:11:15,741 (trainer:753) INFO: 11epoch:train:5301-5400batch: iter_time=9.020e-05, forward_time=0.171, loss_ctc=99.749, loss_interctc_layer6=105.951, loss_interctc_layer12=89.762, loss_interctc_layer15=82.995, loss_interctc_layer21=102.069, loss=96.105, backward_time=0.398, grad_norm=77.430, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.143, optim0_lr0=1.243e-04, train_time=1.619
+[gpua002:0/64] 2024-01-19 00:14:07,377 (trainer:753) INFO: 11epoch:train:5401-5500batch: iter_time=8.285e-05, forward_time=0.164, loss_ctc=120.521, loss_interctc_layer6=101.708, loss_interctc_layer12=85.091, loss_interctc_layer15=78.342, loss_interctc_layer21=124.018, loss=101.936, backward_time=0.386, grad_norm=92.200, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.141, optim0_lr0=1.243e-04, train_time=1.716
+[gpua002:0/64] 2024-01-19 00:16:41,263 (trainer:753) INFO: 11epoch:train:5501-5600batch: iter_time=8.885e-05, forward_time=0.145, loss_ctc=77.851, loss_interctc_layer6=88.821, loss_interctc_layer12=74.169, loss_interctc_layer15=68.121, loss_interctc_layer21=79.547, loss=77.702, backward_time=0.333, grad_norm=83.895, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.139, optim0_lr0=1.242e-04, train_time=1.539
+[gpua002:0/64] 2024-01-19 00:19:36,510 (trainer:753) INFO: 11epoch:train:5601-5700batch: iter_time=9.254e-05, forward_time=0.148, loss_ctc=93.789, loss_interctc_layer6=96.096, loss_interctc_layer12=82.443, loss_interctc_layer15=76.880, loss_interctc_layer21=95.921, loss=89.026, backward_time=0.407, grad_norm=69.334, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.139, optim0_lr0=1.242e-04, train_time=1.752
+[gpua002:0/64] 2024-01-19 00:22:50,691 (trainer:753) INFO: 11epoch:train:5701-5800batch: iter_time=9.589e-05, forward_time=0.142, loss_ctc=97.828, loss_interctc_layer6=103.368, loss_interctc_layer12=87.112, loss_interctc_layer15=80.484, loss_interctc_layer21=99.844, loss=93.727, backward_time=0.374, grad_norm=61.141, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.241e-04, train_time=1.942
+[gpua002:0/64] 2024-01-19 00:25:46,656 (trainer:753) INFO: 11epoch:train:5801-5900batch: iter_time=8.519e-05, forward_time=0.147, loss_ctc=115.138, loss_interctc_layer6=104.004, loss_interctc_layer12=87.618, loss_interctc_layer15=80.892, loss_interctc_layer21=118.195, loss=101.169, backward_time=0.394, grad_norm=66.195, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.139, optim0_lr0=1.241e-04, train_time=1.759
+[gpua002:0/64] 2024-01-19 00:28:18,083 (trainer:753) INFO: 11epoch:train:5901-6000batch: iter_time=9.203e-05, forward_time=0.145, loss_ctc=115.327, loss_interctc_layer6=104.444, loss_interctc_layer12=88.037, loss_interctc_layer15=81.257, loss_interctc_layer21=117.959, loss=101.405, backward_time=0.358, grad_norm=63.749, clip=100.000, loss_scale=9.904e+27, optim_step_time=0.138, optim0_lr0=1.241e-04, train_time=1.513
+[gpua002:0/64] 2024-01-19 00:31:28,210 (trainer:753) INFO: 11epoch:train:6001-6100batch: iter_time=8.614e-05, forward_time=0.165, loss_ctc=86.973, loss_interctc_layer6=91.195, loss_interctc_layer12=76.752, loss_interctc_layer15=71.150, loss_interctc_layer21=89.046, loss=83.023, backward_time=0.421, grad_norm=59.099, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.240e-04, train_time=1.902
+[gpua002:0/64] 2024-01-19 00:35:07,201 (trainer:753) INFO: 11epoch:train:6101-6200batch: iter_time=8.573e-05, forward_time=0.142, loss_ctc=83.098, loss_interctc_layer6=86.691, loss_interctc_layer12=73.916, loss_interctc_layer15=68.636, loss_interctc_layer21=84.760, loss=79.420, backward_time=0.498, grad_norm=61.871, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.240e-04, train_time=2.190
+[gpua002:0/64] 2024-01-19 00:36:42,330 (multiple_iter_factory:32) INFO: Building 5th iter-factory...
+[gpua002:0/64] 2024-01-19 00:37:01,786 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 00:37:05,491 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea456e5480>)
+[gpua002:0/64] 2024-01-19 00:37:05,491 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, 
+[gpua002:0/64] 2024-01-19 00:37:05,803 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 00:45:34,587 (trainer:753) INFO: 11epoch:train:6201-6300batch: iter_time=4.786, forward_time=0.173, loss_ctc=108.949, loss_interctc_layer6=110.937, loss_interctc_layer12=92.826, loss_interctc_layer15=85.467, loss_interctc_layer21=111.386, loss=101.913, backward_time=0.308, grad_norm=133.570, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.140, optim0_lr0=1.239e-04, train_time=6.274
+[gpua002:0/64] 2024-01-19 00:47:41,755 (trainer:753) INFO: 11epoch:train:6301-6400batch: iter_time=8.707e-05, forward_time=0.142, loss_ctc=87.000, loss_interctc_layer6=86.379, loss_interctc_layer12=72.681, loss_interctc_layer15=66.835, loss_interctc_layer21=89.568, loss=80.492, backward_time=0.296, grad_norm=70.186, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.239e-04, train_time=1.271
+[gpua002:0/64] 2024-01-19 00:49:57,262 (trainer:753) INFO: 11epoch:train:6401-6500batch: iter_time=8.697e-05, forward_time=0.144, loss_ctc=85.025, loss_interctc_layer6=80.655, loss_interctc_layer12=67.822, loss_interctc_layer15=62.693, loss_interctc_layer21=86.693, loss=76.578, backward_time=0.307, grad_norm=1.553e+03, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.239e-04, train_time=1.355
+[gpua002:0/64] 2024-01-19 00:52:40,265 (trainer:753) INFO: 11epoch:train:6501-6600batch: iter_time=9.123e-05, forward_time=0.142, loss_ctc=80.210, loss_interctc_layer6=86.801, loss_interctc_layer12=72.960, loss_interctc_layer15=67.293, loss_interctc_layer21=82.039, loss=77.860, backward_time=0.367, grad_norm=59.772, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.238e-04, train_time=1.630
+[gpua002:0/64] 2024-01-19 00:55:41,985 (trainer:753) INFO: 11epoch:train:6601-6700batch: iter_time=9.680e-05, forward_time=0.145, loss_ctc=112.920, loss_interctc_layer6=108.754, loss_interctc_layer12=92.001, loss_interctc_layer15=85.174, loss_interctc_layer21=115.293, loss=102.828, backward_time=0.422, grad_norm=72.527, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.238e-04, train_time=1.817
+[gpua002:0/64] 2024-01-19 00:59:14,219 (trainer:753) INFO: 11epoch:train:6701-6800batch: iter_time=1.016e-04, forward_time=0.143, loss_ctc=99.904, loss_interctc_layer6=97.507, loss_interctc_layer12=80.863, loss_interctc_layer15=73.975, loss_interctc_layer21=102.512, loss=90.952, backward_time=0.470, grad_norm=61.275, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.237e-04, train_time=2.121
+[gpua002:0/64] 2024-01-19 01:02:02,851 (trainer:753) INFO: 11epoch:train:6801-6900batch: iter_time=9.597e-05, forward_time=0.145, loss_ctc=91.696, loss_interctc_layer6=91.443, loss_interctc_layer12=77.080, loss_interctc_layer15=71.205, loss_interctc_layer21=93.893, loss=85.063, backward_time=0.379, grad_norm=71.449, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.237e-04, train_time=1.688
+[gpua002:0/64] 2024-01-19 01:04:40,769 (trainer:753) INFO: 11epoch:train:6901-7000batch: iter_time=9.529e-05, forward_time=0.143, loss_ctc=98.309, loss_interctc_layer6=102.768, loss_interctc_layer12=88.131, loss_interctc_layer15=81.975, loss_interctc_layer21=100.887, loss=94.414, backward_time=0.357, grad_norm=65.989, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.237e-04, train_time=1.579
+[gpua002:0/64] 2024-01-19 01:07:26,927 (trainer:753) INFO: 11epoch:train:7001-7100batch: iter_time=9.821e-05, forward_time=0.143, loss_ctc=102.989, loss_interctc_layer6=99.849, loss_interctc_layer12=83.946, loss_interctc_layer15=77.645, loss_interctc_layer21=105.423, loss=93.970, backward_time=0.345, grad_norm=65.608, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.236e-04, train_time=1.660
+[gpua002:0/64] 2024-01-19 01:10:43,009 (trainer:753) INFO: 11epoch:train:7101-7200batch: iter_time=8.669e-05, forward_time=0.145, loss_ctc=114.903, loss_interctc_layer6=107.838, loss_interctc_layer12=90.776, loss_interctc_layer15=83.944, loss_interctc_layer21=117.659, loss=103.024, backward_time=0.391, grad_norm=74.284, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.236e-04, train_time=1.961
+[gpua002:0/64] 2024-01-19 01:13:32,253 (trainer:753) INFO: 11epoch:train:7201-7300batch: iter_time=9.091e-05, forward_time=0.142, loss_ctc=103.501, loss_interctc_layer6=92.275, loss_interctc_layer12=77.549, loss_interctc_layer15=71.665, loss_interctc_layer21=106.087, loss=90.215, backward_time=0.320, grad_norm=99.048, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.235e-04, train_time=1.693
+[gpua002:0/64] 2024-01-19 01:18:41,329 (trainer:753) INFO: 11epoch:train:7301-7400batch: iter_time=0.368, forward_time=0.292, loss_ctc=78.605, loss_interctc_layer6=89.212, loss_interctc_layer12=75.315, loss_interctc_layer15=69.964, loss_interctc_layer21=79.846, loss=78.589, backward_time=0.556, grad_norm=60.219, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.145, optim0_lr0=1.235e-04, train_time=3.091
+[gpua002:0/64] 2024-01-19 01:21:50,749 (trainer:753) INFO: 11epoch:train:7401-7500batch: iter_time=8.406e-05, forward_time=0.143, loss_ctc=97.263, loss_interctc_layer6=103.171, loss_interctc_layer12=86.787, loss_interctc_layer15=80.164, loss_interctc_layer21=99.327, loss=93.343, backward_time=0.340, grad_norm=69.550, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.235e-04, train_time=1.894
+[gpua002:0/64] 2024-01-19 01:22:10,780 (multiple_iter_factory:32) INFO: Building 6th iter-factory...
+[gpua002:0/64] 2024-01-19 01:22:30,130 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 01:22:33,703 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe6e51fcfd0>)
+[gpua002:0/64] 2024-01-19 01:22:33,704 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, 
+[gpua002:0/64] 2024-01-19 01:22:33,790 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 01:32:22,007 (trainer:753) INFO: 11epoch:train:7501-7600batch: iter_time=4.406, forward_time=0.152, loss_ctc=105.450, loss_interctc_layer6=102.633, loss_interctc_layer12=86.082, loss_interctc_layer15=79.349, loss_interctc_layer21=108.165, loss=96.336, backward_time=0.301, grad_norm=60.958, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.234e-04, train_time=6.312
+[gpua002:0/64] 2024-01-19 01:34:39,636 (trainer:753) INFO: 11epoch:train:7601-7700batch: iter_time=9.025e-05, forward_time=0.142, loss_ctc=84.341, loss_interctc_layer6=77.536, loss_interctc_layer12=64.974, loss_interctc_layer15=59.780, loss_interctc_layer21=86.874, loss=74.701, backward_time=0.296, grad_norm=64.637, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.234e-04, train_time=1.376
+[gpua002:0/64] 2024-01-19 01:38:02,923 (trainer:753) INFO: 11epoch:train:7701-7800batch: iter_time=9.726e-05, forward_time=0.142, loss_ctc=77.015, loss_interctc_layer6=81.233, loss_interctc_layer12=68.130, loss_interctc_layer15=62.975, loss_interctc_layer21=78.611, loss=73.593, backward_time=0.356, grad_norm=62.731, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.233e-04, train_time=2.033
+[gpua002:0/64] 2024-01-19 01:41:08,712 (trainer:753) INFO: 11epoch:train:7801-7900batch: iter_time=9.155e-05, forward_time=0.144, loss_ctc=97.943, loss_interctc_layer6=104.821, loss_interctc_layer12=88.855, loss_interctc_layer15=82.675, loss_interctc_layer21=99.987, loss=94.856, backward_time=0.407, grad_norm=66.775, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.138, optim0_lr0=1.233e-04, train_time=1.858
+[gpua002:0/64] 2024-01-19 01:44:01,605 (trainer:753) INFO: 11epoch:train:7901-8000batch: iter_time=9.043e-05, forward_time=0.143, loss_ctc=119.599, loss_interctc_layer6=101.326, loss_interctc_layer12=84.720, loss_interctc_layer15=77.967, loss_interctc_layer21=122.702, loss=101.263, backward_time=0.396, grad_norm=103.710, clip=100.000, loss_scale=1.981e+28, optim_step_time=0.139, optim0_lr0=1.233e-04, train_time=1.724
+[gpua002:0/64] 2024-01-19 01:48:03,560 (trainer:753) INFO: 11epoch:train:8001-8100batch: iter_time=9.520e-05, forward_time=0.143, loss_ctc=76.178, loss_interctc_layer6=87.554, loss_interctc_layer12=73.038, loss_interctc_layer15=66.998, loss_interctc_layer21=77.869, loss=76.327, backward_time=0.456, grad_norm=57.471, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.139, optim0_lr0=1.232e-04, train_time=2.424
+[gpua002:0/64] 2024-01-19 01:50:33,663 (trainer:753) INFO: 11epoch:train:8101-8200batch: iter_time=9.010e-05, forward_time=0.144, loss_ctc=92.185, loss_interctc_layer6=94.991, loss_interctc_layer12=80.993, loss_interctc_layer15=75.434, loss_interctc_layer21=94.220, loss=87.565, backward_time=0.317, grad_norm=77.516, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.232e-04, train_time=1.500
+[gpua002:0/64] 2024-01-19 01:53:42,764 (trainer:753) INFO: 11epoch:train:8201-8300batch: iter_time=9.298e-05, forward_time=0.142, loss_ctc=95.888, loss_interctc_layer6=103.150, loss_interctc_layer12=86.546, loss_interctc_layer15=80.022, loss_interctc_layer21=98.324, loss=92.786, backward_time=0.413, grad_norm=90.699, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.231e-04, train_time=1.891
+[gpua002:0/64] 2024-01-19 01:56:56,977 (trainer:753) INFO: 11epoch:train:8301-8400batch: iter_time=8.710e-05, forward_time=0.310, loss_ctc=115.236, loss_interctc_layer6=104.121, loss_interctc_layer12=87.375, loss_interctc_layer15=80.662, loss_interctc_layer21=118.209, loss=101.121, backward_time=0.367, grad_norm=67.748, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.163, optim0_lr0=1.231e-04, train_time=1.942
+[gpua002:0/64] 2024-01-19 02:00:23,320 (trainer:753) INFO: 11epoch:train:8401-8500batch: iter_time=8.853e-05, forward_time=0.143, loss_ctc=115.117, loss_interctc_layer6=104.390, loss_interctc_layer12=87.724, loss_interctc_layer15=81.209, loss_interctc_layer21=118.365, loss=101.361, backward_time=0.452, grad_norm=98.078, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.231e-04, train_time=2.063
+[gpua002:0/64] 2024-01-19 02:03:25,857 (trainer:753) INFO: 11epoch:train:8501-8600batch: iter_time=9.401e-05, forward_time=0.150, loss_ctc=86.894, loss_interctc_layer6=90.982, loss_interctc_layer12=76.590, loss_interctc_layer15=71.081, loss_interctc_layer21=89.081, loss=82.926, backward_time=0.399, grad_norm=79.562, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.230e-04, train_time=1.824
+[gpua002:0/64] 2024-01-19 02:07:34,247 (trainer:753) INFO: 11epoch:train:8601-8700batch: iter_time=9.140e-05, forward_time=0.142, loss_ctc=82.754, loss_interctc_layer6=86.842, loss_interctc_layer12=73.474, loss_interctc_layer15=68.233, loss_interctc_layer21=84.897, loss=79.240, backward_time=0.560, grad_norm=54.582, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.230e-04, train_time=2.485
+[gpua002:0/64] 2024-01-19 02:09:43,140 (multiple_iter_factory:32) INFO: Building 7th iter-factory...
+[gpua002:0/64] 2024-01-19 02:10:02,670 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 02:10:06,268 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe734b774f0>)
+[gpua002:0/64] 2024-01-19 02:10:06,268 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, 
+[gpua002:0/64] 2024-01-19 02:10:06,599 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 02:26:42,158 (trainer:753) INFO: 11epoch:train:8701-8800batch: iter_time=5.345, forward_time=0.144, loss_ctc=106.521, loss_interctc_layer6=110.983, loss_interctc_layer12=92.642, loss_interctc_layer15=85.368, loss_interctc_layer21=109.085, loss=100.920, backward_time=0.378, grad_norm=109.601, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.230e-04, train_time=11.477
+[gpua002:0/64] 2024-01-19 02:29:34,915 (trainer:753) INFO: 11epoch:train:8801-8900batch: iter_time=8.151e-05, forward_time=0.143, loss_ctc=81.380, loss_interctc_layer6=86.290, loss_interctc_layer12=72.614, loss_interctc_layer15=66.757, loss_interctc_layer21=83.552, loss=78.119, backward_time=0.385, grad_norm=53.245, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.139, optim0_lr0=1.229e-04, train_time=1.729
+[gpua002:0/64] 2024-01-19 02:32:02,966 (trainer:753) INFO: 11epoch:train:8901-9000batch: iter_time=8.365e-05, forward_time=0.142, loss_ctc=77.014, loss_interctc_layer6=79.884, loss_interctc_layer12=67.123, loss_interctc_layer15=61.933, loss_interctc_layer21=78.912, loss=72.973, backward_time=0.310, grad_norm=50.094, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.139, optim0_lr0=1.229e-04, train_time=1.480
+[gpua002:0/64] 2024-01-19 02:34:18,334 (trainer:753) INFO: 11epoch:train:9001-9100batch: iter_time=8.934e-05, forward_time=0.143, loss_ctc=77.147, loss_interctc_layer6=86.403, loss_interctc_layer12=72.617, loss_interctc_layer15=67.031, loss_interctc_layer21=78.776, loss=76.395, backward_time=0.299, grad_norm=81.230, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.139, optim0_lr0=1.228e-04, train_time=1.353
+[gpua002:0/64] 2024-01-19 02:37:07,694 (trainer:753) INFO: 11epoch:train:9101-9200batch: iter_time=8.825e-05, forward_time=0.144, loss_ctc=106.686, loss_interctc_layer6=108.009, loss_interctc_layer12=91.246, loss_interctc_layer15=84.580, loss_interctc_layer21=109.391, loss=99.982, backward_time=0.334, grad_norm=85.347, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.139, optim0_lr0=1.228e-04, train_time=1.694
+[gpua002:0/64] 2024-01-19 02:41:53,805 (trainer:753) INFO: 11epoch:train:9201-9300batch: iter_time=9.611e-05, forward_time=0.285, loss_ctc=94.305, loss_interctc_layer6=96.072, loss_interctc_layer12=79.578, loss_interctc_layer15=72.916, loss_interctc_layer21=96.902, loss=87.954, backward_time=0.695, grad_norm=67.678, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.157, optim0_lr0=1.228e-04, train_time=2.861
+[gpua002:0/64] 2024-01-19 02:45:51,064 (trainer:753) INFO: 11epoch:train:9301-9400batch: iter_time=9.196e-05, forward_time=0.143, loss_ctc=86.914, loss_interctc_layer6=91.223, loss_interctc_layer12=76.769, loss_interctc_layer15=70.833, loss_interctc_layer21=89.324, loss=83.012, backward_time=0.403, grad_norm=60.914, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.140, optim0_lr0=1.227e-04, train_time=2.372
+[gpua002:0/64] 2024-01-19 02:50:20,241 (trainer:753) INFO: 11epoch:train:9401-9500batch: iter_time=9.191e-05, forward_time=0.146, loss_ctc=94.304, loss_interctc_layer6=102.797, loss_interctc_layer12=87.494, loss_interctc_layer15=81.213, loss_interctc_layer21=96.221, loss=92.406, backward_time=0.504, grad_norm=71.378, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.140, optim0_lr0=1.227e-04, train_time=2.692
+[gpua002:0/64] 2024-01-19 02:54:03,710 (trainer:753) INFO: 11epoch:train:9501-9600batch: iter_time=9.438e-05, forward_time=0.153, loss_ctc=97.341, loss_interctc_layer6=98.678, loss_interctc_layer12=82.840, loss_interctc_layer15=76.705, loss_interctc_layer21=99.819, loss=91.077, backward_time=0.496, grad_norm=69.496, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.141, optim0_lr0=1.226e-04, train_time=2.233
+[gpua002:0/64] 2024-01-19 02:57:49,722 (trainer:753) INFO: 11epoch:train:9601-9700batch: iter_time=9.657e-05, forward_time=0.144, loss_ctc=109.070, loss_interctc_layer6=107.203, loss_interctc_layer12=90.194, loss_interctc_layer15=83.348, loss_interctc_layer21=111.559, loss=100.275, backward_time=0.536, grad_norm=66.806, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.139, optim0_lr0=1.226e-04, train_time=2.262
+[gpua002:0/64] 2024-01-19 03:00:32,627 (trainer:753) INFO: 11epoch:train:9701-9800batch: iter_time=9.407e-05, forward_time=0.142, loss_ctc=98.318, loss_interctc_layer6=92.309, loss_interctc_layer12=77.586, loss_interctc_layer15=71.815, loss_interctc_layer21=100.662, loss=88.138, backward_time=0.349, grad_norm=86.180, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.226e-04, train_time=1.629
+[gpua002:0/64] 2024-01-19 03:04:12,716 (trainer:753) INFO: 11epoch:train:9801-9900batch: iter_time=9.521e-05, forward_time=0.142, loss_ctc=75.031, loss_interctc_layer6=88.676, loss_interctc_layer12=74.496, loss_interctc_layer15=69.081, loss_interctc_layer21=76.816, loss=76.820, backward_time=0.504, grad_norm=64.288, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.138, optim0_lr0=1.225e-04, train_time=2.198
+[gpua002:0/64] 2024-01-19 03:08:25,951 (trainer:753) INFO: 11epoch:train:9901-10000batch: iter_time=6.219e-04, forward_time=0.373, loss_ctc=95.280, loss_interctc_layer6=102.980, loss_interctc_layer12=86.744, loss_interctc_layer15=80.159, loss_interctc_layer21=97.627, loss=92.558, backward_time=0.504, grad_norm=64.674, clip=100.000, loss_scale=3.961e+28, optim_step_time=0.173, optim0_lr0=1.225e-04, train_time=2.534
+[gpua002:0/64] 2024-01-19 03:08:46,024 (multiple_iter_factory:32) INFO: Building 8th iter-factory...
+[gpua002:0/64] 2024-01-19 03:09:05,149 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 03:09:08,708 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe72a8247c0>)
+[gpua002:0/64] 2024-01-19 03:09:08,708 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, 
+[gpua002:0/64] 2024-01-19 03:09:08,712 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 03:19:08,071 (trainer:753) INFO: 11epoch:train:10001-10100batch: iter_time=4.852, forward_time=0.144, loss_ctc=104.632, loss_interctc_layer6=101.982, loss_interctc_layer12=85.446, loss_interctc_layer15=78.656, loss_interctc_layer21=107.368, loss=95.617, backward_time=0.339, grad_norm=309.373, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.225e-04, train_time=6.422
+[gpua002:0/64] 2024-01-19 03:21:23,718 (trainer:753) INFO: 11epoch:train:10101-10200batch: iter_time=8.252e-05, forward_time=0.142, loss_ctc=82.500, loss_interctc_layer6=76.266, loss_interctc_layer12=63.795, loss_interctc_layer15=58.629, loss_interctc_layer21=84.559, loss=73.150, backward_time=0.300, grad_norm=61.916, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.224e-04, train_time=1.356
+[gpua002:0/64] 2024-01-19 03:24:25,968 (trainer:753) INFO: 11epoch:train:10201-10300batch: iter_time=8.648e-05, forward_time=0.153, loss_ctc=75.903, loss_interctc_layer6=80.516, loss_interctc_layer12=67.609, loss_interctc_layer15=62.248, loss_interctc_layer21=77.590, loss=72.773, backward_time=0.384, grad_norm=60.372, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.140, optim0_lr0=1.224e-04, train_time=1.822
+[gpua002:0/64] 2024-01-19 03:28:01,322 (trainer:753) INFO: 11epoch:train:10301-10400batch: iter_time=8.518e-05, forward_time=0.144, loss_ctc=97.443, loss_interctc_layer6=105.249, loss_interctc_layer12=88.859, loss_interctc_layer15=82.383, loss_interctc_layer21=99.997, loss=94.786, backward_time=0.427, grad_norm=77.319, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.223e-04, train_time=2.152
+[gpua002:0/64] 2024-01-19 03:33:08,994 (trainer:753) INFO: 11epoch:train:10401-10500batch: iter_time=8.502e-05, forward_time=0.144, loss_ctc=119.097, loss_interctc_layer6=100.597, loss_interctc_layer12=83.821, loss_interctc_layer15=76.921, loss_interctc_layer21=122.218, loss=100.531, backward_time=0.627, grad_norm=67.844, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.138, optim0_lr0=1.223e-04, train_time=3.078
+[gpua002:0/64] 2024-01-19 03:36:34,946 (trainer:753) INFO: 11epoch:train:10501-10600batch: iter_time=8.481e-05, forward_time=0.142, loss_ctc=76.143, loss_interctc_layer6=87.518, loss_interctc_layer12=72.731, loss_interctc_layer15=66.766, loss_interctc_layer21=77.889, loss=76.209, backward_time=0.367, grad_norm=94.279, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.138, optim0_lr0=1.223e-04, train_time=2.059
+[gpua002:0/64] 2024-01-19 03:39:23,491 (trainer:753) INFO: 11epoch:train:10601-10700batch: iter_time=8.886e-05, forward_time=0.145, loss_ctc=91.850, loss_interctc_layer6=94.985, loss_interctc_layer12=81.253, loss_interctc_layer15=75.653, loss_interctc_layer21=94.251, loss=87.599, backward_time=0.389, grad_norm=63.228, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.222e-04, train_time=1.685
+[gpua002:0/64] 2024-01-19 03:42:05,721 (trainer:753) INFO: 11epoch:train:10701-10800batch: iter_time=8.613e-05, forward_time=0.143, loss_ctc=95.954, loss_interctc_layer6=102.045, loss_interctc_layer12=85.932, loss_interctc_layer15=79.280, loss_interctc_layer21=98.123, loss=92.267, backward_time=0.335, grad_norm=94.459, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.222e-04, train_time=1.621
+[gpua002:0/64] 2024-01-19 03:45:08,313 (trainer:753) INFO: 11epoch:train:10801-10900batch: iter_time=8.980e-05, forward_time=0.221, loss_ctc=115.855, loss_interctc_layer6=104.830, loss_interctc_layer12=87.950, loss_interctc_layer15=80.964, loss_interctc_layer21=118.927, loss=101.705, backward_time=0.426, grad_norm=73.289, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.148, optim0_lr0=1.221e-04, train_time=1.827
+[gpua002:0/64] 2024-01-19 03:48:00,421 (trainer:753) INFO: 11epoch:train:10901-11000batch: iter_time=7.977e-05, forward_time=0.143, loss_ctc=114.979, loss_interctc_layer6=104.210, loss_interctc_layer12=87.551, loss_interctc_layer15=80.713, loss_interctc_layer21=117.776, loss=101.046, backward_time=0.354, grad_norm=68.732, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.221e-04, train_time=1.720
+[gpua002:0/64] 2024-01-19 03:51:35,658 (trainer:753) INFO: 11epoch:train:11001-11100batch: iter_time=8.071e-05, forward_time=0.143, loss_ctc=84.944, loss_interctc_layer6=90.640, loss_interctc_layer12=75.996, loss_interctc_layer15=70.179, loss_interctc_layer21=87.040, loss=81.760, backward_time=0.418, grad_norm=64.921, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.221e-04, train_time=2.153
+[gpua002:0/64] 2024-01-19 03:55:15,084 (trainer:753) INFO: 11epoch:train:11101-11200batch: iter_time=1.012e-04, forward_time=0.142, loss_ctc=83.047, loss_interctc_layer6=87.130, loss_interctc_layer12=74.151, loss_interctc_layer15=68.622, loss_interctc_layer21=85.227, loss=79.635, backward_time=0.373, grad_norm=65.094, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.138, optim0_lr0=1.220e-04, train_time=2.194
+[gpua002:0/64] 2024-01-19 03:57:29,471 (multiple_iter_factory:32) INFO: Building 9th iter-factory...
+[gpua002:0/64] 2024-01-19 03:57:49,243 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 03:57:52,775 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe75877c670>)
+[gpua002:0/64] 2024-01-19 03:57:52,775 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, 
+[gpua002:0/64] 2024-01-19 03:57:52,911 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 04:07:45,654 (trainer:753) INFO: 11epoch:train:11201-11300batch: iter_time=5.249, forward_time=0.152, loss_ctc=104.774, loss_interctc_layer6=109.855, loss_interctc_layer12=91.556, loss_interctc_layer15=84.180, loss_interctc_layer21=107.283, loss=99.530, backward_time=0.436, grad_norm=68.952, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.138, optim0_lr0=1.220e-04, train_time=7.505
+[gpua002:0/64] 2024-01-19 04:10:03,995 (trainer:753) INFO: 11epoch:train:11301-11400batch: iter_time=9.013e-05, forward_time=0.141, loss_ctc=81.048, loss_interctc_layer6=85.808, loss_interctc_layer12=71.948, loss_interctc_layer15=66.379, loss_interctc_layer21=83.156, loss=77.668, backward_time=0.300, grad_norm=53.880, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.138, optim0_lr0=1.220e-04, train_time=1.383
+[gpua002:0/64] 2024-01-19 04:12:58,956 (trainer:753) INFO: 11epoch:train:11401-11500batch: iter_time=1.020e-04, forward_time=0.142, loss_ctc=76.527, loss_interctc_layer6=80.281, loss_interctc_layer12=67.540, loss_interctc_layer15=62.289, loss_interctc_layer21=78.426, loss=73.013, backward_time=0.433, grad_norm=52.778, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.219e-04, train_time=1.749
+[gpua002:0/64] 2024-01-19 04:15:15,908 (trainer:753) INFO: 11epoch:train:11501-11600batch: iter_time=9.330e-05, forward_time=0.143, loss_ctc=77.012, loss_interctc_layer6=86.128, loss_interctc_layer12=72.387, loss_interctc_layer15=66.847, loss_interctc_layer21=78.835, loss=76.242, backward_time=0.315, grad_norm=76.207, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.219e-04, train_time=1.369
+[gpua002:0/64] 2024-01-19 04:18:38,486 (trainer:753) INFO: 11epoch:train:11601-11700batch: iter_time=9.336e-05, forward_time=0.272, loss_ctc=105.779, loss_interctc_layer6=108.305, loss_interctc_layer12=91.414, loss_interctc_layer15=84.596, loss_interctc_layer21=108.567, loss=99.732, backward_time=0.406, grad_norm=70.466, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.145, optim0_lr0=1.218e-04, train_time=2.025
+[gpua002:0/64] 2024-01-19 04:22:02,287 (trainer:753) INFO: 11epoch:train:11701-11800batch: iter_time=8.845e-05, forward_time=0.144, loss_ctc=92.859, loss_interctc_layer6=95.559, loss_interctc_layer12=79.121, loss_interctc_layer15=72.362, loss_interctc_layer21=95.216, loss=87.023, backward_time=0.388, grad_norm=81.481, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.218e-04, train_time=2.038
+[gpua002:0/64] 2024-01-19 04:25:22,580 (trainer:753) INFO: 11epoch:train:11801-11900batch: iter_time=8.961e-05, forward_time=0.142, loss_ctc=86.019, loss_interctc_layer6=90.582, loss_interctc_layer12=76.268, loss_interctc_layer15=70.290, loss_interctc_layer21=88.301, loss=82.292, backward_time=0.362, grad_norm=66.188, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.139, optim0_lr0=1.218e-04, train_time=2.003
+[gpua002:0/64] 2024-01-19 04:28:10,284 (trainer:753) INFO: 11epoch:train:11901-12000batch: iter_time=9.056e-05, forward_time=0.198, loss_ctc=94.906, loss_interctc_layer6=102.158, loss_interctc_layer12=87.004, loss_interctc_layer15=80.681, loss_interctc_layer21=97.066, loss=92.363, backward_time=0.315, grad_norm=66.307, clip=100.000, loss_scale=7.923e+28, optim_step_time=0.140, optim0_lr0=1.217e-04, train_time=1.677
+[gpua002:0/64] 2024-01-19 04:31:55,130 (trainer:753) INFO: 11epoch:train:12001-12100batch: iter_time=9.031e-05, forward_time=0.143, loss_ctc=96.828, loss_interctc_layer6=98.504, loss_interctc_layer12=82.659, loss_interctc_layer15=76.383, loss_interctc_layer21=99.430, loss=90.761, backward_time=0.387, grad_norm=414.451, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.140, optim0_lr0=1.217e-04, train_time=2.248
+[gpua002:0/64] 2024-01-19 04:36:27,365 (trainer:753) INFO: 11epoch:train:12101-12200batch: iter_time=9.265e-05, forward_time=0.144, loss_ctc=109.739, loss_interctc_layer6=107.542, loss_interctc_layer12=90.331, loss_interctc_layer15=83.503, loss_interctc_layer21=112.291, loss=100.681, backward_time=0.600, grad_norm=71.616, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.140, optim0_lr0=1.217e-04, train_time=2.722
+[gpua002:0/64] 2024-01-19 04:39:05,154 (trainer:753) INFO: 11epoch:train:12201-12300batch: iter_time=9.197e-05, forward_time=0.145, loss_ctc=98.065, loss_interctc_layer6=91.363, loss_interctc_layer12=76.856, loss_interctc_layer15=70.968, loss_interctc_layer21=100.562, loss=87.563, backward_time=0.315, grad_norm=66.628, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.216e-04, train_time=1.578
+[gpua002:0/64] 2024-01-19 04:42:31,654 (trainer:753) INFO: 11epoch:train:12301-12400batch: iter_time=9.573e-05, forward_time=0.150, loss_ctc=74.164, loss_interctc_layer6=88.007, loss_interctc_layer12=74.205, loss_interctc_layer15=68.798, loss_interctc_layer21=75.723, loss=76.179, backward_time=0.474, grad_norm=56.831, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.216e-04, train_time=2.063
+[gpua002:0/64] 2024-01-19 04:45:45,742 (trainer:753) INFO: 11epoch:train:12401-12500batch: iter_time=9.230e-05, forward_time=0.143, loss_ctc=94.644, loss_interctc_layer6=102.305, loss_interctc_layer12=86.045, loss_interctc_layer15=79.344, loss_interctc_layer21=96.598, loss=91.787, backward_time=0.461, grad_norm=195.884, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.215e-04, train_time=1.942
+[gpua002:0/64] 2024-01-19 04:46:05,771 (multiple_iter_factory:32) INFO: Building 10th iter-factory...
+[gpua002:0/64] 2024-01-19 04:46:25,055 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 04:46:28,650 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe73301d090>)
+[gpua002:0/64] 2024-01-19 04:46:28,650 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, 
+[gpua002:0/64] 2024-01-19 04:46:28,654 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 05:00:34,276 (trainer:753) INFO: 11epoch:train:12501-12600batch: iter_time=3.921, forward_time=0.190, loss_ctc=104.230, loss_interctc_layer6=101.551, loss_interctc_layer12=84.919, loss_interctc_layer15=78.074, loss_interctc_layer21=107.386, loss=95.232, backward_time=0.310, grad_norm=63.565, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.141, optim0_lr0=1.215e-04, train_time=8.883
+[gpua002:0/64] 2024-01-19 05:02:44,900 (trainer:753) INFO: 11epoch:train:12601-12700batch: iter_time=8.779e-05, forward_time=0.141, loss_ctc=83.489, loss_interctc_layer6=76.252, loss_interctc_layer12=63.794, loss_interctc_layer15=58.561, loss_interctc_layer21=85.792, loss=73.578, backward_time=0.296, grad_norm=64.145, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.215e-04, train_time=1.308
+[gpua002:0/64] 2024-01-19 05:06:36,968 (trainer:753) INFO: 11epoch:train:12701-12800batch: iter_time=9.377e-05, forward_time=0.142, loss_ctc=76.091, loss_interctc_layer6=80.711, loss_interctc_layer12=67.699, loss_interctc_layer15=62.436, loss_interctc_layer21=77.889, loss=72.965, backward_time=0.418, grad_norm=58.561, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.214e-04, train_time=2.320
+[gpua002:0/64] 2024-01-19 05:08:58,027 (trainer:753) INFO: 11epoch:train:12801-12900batch: iter_time=9.079e-05, forward_time=0.143, loss_ctc=97.220, loss_interctc_layer6=104.246, loss_interctc_layer12=88.004, loss_interctc_layer15=81.386, loss_interctc_layer21=99.241, loss=94.019, backward_time=0.319, grad_norm=77.067, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.214e-04, train_time=1.410
+[gpua002:0/64] 2024-01-19 05:12:34,386 (trainer:753) INFO: 11epoch:train:12901-13000batch: iter_time=9.242e-05, forward_time=0.143, loss_ctc=118.657, loss_interctc_layer6=100.475, loss_interctc_layer12=83.948, loss_interctc_layer15=76.988, loss_interctc_layer21=122.202, loss=100.454, backward_time=0.371, grad_norm=107.205, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.214e-04, train_time=2.163
+[gpua002:0/64] 2024-01-19 05:15:01,173 (trainer:753) INFO: 11epoch:train:13001-13100batch: iter_time=9.177e-05, forward_time=0.147, loss_ctc=75.821, loss_interctc_layer6=87.359, loss_interctc_layer12=72.511, loss_interctc_layer15=66.574, loss_interctc_layer21=77.696, loss=75.992, backward_time=0.306, grad_norm=71.270, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.213e-04, train_time=1.467
+[gpua002:0/64] 2024-01-19 05:18:49,365 (trainer:753) INFO: 11epoch:train:13101-13200batch: iter_time=8.572e-05, forward_time=0.143, loss_ctc=90.465, loss_interctc_layer6=93.607, loss_interctc_layer12=79.696, loss_interctc_layer15=73.985, loss_interctc_layer21=92.842, loss=86.119, backward_time=0.517, grad_norm=64.712, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.213e-04, train_time=2.282
+[gpua002:0/64] 2024-01-19 05:22:25,125 (trainer:753) INFO: 11epoch:train:13201-13300batch: iter_time=8.612e-05, forward_time=0.145, loss_ctc=95.692, loss_interctc_layer6=101.989, loss_interctc_layer12=86.030, loss_interctc_layer15=79.579, loss_interctc_layer21=97.811, loss=92.220, backward_time=0.379, grad_norm=102.407, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.212e-04, train_time=2.157
+[gpua002:0/64] 2024-01-19 05:25:54,316 (trainer:753) INFO: 11epoch:train:13301-13400batch: iter_time=0.002, forward_time=0.257, loss_ctc=114.003, loss_interctc_layer6=103.214, loss_interctc_layer12=86.476, loss_interctc_layer15=79.800, loss_interctc_layer21=116.760, loss=100.051, backward_time=0.453, grad_norm=72.274, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.145, optim0_lr0=1.212e-04, train_time=2.092
+[gpua002:0/64] 2024-01-19 05:29:29,842 (trainer:753) INFO: 11epoch:train:13401-13500batch: iter_time=8.800e-05, forward_time=0.144, loss_ctc=113.786, loss_interctc_layer6=102.942, loss_interctc_layer12=86.298, loss_interctc_layer15=79.745, loss_interctc_layer21=116.992, loss=99.953, backward_time=0.429, grad_norm=82.179, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.212e-04, train_time=2.154
+[gpua002:0/64] 2024-01-19 05:32:20,225 (trainer:753) INFO: 11epoch:train:13501-13600batch: iter_time=8.832e-05, forward_time=0.143, loss_ctc=85.165, loss_interctc_layer6=89.865, loss_interctc_layer12=75.364, loss_interctc_layer15=69.591, loss_interctc_layer21=87.229, loss=81.443, backward_time=0.323, grad_norm=57.460, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.211e-04, train_time=1.704
+[gpua002:0/64] 2024-01-19 05:35:33,453 (trainer:753) INFO: 11epoch:train:13601-13700batch: iter_time=8.447e-05, forward_time=0.142, loss_ctc=81.827, loss_interctc_layer6=86.329, loss_interctc_layer12=72.666, loss_interctc_layer15=67.255, loss_interctc_layer21=83.686, loss=78.353, backward_time=0.399, grad_norm=61.719, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.211e-04, train_time=1.932
+[gpua002:0/64] 2024-01-19 05:37:27,942 (multiple_iter_factory:32) INFO: Building 11th iter-factory...
+[gpua002:0/64] 2024-01-19 05:37:47,490 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 05:37:51,049 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe77975bf70>)
+[gpua002:0/64] 2024-01-19 05:37:51,049 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, 
+[gpua002:0/64] 2024-01-19 05:37:51,052 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 05:45:07,838 (trainer:753) INFO: 11epoch:train:13701-13800batch: iter_time=3.894, forward_time=0.150, loss_ctc=104.288, loss_interctc_layer6=109.414, loss_interctc_layer12=91.259, loss_interctc_layer15=83.960, loss_interctc_layer21=106.877, loss=99.160, backward_time=0.365, grad_norm=98.898, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.140, optim0_lr0=1.211e-04, train_time=5.744
+[gpua002:0/64] 2024-01-19 05:47:33,237 (trainer:753) INFO: 11epoch:train:13801-13900batch: iter_time=9.476e-05, forward_time=0.145, loss_ctc=80.153, loss_interctc_layer6=85.377, loss_interctc_layer12=71.405, loss_interctc_layer15=65.624, loss_interctc_layer21=82.032, loss=76.918, backward_time=0.315, grad_norm=64.928, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.210e-04, train_time=1.454
+[gpua002:0/64] 2024-01-19 05:50:56,755 (trainer:753) INFO: 11epoch:train:13901-14000batch: iter_time=9.102e-05, forward_time=0.142, loss_ctc=76.276, loss_interctc_layer6=79.393, loss_interctc_layer12=66.776, loss_interctc_layer15=61.513, loss_interctc_layer21=78.306, loss=72.453, backward_time=0.420, grad_norm=51.164, clip=100.000, loss_scale=1.585e+29, optim_step_time=0.139, optim0_lr0=1.210e-04, train_time=2.035
+[gpua002:0/64] 2024-01-19 05:53:47,881 (trainer:753) INFO: 11epoch:train:14001-14100batch: iter_time=8.399e-05, forward_time=0.148, loss_ctc=76.247, loss_interctc_layer6=85.956, loss_interctc_layer12=72.257, loss_interctc_layer15=66.560, loss_interctc_layer21=78.157, loss=75.836, backward_time=0.394, grad_norm=73.789, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.140, optim0_lr0=1.210e-04, train_time=1.711
+[gpua002:0/64] 2024-01-19 05:56:53,106 (trainer:753) INFO: 11epoch:train:14101-14200batch: iter_time=9.072e-05, forward_time=0.245, loss_ctc=105.470, loss_interctc_layer6=107.860, loss_interctc_layer12=90.774, loss_interctc_layer15=84.018, loss_interctc_layer21=108.995, loss=99.423, backward_time=0.395, grad_norm=84.600, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.147, optim0_lr0=1.209e-04, train_time=1.852
+[gpua002:0/64] 2024-01-19 06:00:06,383 (trainer:753) INFO: 11epoch:train:14201-14300batch: iter_time=8.787e-05, forward_time=0.142, loss_ctc=93.773, loss_interctc_layer6=96.305, loss_interctc_layer12=79.626, loss_interctc_layer15=73.066, loss_interctc_layer21=96.254, loss=87.805, backward_time=0.520, grad_norm=67.116, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.138, optim0_lr0=1.209e-04, train_time=1.933
+[gpua002:0/64] 2024-01-19 06:02:46,229 (trainer:753) INFO: 11epoch:train:14301-14400batch: iter_time=9.470e-05, forward_time=0.142, loss_ctc=86.318, loss_interctc_layer6=90.232, loss_interctc_layer12=75.916, loss_interctc_layer15=69.987, loss_interctc_layer21=88.596, loss=82.210, backward_time=0.331, grad_norm=77.373, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.138, optim0_lr0=1.208e-04, train_time=1.598
+[gpua002:0/64] 2024-01-19 06:06:27,281 (trainer:753) INFO: 11epoch:train:14401-14500batch: iter_time=1.002e-04, forward_time=0.142, loss_ctc=92.732, loss_interctc_layer6=101.152, loss_interctc_layer12=85.806, loss_interctc_layer15=79.621, loss_interctc_layer21=94.805, loss=90.823, backward_time=0.467, grad_norm=62.868, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.138, optim0_lr0=1.208e-04, train_time=2.211
+[gpua002:0/64] 2024-01-19 06:09:37,315 (trainer:753) INFO: 11epoch:train:14501-14600batch: iter_time=9.887e-05, forward_time=0.145, loss_ctc=95.764, loss_interctc_layer6=97.980, loss_interctc_layer12=82.229, loss_interctc_layer15=75.954, loss_interctc_layer21=98.557, loss=90.097, backward_time=0.341, grad_norm=72.660, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.138, optim0_lr0=1.208e-04, train_time=1.900
+[gpua002:0/64] 2024-01-19 06:12:12,193 (trainer:753) INFO: 11epoch:train:14601-14700batch: iter_time=9.907e-05, forward_time=0.144, loss_ctc=108.203, loss_interctc_layer6=106.842, loss_interctc_layer12=89.782, loss_interctc_layer15=82.862, loss_interctc_layer21=110.859, loss=99.710, backward_time=0.329, grad_norm=63.961, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.139, optim0_lr0=1.207e-04, train_time=1.549
+[gpua002:0/64] 2024-01-19 06:15:24,400 (trainer:753) INFO: 11epoch:train:14701-14800batch: iter_time=8.759e-05, forward_time=0.147, loss_ctc=97.487, loss_interctc_layer6=91.495, loss_interctc_layer12=76.886, loss_interctc_layer15=70.687, loss_interctc_layer21=100.201, loss=87.351, backward_time=0.554, grad_norm=69.448, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.138, optim0_lr0=1.207e-04, train_time=1.920
+[gpua002:0/64] 2024-01-19 06:19:10,271 (trainer:753) INFO: 11epoch:train:14801-14900batch: iter_time=9.145e-05, forward_time=0.145, loss_ctc=74.020, loss_interctc_layer6=87.721, loss_interctc_layer12=73.692, loss_interctc_layer15=68.028, loss_interctc_layer21=75.576, loss=75.807, backward_time=0.445, grad_norm=63.224, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.139, optim0_lr0=1.207e-04, train_time=2.260
+[gpua002:0/64] 2024-01-19 06:21:38,519 (trainer:753) INFO: 11epoch:train:14901-15000batch: iter_time=8.935e-05, forward_time=0.143, loss_ctc=94.153, loss_interctc_layer6=102.437, loss_interctc_layer12=85.831, loss_interctc_layer15=79.186, loss_interctc_layer21=96.229, loss=91.567, backward_time=0.333, grad_norm=67.058, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.138, optim0_lr0=1.206e-04, train_time=1.482
+[gpua002:0/64] 2024-01-19 06:52:09,002 (trainer:352) INFO: 11epoch results: [train] iter_time=0.345, forward_time=0.156, loss_ctc=94.532, loss_interctc_layer6=96.139, loss_interctc_layer12=81.029, loss_interctc_layer15=74.902, loss_interctc_layer21=96.794, loss=88.679, backward_time=0.383, grad_norm=85.357, clip=100.000, loss_scale=6.305e+28, optim_step_time=0.140, optim0_lr0=1.235e-04, train_time=2.244, time=9 hours, 21 minutes and 22.97 seconds, total_count=165000, gpu_max_cached_mem_GB=34.396, [valid] loss_ctc=55.722, cer_ctc=0.256, loss_interctc_layer6=59.223, cer_interctc_layer6=0.269, loss_interctc_layer12=46.314, cer_interctc_layer12=0.199, loss_interctc_layer15=41.850, cer_interctc_layer15=0.172, loss_interctc_layer21=58.130, cer_interctc_layer21=0.266, loss=52.248, time=30 minutes and 6.48 seconds, total_count=51381, gpu_max_cached_mem_GB=34.396
+[gpua002:0/64] 2024-01-19 06:52:30,487 (trainer:407) INFO: The best model has been updated: valid.cer_ctc, valid.loss_ctc, valid.total_count
+[gpua002:0/64] 2024-01-19 06:52:30,693 (trainer:461) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/6epoch.pth
+[gpua002:0/64] 2024-01-19 06:52:30,733 (trainer:286) INFO: 12/45epoch started. Estimated time to finish: 2 weeks, 1 day and 13 hours
+[gpua002:0/64] 2024-01-19 06:52:32,291 (multiple_iter_factory:32) INFO: Building 0th iter-factory...
+[gpua002:0/64] 2024-01-19 06:52:50,549 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 06:52:53,902 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.4", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.4", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.4", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.4", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7ff4c6fd9300>)
+[gpua002:0/64] 2024-01-19 06:52:53,903 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.4, 
+[gpua002:0/64] 2024-01-19 06:52:53,906 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 06:59:52,421 (trainer:753) INFO: 12epoch:train:1-100batch: iter_time=2.701, forward_time=0.178, loss_ctc=91.257, loss_interctc_layer6=89.594, loss_interctc_layer12=75.955, loss_interctc_layer15=70.441, loss_interctc_layer21=93.417, loss=84.133, backward_time=0.303, grad_norm=66.649, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.141, optim0_lr0=1.206e-04, train_time=4.407
+[gpua002:0/64] 2024-01-19 07:01:59,479 (trainer:753) INFO: 12epoch:train:101-200batch: iter_time=8.693e-05, forward_time=0.142, loss_ctc=87.071, loss_interctc_layer6=93.043, loss_interctc_layer12=78.781, loss_interctc_layer15=73.052, loss_interctc_layer21=89.255, loss=84.240, backward_time=0.297, grad_norm=60.900, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.139, optim0_lr0=1.205e-04, train_time=1.271
+[gpua002:0/64] 2024-01-19 07:04:57,699 (trainer:753) INFO: 12epoch:train:201-300batch: iter_time=9.106e-05, forward_time=0.142, loss_ctc=90.570, loss_interctc_layer6=92.628, loss_interctc_layer12=78.099, loss_interctc_layer15=72.222, loss_interctc_layer21=92.733, loss=85.251, backward_time=0.366, grad_norm=99.886, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.139, optim0_lr0=1.205e-04, train_time=1.782
+[gpua002:0/64] 2024-01-19 07:08:03,709 (trainer:753) INFO: 12epoch:train:301-400batch: iter_time=8.560e-05, forward_time=0.174, loss_ctc=86.132, loss_interctc_layer6=98.922, loss_interctc_layer12=84.648, loss_interctc_layer15=79.071, loss_interctc_layer21=87.681, loss=87.291, backward_time=0.339, grad_norm=74.760, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.142, optim0_lr0=1.205e-04, train_time=1.860
+[gpua002:0/64] 2024-01-19 07:11:00,800 (trainer:753) INFO: 12epoch:train:401-500batch: iter_time=9.541e-05, forward_time=0.185, loss_ctc=98.921, loss_interctc_layer6=95.553, loss_interctc_layer12=80.910, loss_interctc_layer15=75.147, loss_interctc_layer21=101.525, loss=90.411, backward_time=0.384, grad_norm=67.869, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.149, optim0_lr0=1.204e-04, train_time=1.769
+[gpua002:0/64] 2024-01-19 07:13:40,258 (trainer:753) INFO: 12epoch:train:501-600batch: iter_time=8.460e-05, forward_time=0.147, loss_ctc=86.210, loss_interctc_layer6=93.639, loss_interctc_layer12=79.338, loss_interctc_layer15=73.518, loss_interctc_layer21=88.051, loss=84.151, backward_time=0.380, grad_norm=66.826, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.139, optim0_lr0=1.204e-04, train_time=1.596
+[gpua002:0/64] 2024-01-19 07:16:38,390 (trainer:753) INFO: 12epoch:train:601-700batch: iter_time=5.769e-04, forward_time=0.169, loss_ctc=98.476, loss_interctc_layer6=95.259, loss_interctc_layer12=80.734, loss_interctc_layer15=74.766, loss_interctc_layer21=100.711, loss=89.989, backward_time=0.400, grad_norm=74.912, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.139, optim0_lr0=1.204e-04, train_time=1.781
+[gpua002:0/64] 2024-01-19 07:20:05,648 (trainer:753) INFO: 12epoch:train:701-800batch: iter_time=9.527e-05, forward_time=0.201, loss_ctc=76.245, loss_interctc_layer6=80.588, loss_interctc_layer12=67.907, loss_interctc_layer15=62.899, loss_interctc_layer21=77.782, loss=73.084, backward_time=0.343, grad_norm=71.297, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.145, optim0_lr0=1.203e-04, train_time=2.072
+[gpua002:0/64] 2024-01-19 07:22:42,641 (trainer:753) INFO: 12epoch:train:801-900batch: iter_time=9.460e-05, forward_time=0.143, loss_ctc=83.594, loss_interctc_layer6=87.966, loss_interctc_layer12=74.419, loss_interctc_layer15=68.809, loss_interctc_layer21=85.524, loss=80.062, backward_time=0.347, grad_norm=60.630, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.139, optim0_lr0=1.203e-04, train_time=1.570
+[gpua002:0/64] 2024-01-19 07:25:43,084 (trainer:753) INFO: 12epoch:train:901-1000batch: iter_time=9.325e-05, forward_time=0.183, loss_ctc=97.615, loss_interctc_layer6=101.945, loss_interctc_layer12=86.845, loss_interctc_layer15=81.079, loss_interctc_layer21=99.779, loss=93.452, backward_time=0.366, grad_norm=83.047, clip=100.000, loss_scale=3.169e+29, optim_step_time=0.158, optim0_lr0=1.203e-04, train_time=1.803
+[gpua002:0/64] 2024-01-19 07:28:20,398 (trainer:753) INFO: 12epoch:train:1001-1100batch: iter_time=1.241e-04, forward_time=0.145, loss_ctc=89.133, loss_interctc_layer6=89.323, loss_interctc_layer12=75.858, loss_interctc_layer15=70.544, loss_interctc_layer21=91.031, loss=83.178, backward_time=0.327, grad_norm=65.308, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.139, optim0_lr0=1.202e-04, train_time=1.574
+[gpua002:0/64] 2024-01-19 07:30:54,759 (trainer:753) INFO: 12epoch:train:1101-1200batch: iter_time=2.498e-04, forward_time=0.159, loss_ctc=89.432, loss_interctc_layer6=94.301, loss_interctc_layer12=79.664, loss_interctc_layer15=73.700, loss_interctc_layer21=91.635, loss=85.747, backward_time=0.338, grad_norm=65.523, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.202e-04, train_time=1.542
+[gpua002:0/64] 2024-01-19 07:32:31,781 (multiple_iter_factory:32) INFO: Building 1th iter-factory...
+[gpua002:0/64] 2024-01-19 07:32:50,941 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 07:32:54,463 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea44dab190>)
+[gpua002:0/64] 2024-01-19 07:32:54,464 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, 
+[gpua002:0/64] 2024-01-19 07:32:54,467 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 07:38:28,370 (trainer:753) INFO: 12epoch:train:1201-1300batch: iter_time=2.701, forward_time=0.171, loss_ctc=97.764, loss_interctc_layer6=92.501, loss_interctc_layer12=77.576, loss_interctc_layer15=71.644, loss_interctc_layer21=100.603, loss=88.018, backward_time=0.319, grad_norm=74.808, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.201e-04, train_time=4.537
+[gpua002:0/64] 2024-01-19 07:40:52,903 (trainer:753) INFO: 12epoch:train:1301-1400batch: iter_time=8.355e-05, forward_time=0.212, loss_ctc=84.159, loss_interctc_layer6=87.489, loss_interctc_layer12=74.273, loss_interctc_layer15=68.888, loss_interctc_layer21=86.039, loss=80.170, backward_time=0.341, grad_norm=74.297, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.143, optim0_lr0=1.201e-04, train_time=1.443
+[gpua002:0/64] 2024-01-19 07:43:16,983 (trainer:753) INFO: 12epoch:train:1401-1500batch: iter_time=3.339e-04, forward_time=0.188, loss_ctc=92.212, loss_interctc_layer6=98.411, loss_interctc_layer12=83.248, loss_interctc_layer15=76.923, loss_interctc_layer21=94.382, loss=89.035, backward_time=0.314, grad_norm=60.892, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.142, optim0_lr0=1.201e-04, train_time=1.442
+[gpua002:0/64] 2024-01-19 07:45:33,560 (trainer:753) INFO: 12epoch:train:1501-1600batch: iter_time=9.033e-05, forward_time=0.142, loss_ctc=87.503, loss_interctc_layer6=100.199, loss_interctc_layer12=85.028, loss_interctc_layer15=78.701, loss_interctc_layer21=89.228, loss=88.132, backward_time=0.300, grad_norm=70.739, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.200e-04, train_time=1.366
+[gpua002:0/64] 2024-01-19 07:47:52,368 (trainer:753) INFO: 12epoch:train:1601-1700batch: iter_time=9.049e-05, forward_time=0.142, loss_ctc=85.033, loss_interctc_layer6=94.157, loss_interctc_layer12=79.215, loss_interctc_layer15=73.024, loss_interctc_layer21=86.885, loss=83.663, backward_time=0.304, grad_norm=63.124, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.200e-04, train_time=1.388
+[gpua002:0/64] 2024-01-19 07:50:18,187 (trainer:753) INFO: 12epoch:train:1701-1800batch: iter_time=8.709e-05, forward_time=0.142, loss_ctc=84.447, loss_interctc_layer6=92.040, loss_interctc_layer12=77.833, loss_interctc_layer15=72.127, loss_interctc_layer21=86.720, loss=82.633, backward_time=0.346, grad_norm=152.421, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.200e-04, train_time=1.458
+[gpua002:0/64] 2024-01-19 07:52:59,065 (trainer:753) INFO: 12epoch:train:1801-1900batch: iter_time=8.447e-04, forward_time=0.235, loss_ctc=85.341, loss_interctc_layer6=93.159, loss_interctc_layer12=78.895, loss_interctc_layer15=72.957, loss_interctc_layer21=87.324, loss=83.535, backward_time=0.357, grad_norm=53.102, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.164, optim0_lr0=1.199e-04, train_time=1.607
+[gpua002:0/64] 2024-01-19 07:55:42,541 (trainer:753) INFO: 12epoch:train:1901-2000batch: iter_time=9.044e-05, forward_time=0.143, loss_ctc=80.674, loss_interctc_layer6=87.299, loss_interctc_layer12=73.990, loss_interctc_layer15=68.632, loss_interctc_layer21=82.574, loss=78.634, backward_time=0.335, grad_norm=66.081, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.139, optim0_lr0=1.199e-04, train_time=1.636
+[gpua002:0/64] 2024-01-19 07:58:30,773 (trainer:753) INFO: 12epoch:train:2001-2100batch: iter_time=9.052e-05, forward_time=0.142, loss_ctc=72.201, loss_interctc_layer6=79.460, loss_interctc_layer12=66.340, loss_interctc_layer15=60.990, loss_interctc_layer21=73.942, loss=70.586, backward_time=0.339, grad_norm=58.448, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.139, optim0_lr0=1.199e-04, train_time=1.682
+[gpua002:0/64] 2024-01-19 08:01:24,017 (trainer:753) INFO: 12epoch:train:2101-2200batch: iter_time=9.300e-05, forward_time=0.142, loss_ctc=90.642, loss_interctc_layer6=99.034, loss_interctc_layer12=84.451, loss_interctc_layer15=78.657, loss_interctc_layer21=92.548, loss=89.066, backward_time=0.402, grad_norm=65.873, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.198e-04, train_time=1.732
+[gpua002:0/64] 2024-01-19 08:04:08,952 (trainer:753) INFO: 12epoch:train:2201-2300batch: iter_time=7.996e-05, forward_time=0.151, loss_ctc=79.311, loss_interctc_layer6=85.211, loss_interctc_layer12=71.293, loss_interctc_layer15=65.738, loss_interctc_layer21=81.229, loss=76.556, backward_time=0.400, grad_norm=68.691, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.139, optim0_lr0=1.198e-04, train_time=1.649
+[gpua002:0/64] 2024-01-19 08:06:55,348 (trainer:753) INFO: 12epoch:train:2301-2400batch: iter_time=6.520e-04, forward_time=0.195, loss_ctc=90.176, loss_interctc_layer6=100.164, loss_interctc_layer12=84.821, loss_interctc_layer15=78.702, loss_interctc_layer21=92.277, loss=89.228, backward_time=0.322, grad_norm=70.818, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.139, optim0_lr0=1.198e-04, train_time=1.664
+[gpua002:0/64] 2024-01-19 08:09:32,017 (trainer:753) INFO: 12epoch:train:2401-2500batch: iter_time=0.001, forward_time=0.195, loss_ctc=91.618, loss_interctc_layer6=92.704, loss_interctc_layer12=77.987, loss_interctc_layer15=72.031, loss_interctc_layer21=93.647, loss=85.597, backward_time=0.341, grad_norm=56.581, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.141, optim0_lr0=1.197e-04, train_time=1.566
+[gpua002:0/64] 2024-01-19 08:09:52,125 (multiple_iter_factory:32) INFO: Building 2th iter-factory...
+[gpua002:0/64] 2024-01-19 08:10:11,487 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 08:10:15,048 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.5", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.5", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.5", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.5", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea4d7a5990>)
+[gpua002:0/64] 2024-01-19 08:10:15,048 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.5, 
+[gpua002:0/64] 2024-01-19 08:10:15,051 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 08:15:43,849 (trainer:753) INFO: 12epoch:train:2501-2600batch: iter_time=2.304, forward_time=0.144, loss_ctc=86.468, loss_interctc_layer6=88.419, loss_interctc_layer12=74.394, loss_interctc_layer15=68.883, loss_interctc_layer21=88.858, loss=81.404, backward_time=0.301, grad_norm=59.869, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.139, optim0_lr0=1.197e-04, train_time=3.719
+[gpua002:0/64] 2024-01-19 08:18:08,718 (trainer:753) INFO: 12epoch:train:2601-2700batch: iter_time=8.280e-05, forward_time=0.144, loss_ctc=82.263, loss_interctc_layer6=93.079, loss_interctc_layer12=78.798, loss_interctc_layer15=72.927, loss_interctc_layer21=84.256, loss=82.265, backward_time=0.335, grad_norm=122.359, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.196e-04, train_time=1.448
+[gpua002:0/64] 2024-01-19 08:20:31,054 (trainer:753) INFO: 12epoch:train:2701-2800batch: iter_time=8.926e-05, forward_time=0.184, loss_ctc=86.106, loss_interctc_layer6=91.454, loss_interctc_layer12=76.687, loss_interctc_layer15=70.807, loss_interctc_layer21=88.267, loss=82.664, backward_time=0.303, grad_norm=63.168, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.196e-04, train_time=1.423
+[gpua002:0/64] 2024-01-19 08:23:47,114 (trainer:753) INFO: 12epoch:train:2801-2900batch: iter_time=9.120e-05, forward_time=0.155, loss_ctc=79.902, loss_interctc_layer6=94.813, loss_interctc_layer12=80.387, loss_interctc_layer15=74.775, loss_interctc_layer21=81.555, loss=82.286, backward_time=0.345, grad_norm=618.419, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.138, optim0_lr0=1.196e-04, train_time=1.960
+[gpua002:0/64] 2024-01-19 08:28:27,622 (trainer:753) INFO: 12epoch:train:2901-3000batch: iter_time=0.009, forward_time=0.367, loss_ctc=87.321, loss_interctc_layer6=92.692, loss_interctc_layer12=78.185, loss_interctc_layer15=72.374, loss_interctc_layer21=89.402, loss=83.995, backward_time=0.640, grad_norm=60.564, clip=100.000, loss_scale=6.338e+29, optim_step_time=0.147, optim0_lr0=1.195e-04, train_time=2.804
+[gpua002:0/64] 2024-01-19 08:31:19,025 (trainer:753) INFO: 12epoch:train:3001-3100batch: iter_time=8.921e-05, forward_time=0.143, loss_ctc=81.010, loss_interctc_layer6=91.805, loss_interctc_layer12=77.538, loss_interctc_layer15=71.777, loss_interctc_layer21=82.754, loss=80.977, backward_time=0.361, grad_norm=95.833, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.195e-04, train_time=1.714
+[gpua002:0/64] 2024-01-19 08:34:14,987 (trainer:753) INFO: 12epoch:train:3101-3200batch: iter_time=1.002e-04, forward_time=0.177, loss_ctc=87.964, loss_interctc_layer6=94.348, loss_interctc_layer12=80.024, loss_interctc_layer15=73.912, loss_interctc_layer21=90.041, loss=85.258, backward_time=0.377, grad_norm=70.879, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.195e-04, train_time=1.761
+[gpua002:0/64] 2024-01-19 08:38:00,439 (trainer:753) INFO: 12epoch:train:3201-3300batch: iter_time=8.719e-05, forward_time=0.143, loss_ctc=72.238, loss_interctc_layer6=79.890, loss_interctc_layer12=67.004, loss_interctc_layer15=61.744, loss_interctc_layer21=74.001, loss=70.976, backward_time=0.369, grad_norm=50.541, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.194e-04, train_time=2.254
+[gpua002:0/64] 2024-01-19 08:40:12,430 (trainer:753) INFO: 12epoch:train:3301-3400batch: iter_time=8.945e-05, forward_time=0.143, loss_ctc=80.121, loss_interctc_layer6=87.602, loss_interctc_layer12=73.573, loss_interctc_layer15=67.983, loss_interctc_layer21=82.108, loss=78.277, backward_time=0.298, grad_norm=84.369, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.194e-04, train_time=1.320
+[gpua002:0/64] 2024-01-19 08:44:03,052 (trainer:753) INFO: 12epoch:train:3401-3500batch: iter_time=1.049e-04, forward_time=0.149, loss_ctc=92.816, loss_interctc_layer6=99.536, loss_interctc_layer12=83.650, loss_interctc_layer15=77.382, loss_interctc_layer21=95.137, loss=89.704, backward_time=0.440, grad_norm=66.881, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.194e-04, train_time=2.306
+[gpua002:0/64] 2024-01-19 08:47:48,870 (trainer:753) INFO: 12epoch:train:3501-3600batch: iter_time=0.027, forward_time=0.142, loss_ctc=81.707, loss_interctc_layer6=88.381, loss_interctc_layer12=74.799, loss_interctc_layer15=69.214, loss_interctc_layer21=83.522, loss=79.524, backward_time=0.324, grad_norm=61.118, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.193e-04, train_time=2.258
+[gpua002:0/64] 2024-01-19 08:51:11,210 (trainer:753) INFO: 12epoch:train:3601-3700batch: iter_time=1.061e-04, forward_time=0.328, loss_ctc=83.881, loss_interctc_layer6=93.795, loss_interctc_layer12=78.772, loss_interctc_layer15=72.733, loss_interctc_layer21=85.655, loss=82.967, backward_time=0.418, grad_norm=67.656, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.156, optim0_lr0=1.193e-04, train_time=2.023
+[gpua002:0/64] 2024-01-19 08:52:41,993 (multiple_iter_factory:32) INFO: Building 3th iter-factory...
+[gpua002:0/64] 2024-01-19 08:53:01,141 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 08:53:04,884 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe6f963bfa0>)
+[gpua002:0/64] 2024-01-19 08:53:04,885 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, 
+[gpua002:0/64] 2024-01-19 08:53:04,888 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 09:01:42,781 (trainer:753) INFO: 12epoch:train:3701-3800batch: iter_time=4.391, forward_time=0.144, loss_ctc=93.034, loss_interctc_layer6=90.695, loss_interctc_layer12=75.928, loss_interctc_layer15=69.871, loss_interctc_layer21=95.458, loss=84.997, backward_time=0.307, grad_norm=58.549, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.193e-04, train_time=6.316
+[gpua002:0/64] 2024-01-19 09:04:02,021 (trainer:753) INFO: 12epoch:train:3801-3900batch: iter_time=9.361e-05, forward_time=0.142, loss_ctc=88.159, loss_interctc_layer6=86.854, loss_interctc_layer12=73.607, loss_interctc_layer15=68.183, loss_interctc_layer21=90.437, loss=81.448, backward_time=0.299, grad_norm=72.808, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.192e-04, train_time=1.392
+[gpua002:0/64] 2024-01-19 09:06:41,815 (trainer:753) INFO: 12epoch:train:3901-4000batch: iter_time=8.241e-05, forward_time=0.143, loss_ctc=94.562, loss_interctc_layer6=97.766, loss_interctc_layer12=82.071, loss_interctc_layer15=75.903, loss_interctc_layer21=97.055, loss=89.471, backward_time=0.339, grad_norm=66.709, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.192e-04, train_time=1.598
+[gpua002:0/64] 2024-01-19 09:09:12,844 (trainer:753) INFO: 12epoch:train:4001-4100batch: iter_time=8.408e-05, forward_time=0.152, loss_ctc=89.527, loss_interctc_layer6=98.423, loss_interctc_layer12=85.490, loss_interctc_layer15=78.508, loss_interctc_layer21=91.137, loss=88.617, backward_time=0.329, grad_norm=73.927, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.140, optim0_lr0=1.192e-04, train_time=1.510
+[gpua002:0/64] 2024-01-19 09:11:53,754 (trainer:753) INFO: 12epoch:train:4101-4200batch: iter_time=8.433e-05, forward_time=0.158, loss_ctc=89.569, loss_interctc_layer6=92.431, loss_interctc_layer12=77.421, loss_interctc_layer15=71.566, loss_interctc_layer21=91.745, loss=84.547, backward_time=0.340, grad_norm=64.351, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.191e-04, train_time=1.609
+[gpua002:0/64] 2024-01-19 09:15:30,928 (trainer:753) INFO: 12epoch:train:4201-4300batch: iter_time=9.021e-05, forward_time=0.285, loss_ctc=88.834, loss_interctc_layer6=90.985, loss_interctc_layer12=76.513, loss_interctc_layer15=70.736, loss_interctc_layer21=91.240, loss=83.662, backward_time=0.475, grad_norm=64.445, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.157, optim0_lr0=1.191e-04, train_time=2.171
+[gpua002:0/64] 2024-01-19 09:18:46,142 (trainer:753) INFO: 12epoch:train:4301-4400batch: iter_time=8.923e-05, forward_time=0.143, loss_ctc=88.038, loss_interctc_layer6=92.511, loss_interctc_layer12=78.013, loss_interctc_layer15=71.975, loss_interctc_layer21=90.186, loss=84.144, backward_time=0.400, grad_norm=58.943, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.190e-04, train_time=1.950
+[gpua002:0/64] 2024-01-19 09:21:54,411 (trainer:753) INFO: 12epoch:train:4401-4500batch: iter_time=8.632e-05, forward_time=0.142, loss_ctc=89.367, loss_interctc_layer6=86.920, loss_interctc_layer12=73.414, loss_interctc_layer15=67.964, loss_interctc_layer21=91.617, loss=81.856, backward_time=0.446, grad_norm=70.942, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.190e-04, train_time=1.884
+[gpua002:0/64] 2024-01-19 09:25:24,240 (trainer:753) INFO: 12epoch:train:4501-4600batch: iter_time=9.530e-05, forward_time=0.142, loss_ctc=71.478, loss_interctc_layer6=78.896, loss_interctc_layer12=65.712, loss_interctc_layer15=60.348, loss_interctc_layer21=73.088, loss=69.904, backward_time=0.409, grad_norm=70.885, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.190e-04, train_time=2.098
+[gpua002:0/64] 2024-01-19 09:28:52,367 (trainer:753) INFO: 12epoch:train:4601-4700batch: iter_time=9.704e-05, forward_time=0.142, loss_ctc=92.162, loss_interctc_layer6=97.405, loss_interctc_layer12=82.706, loss_interctc_layer15=77.272, loss_interctc_layer21=94.156, loss=88.740, backward_time=0.505, grad_norm=73.318, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.189e-04, train_time=2.077
+[gpua002:0/64] 2024-01-19 09:31:44,630 (trainer:753) INFO: 12epoch:train:4701-4800batch: iter_time=8.946e-05, forward_time=0.142, loss_ctc=83.096, loss_interctc_layer6=85.589, loss_interctc_layer12=71.425, loss_interctc_layer15=65.773, loss_interctc_layer21=85.268, loss=78.230, backward_time=0.379, grad_norm=106.613, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.139, optim0_lr0=1.189e-04, train_time=1.726
+[gpua002:0/64] 2024-01-19 09:35:22,064 (trainer:753) INFO: 12epoch:train:4801-4900batch: iter_time=9.001e-05, forward_time=0.143, loss_ctc=95.934, loss_interctc_layer6=100.052, loss_interctc_layer12=84.343, loss_interctc_layer15=77.794, loss_interctc_layer21=98.305, loss=91.286, backward_time=0.388, grad_norm=71.154, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.189e-04, train_time=2.174
+[gpua002:0/64] 2024-01-19 09:38:04,305 (trainer:753) INFO: 12epoch:train:4901-5000batch: iter_time=9.720e-04, forward_time=0.162, loss_ctc=96.676, loss_interctc_layer6=92.435, loss_interctc_layer12=77.578, loss_interctc_layer15=71.433, loss_interctc_layer21=99.142, loss=87.453, backward_time=0.342, grad_norm=72.791, clip=100.000, loss_scale=1.268e+30, optim_step_time=0.138, optim0_lr0=1.188e-04, train_time=1.622
+[gpua002:0/64] 2024-01-19 09:38:24,335 (multiple_iter_factory:32) INFO: Building 4th iter-factory...
+[gpua002:0/64] 2024-01-19 09:38:43,475 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 09:38:47,019 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.6", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.6", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.6", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.6", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea44dabb80>)
+[gpua002:0/64] 2024-01-19 09:38:47,019 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.6, 
+[gpua002:0/64] 2024-01-19 09:38:47,023 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 09:48:54,911 (trainer:753) INFO: 12epoch:train:5001-5100batch: iter_time=4.240, forward_time=0.213, loss_ctc=88.738, loss_interctc_layer6=87.530, loss_interctc_layer12=73.445, loss_interctc_layer15=67.928, loss_interctc_layer21=90.928, loss=81.714, backward_time=0.315, grad_norm=83.971, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.139, optim0_lr0=1.188e-04, train_time=6.506
+[gpua002:0/64] 2024-01-19 09:51:03,209 (trainer:753) INFO: 12epoch:train:5101-5200batch: iter_time=8.221e-05, forward_time=0.142, loss_ctc=86.393, loss_interctc_layer6=92.565, loss_interctc_layer12=78.245, loss_interctc_layer15=71.993, loss_interctc_layer21=88.405, loss=83.520, backward_time=0.297, grad_norm=62.300, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.138, optim0_lr0=1.188e-04, train_time=1.283
+[gpua002:0/64] 2024-01-19 09:54:11,330 (trainer:753) INFO: 12epoch:train:5201-5300batch: iter_time=8.418e-05, forward_time=0.143, loss_ctc=89.713, loss_interctc_layer6=92.017, loss_interctc_layer12=77.105, loss_interctc_layer15=71.126, loss_interctc_layer21=91.793, loss=84.351, backward_time=0.352, grad_norm=66.324, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.138, optim0_lr0=1.187e-04, train_time=1.880
+[gpua002:0/64] 2024-01-19 09:57:24,613 (trainer:753) INFO: 12epoch:train:5301-5400batch: iter_time=8.511e-05, forward_time=0.143, loss_ctc=80.831, loss_interctc_layer6=93.635, loss_interctc_layer12=79.221, loss_interctc_layer15=73.676, loss_interctc_layer21=82.788, loss=82.030, backward_time=0.487, grad_norm=93.600, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.138, optim0_lr0=1.187e-04, train_time=1.934
+[gpua002:0/64] 2024-01-19 10:00:28,303 (trainer:753) INFO: 12epoch:train:5401-5500batch: iter_time=8.830e-05, forward_time=0.142, loss_ctc=95.943, loss_interctc_layer6=93.387, loss_interctc_layer12=78.560, loss_interctc_layer15=72.583, loss_interctc_layer21=98.521, loss=87.799, backward_time=0.381, grad_norm=67.746, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.138, optim0_lr0=1.187e-04, train_time=1.837
+[gpua002:0/64] 2024-01-19 10:03:47,268 (trainer:753) INFO: 12epoch:train:5501-5600batch: iter_time=8.889e-05, forward_time=0.142, loss_ctc=82.904, loss_interctc_layer6=91.754, loss_interctc_layer12=77.279, loss_interctc_layer15=71.188, loss_interctc_layer21=85.131, loss=81.651, backward_time=0.403, grad_norm=84.446, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.138, optim0_lr0=1.186e-04, train_time=1.986
+[gpua002:0/64] 2024-01-19 10:06:27,237 (trainer:753) INFO: 12epoch:train:5601-5700batch: iter_time=8.299e-05, forward_time=0.143, loss_ctc=96.125, loss_interctc_layer6=93.744, loss_interctc_layer12=79.303, loss_interctc_layer15=73.070, loss_interctc_layer21=98.539, loss=88.156, backward_time=0.377, grad_norm=63.477, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.138, optim0_lr0=1.186e-04, train_time=1.603
+[gpua002:0/64] 2024-01-19 10:09:27,293 (trainer:753) INFO: 12epoch:train:5701-5800batch: iter_time=0.001, forward_time=0.171, loss_ctc=73.039, loss_interctc_layer6=79.410, loss_interctc_layer12=66.457, loss_interctc_layer15=61.166, loss_interctc_layer21=74.870, loss=70.988, backward_time=0.344, grad_norm=51.804, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.139, optim0_lr0=1.186e-04, train_time=1.800
+[gpua002:0/64] 2024-01-19 10:13:05,260 (trainer:753) INFO: 12epoch:train:5801-5900batch: iter_time=4.941e-04, forward_time=0.175, loss_ctc=83.323, loss_interctc_layer6=87.725, loss_interctc_layer12=73.631, loss_interctc_layer15=67.815, loss_interctc_layer21=85.391, loss=79.577, backward_time=0.408, grad_norm=60.781, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.139, optim0_lr0=1.185e-04, train_time=2.179
+[gpua002:0/64] 2024-01-19 10:16:31,590 (trainer:753) INFO: 12epoch:train:5901-6000batch: iter_time=0.005, forward_time=0.355, loss_ctc=94.167, loss_interctc_layer6=99.470, loss_interctc_layer12=83.417, loss_interctc_layer15=77.177, loss_interctc_layer21=96.532, loss=90.153, backward_time=0.392, grad_norm=62.052, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.144, optim0_lr0=1.185e-04, train_time=2.054
+[gpua002:0/64] 2024-01-19 10:19:57,005 (trainer:753) INFO: 12epoch:train:6001-6100batch: iter_time=9.236e-05, forward_time=0.142, loss_ctc=87.171, loss_interctc_layer6=88.036, loss_interctc_layer12=74.341, loss_interctc_layer15=68.878, loss_interctc_layer21=89.132, loss=81.512, backward_time=0.401, grad_norm=57.069, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.139, optim0_lr0=1.185e-04, train_time=2.055
+[gpua002:0/64] 2024-01-19 10:23:23,379 (trainer:753) INFO: 12epoch:train:6101-6200batch: iter_time=8.798e-05, forward_time=0.143, loss_ctc=87.146, loss_interctc_layer6=92.868, loss_interctc_layer12=77.871, loss_interctc_layer15=71.648, loss_interctc_layer21=89.311, loss=83.769, backward_time=0.417, grad_norm=72.206, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.139, optim0_lr0=1.184e-04, train_time=2.064
+[gpua002:0/64] 2024-01-19 10:26:15,277 (multiple_iter_factory:32) INFO: Building 5th iter-factory...
+[gpua002:0/64] 2024-01-19 10:26:34,491 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 10:26:38,331 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea89397e20>)
+[gpua002:0/64] 2024-01-19 10:26:38,331 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, 
+[gpua002:0/64] 2024-01-19 10:26:38,409 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 10:35:53,100 (trainer:753) INFO: 12epoch:train:6201-6300batch: iter_time=4.824, forward_time=0.144, loss_ctc=96.129, loss_interctc_layer6=90.264, loss_interctc_layer12=75.450, loss_interctc_layer15=69.386, loss_interctc_layer21=98.999, loss=86.046, backward_time=0.416, grad_norm=63.945, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.140, optim0_lr0=1.184e-04, train_time=7.496
+[gpua002:0/64] 2024-01-19 10:38:00,713 (trainer:753) INFO: 12epoch:train:6301-6400batch: iter_time=8.634e-05, forward_time=0.143, loss_ctc=87.068, loss_interctc_layer6=85.819, loss_interctc_layer12=72.484, loss_interctc_layer15=67.194, loss_interctc_layer21=89.380, loss=80.389, backward_time=0.297, grad_norm=59.115, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.140, optim0_lr0=1.183e-04, train_time=1.277
+[gpua002:0/64] 2024-01-19 10:40:12,662 (trainer:753) INFO: 12epoch:train:6401-6500batch: iter_time=8.840e-05, forward_time=0.144, loss_ctc=92.612, loss_interctc_layer6=97.276, loss_interctc_layer12=81.727, loss_interctc_layer15=75.483, loss_interctc_layer21=94.897, loss=88.399, backward_time=0.297, grad_norm=71.233, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.140, optim0_lr0=1.183e-04, train_time=1.319
+[gpua002:0/64] 2024-01-19 10:43:14,635 (trainer:753) INFO: 12epoch:train:6501-6600batch: iter_time=9.269e-05, forward_time=0.176, loss_ctc=88.849, loss_interctc_layer6=98.425, loss_interctc_layer12=83.390, loss_interctc_layer15=78.107, loss_interctc_layer21=93.114, loss=88.377, backward_time=0.462, grad_norm=115.556, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.143, optim0_lr0=1.183e-04, train_time=1.820
+[gpua002:0/64] 2024-01-19 10:47:38,970 (trainer:753) INFO: 12epoch:train:6601-6700batch: iter_time=8.875e-05, forward_time=0.279, loss_ctc=88.365, loss_interctc_layer6=92.323, loss_interctc_layer12=77.324, loss_interctc_layer15=71.384, loss_interctc_layer21=90.663, loss=84.012, backward_time=0.480, grad_norm=62.095, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.154, optim0_lr0=1.182e-04, train_time=2.642
+[gpua002:0/64] 2024-01-19 10:51:03,279 (trainer:753) INFO: 12epoch:train:6701-6800batch: iter_time=9.337e-05, forward_time=0.143, loss_ctc=87.441, loss_interctc_layer6=89.947, loss_interctc_layer12=75.901, loss_interctc_layer15=69.843, loss_interctc_layer21=89.684, loss=82.563, backward_time=0.440, grad_norm=60.348, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.140, optim0_lr0=1.182e-04, train_time=2.043
+[gpua002:0/64] 2024-01-19 10:54:25,590 (trainer:753) INFO: 12epoch:train:6801-6900batch: iter_time=8.760e-05, forward_time=0.142, loss_ctc=86.074, loss_interctc_layer6=92.095, loss_interctc_layer12=77.637, loss_interctc_layer15=71.616, loss_interctc_layer21=87.841, loss=83.053, backward_time=0.431, grad_norm=57.720, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.139, optim0_lr0=1.182e-04, train_time=2.023
+[gpua002:0/64] 2024-01-19 10:57:25,564 (trainer:753) INFO: 12epoch:train:6901-7000batch: iter_time=8.602e-05, forward_time=0.141, loss_ctc=87.053, loss_interctc_layer6=85.787, loss_interctc_layer12=72.514, loss_interctc_layer15=67.036, loss_interctc_layer21=89.112, loss=80.300, backward_time=0.351, grad_norm=230.651, clip=100.000, loss_scale=2.535e+30, optim_step_time=0.139, optim0_lr0=1.181e-04, train_time=1.800
+[gpua002:0/64] 2024-01-19 11:00:36,071 (trainer:753) INFO: 12epoch:train:7001-7100batch: iter_time=8.839e-05, forward_time=0.142, loss_ctc=72.147, loss_interctc_layer6=78.760, loss_interctc_layer12=65.544, loss_interctc_layer15=60.164, loss_interctc_layer21=73.912, loss=70.105, backward_time=0.376, grad_norm=70.879, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.181e-04, train_time=1.903
+[gpua002:0/64] 2024-01-19 11:03:48,685 (trainer:753) INFO: 12epoch:train:7101-7200batch: iter_time=9.266e-05, forward_time=0.142, loss_ctc=91.143, loss_interctc_layer6=96.907, loss_interctc_layer12=81.978, loss_interctc_layer15=76.471, loss_interctc_layer21=93.721, loss=88.044, backward_time=0.401, grad_norm=69.037, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.181e-04, train_time=1.927
+[gpua002:0/64] 2024-01-19 11:07:08,088 (trainer:753) INFO: 12epoch:train:7201-7300batch: iter_time=8.915e-05, forward_time=0.142, loss_ctc=81.920, loss_interctc_layer6=84.510, loss_interctc_layer12=70.538, loss_interctc_layer15=64.813, loss_interctc_layer21=83.984, loss=77.153, backward_time=0.405, grad_norm=61.179, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.180e-04, train_time=1.995
+[gpua002:0/64] 2024-01-19 11:10:39,281 (trainer:753) INFO: 12epoch:train:7301-7400batch: iter_time=9.286e-05, forward_time=0.154, loss_ctc=95.783, loss_interctc_layer6=100.083, loss_interctc_layer12=84.523, loss_interctc_layer15=78.092, loss_interctc_layer21=98.225, loss=91.341, backward_time=0.403, grad_norm=77.695, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.180e-04, train_time=2.112
+[gpua002:0/64] 2024-01-19 11:13:40,180 (trainer:753) INFO: 12epoch:train:7401-7500batch: iter_time=0.002, forward_time=0.241, loss_ctc=95.744, loss_interctc_layer6=92.279, loss_interctc_layer12=77.459, loss_interctc_layer15=71.460, loss_interctc_layer21=98.189, loss=87.026, backward_time=0.425, grad_norm=62.290, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.147, optim0_lr0=1.180e-04, train_time=1.808
+[gpua002:0/64] 2024-01-19 11:14:00,241 (multiple_iter_factory:32) INFO: Building 6th iter-factory...
+[gpua002:0/64] 2024-01-19 11:14:19,789 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 11:14:23,682 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.7", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.7", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.7", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.7", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe736e74cd0>)
+[gpua002:0/64] 2024-01-19 11:14:23,682 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.7, 
+[gpua002:0/64] 2024-01-19 11:14:23,685 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 11:23:04,064 (trainer:753) INFO: 12epoch:train:7501-7600batch: iter_time=4.079, forward_time=0.142, loss_ctc=85.835, loss_interctc_layer6=86.908, loss_interctc_layer12=72.995, loss_interctc_layer15=67.545, loss_interctc_layer21=87.784, loss=80.213, backward_time=0.307, grad_norm=71.658, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.179e-04, train_time=5.639
+[gpua002:0/64] 2024-01-19 11:25:11,706 (trainer:753) INFO: 12epoch:train:7601-7700batch: iter_time=8.058e-05, forward_time=0.142, loss_ctc=82.656, loss_interctc_layer6=92.416, loss_interctc_layer12=77.940, loss_interctc_layer15=71.841, loss_interctc_layer21=84.821, loss=81.935, backward_time=0.297, grad_norm=62.996, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.179e-04, train_time=1.276
+[gpua002:0/64] 2024-01-19 11:28:01,091 (trainer:753) INFO: 12epoch:train:7701-7800batch: iter_time=8.357e-05, forward_time=0.142, loss_ctc=85.560, loss_interctc_layer6=90.509, loss_interctc_layer12=75.747, loss_interctc_layer15=69.660, loss_interctc_layer21=87.677, loss=81.830, backward_time=0.344, grad_norm=55.806, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.179e-04, train_time=1.691
+[gpua002:0/64] 2024-01-19 11:30:46,732 (trainer:753) INFO: 12epoch:train:7801-7900batch: iter_time=8.457e-05, forward_time=0.143, loss_ctc=78.421, loss_interctc_layer6=93.326, loss_interctc_layer12=78.699, loss_interctc_layer15=73.276, loss_interctc_layer21=80.266, loss=80.798, backward_time=0.346, grad_norm=71.653, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.178e-04, train_time=1.659
+[gpua002:0/64] 2024-01-19 11:33:30,069 (trainer:753) INFO: 12epoch:train:7901-8000batch: iter_time=9.275e-05, forward_time=0.143, loss_ctc=88.411, loss_interctc_layer6=91.583, loss_interctc_layer12=77.113, loss_interctc_layer15=71.319, loss_interctc_layer21=90.907, loss=83.866, backward_time=0.370, grad_norm=66.454, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.178e-04, train_time=1.633
+[gpua002:0/64] 2024-01-19 11:36:42,226 (trainer:753) INFO: 12epoch:train:8001-8100batch: iter_time=8.456e-05, forward_time=0.150, loss_ctc=80.713, loss_interctc_layer6=91.592, loss_interctc_layer12=77.288, loss_interctc_layer15=71.234, loss_interctc_layer21=82.432, loss=80.652, backward_time=0.444, grad_norm=178.593, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.178e-04, train_time=1.921
+[gpua002:0/64] 2024-01-19 11:40:32,657 (trainer:753) INFO: 12epoch:train:8101-8200batch: iter_time=8.831e-05, forward_time=0.294, loss_ctc=87.206, loss_interctc_layer6=92.942, loss_interctc_layer12=78.529, loss_interctc_layer15=72.492, loss_interctc_layer21=89.331, loss=84.100, backward_time=0.459, grad_norm=61.366, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.146, optim0_lr0=1.177e-04, train_time=2.304
+[gpua002:0/64] 2024-01-19 11:43:33,142 (trainer:753) INFO: 12epoch:train:8201-8300batch: iter_time=8.757e-05, forward_time=0.142, loss_ctc=70.825, loss_interctc_layer6=78.709, loss_interctc_layer12=65.906, loss_interctc_layer15=60.651, loss_interctc_layer21=72.479, loss=69.714, backward_time=0.358, grad_norm=50.704, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=1.177e-04, train_time=1.805
+[gpua002:0/64] 2024-01-19 11:46:07,082 (trainer:753) INFO: 12epoch:train:8301-8400batch: iter_time=9.457e-05, forward_time=0.142, loss_ctc=77.720, loss_interctc_layer6=86.757, loss_interctc_layer12=72.385, loss_interctc_layer15=66.692, loss_interctc_layer21=79.757, loss=76.662, backward_time=0.376, grad_norm=72.577, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=1.177e-04, train_time=1.539
+[gpua002:0/64] 2024-01-19 11:49:09,553 (trainer:753) INFO: 12epoch:train:8401-8500batch: iter_time=9.558e-05, forward_time=0.142, loss_ctc=91.104, loss_interctc_layer6=98.469, loss_interctc_layer12=82.904, loss_interctc_layer15=76.608, loss_interctc_layer21=93.326, loss=88.482, backward_time=0.394, grad_norm=62.632, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.176e-04, train_time=1.824
+[gpua002:0/64] 2024-01-19 11:51:52,893 (trainer:753) INFO: 12epoch:train:8501-8600batch: iter_time=9.312e-05, forward_time=0.142, loss_ctc=80.780, loss_interctc_layer6=87.441, loss_interctc_layer12=73.876, loss_interctc_layer15=68.325, loss_interctc_layer21=82.789, loss=78.642, backward_time=0.316, grad_norm=60.842, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.176e-04, train_time=1.633
+[gpua002:0/64] 2024-01-19 11:54:34,357 (trainer:753) INFO: 12epoch:train:8601-8700batch: iter_time=8.941e-05, forward_time=0.142, loss_ctc=82.402, loss_interctc_layer6=91.938, loss_interctc_layer12=77.010, loss_interctc_layer15=70.992, loss_interctc_layer21=84.467, loss=81.362, backward_time=0.404, grad_norm=85.518, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.138, optim0_lr0=1.176e-04, train_time=1.614
+[gpua002:0/64] 2024-01-19 11:56:01,548 (multiple_iter_factory:32) INFO: Building 7th iter-factory...
+[gpua002:0/64] 2024-01-19 11:56:21,279 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 11:56:24,832 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.3", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.3", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.3", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.3", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe73fc83220>)
+[gpua002:0/64] 2024-01-19 11:56:24,832 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.3, 
+[gpua002:0/64] 2024-01-19 11:56:24,836 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 12:04:32,142 (trainer:753) INFO: 12epoch:train:8701-8800batch: iter_time=4.550, forward_time=0.192, loss_ctc=91.017, loss_interctc_layer6=89.818, loss_interctc_layer12=75.140, loss_interctc_layer15=68.930, loss_interctc_layer21=93.731, loss=83.727, backward_time=0.323, grad_norm=70.172, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.142, optim0_lr0=1.175e-04, train_time=5.977
+[gpua002:0/64] 2024-01-19 12:06:45,046 (trainer:753) INFO: 12epoch:train:8801-8900batch: iter_time=9.599e-05, forward_time=0.144, loss_ctc=82.332, loss_interctc_layer6=85.303, loss_interctc_layer12=71.986, loss_interctc_layer15=66.611, loss_interctc_layer21=84.494, loss=78.145, backward_time=0.302, grad_norm=55.359, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.175e-04, train_time=1.329
+[gpua002:0/64] 2024-01-19 12:09:00,190 (trainer:753) INFO: 12epoch:train:8901-9000batch: iter_time=8.492e-05, forward_time=0.143, loss_ctc=90.026, loss_interctc_layer6=96.430, loss_interctc_layer12=81.004, loss_interctc_layer15=74.783, loss_interctc_layer21=92.219, loss=86.893, backward_time=0.298, grad_norm=70.732, clip=100.000, loss_scale=5.071e+30, optim_step_time=0.139, optim0_lr0=1.175e-04, train_time=1.351
+[gpua002:0/64] 2024-01-19 12:11:29,878 (trainer:753) INFO: 12epoch:train:9001-9100batch: iter_time=9.385e-05, forward_time=0.143, loss_ctc=84.488, loss_interctc_layer6=97.008, loss_interctc_layer12=82.219, loss_interctc_layer15=77.134, loss_interctc_layer21=86.415, loss=85.453, backward_time=0.336, grad_norm=74.180, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.174e-04, train_time=1.496
+[gpua002:0/64] 2024-01-19 12:14:46,873 (trainer:753) INFO: 12epoch:train:9101-9200batch: iter_time=1.026e-04, forward_time=0.143, loss_ctc=83.332, loss_interctc_layer6=91.922, loss_interctc_layer12=76.838, loss_interctc_layer15=70.868, loss_interctc_layer21=85.475, loss=81.687, backward_time=0.421, grad_norm=54.612, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.174e-04, train_time=1.971
+[gpua002:0/64] 2024-01-19 12:18:18,615 (trainer:753) INFO: 12epoch:train:9201-9300batch: iter_time=9.122e-05, forward_time=0.143, loss_ctc=80.857, loss_interctc_layer6=89.486, loss_interctc_layer12=75.060, loss_interctc_layer15=69.274, loss_interctc_layer21=82.889, loss=79.513, backward_time=0.440, grad_norm=71.931, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.174e-04, train_time=2.117
+[gpua002:0/64] 2024-01-19 12:21:24,323 (trainer:753) INFO: 12epoch:train:9301-9400batch: iter_time=8.730e-05, forward_time=0.142, loss_ctc=84.410, loss_interctc_layer6=91.814, loss_interctc_layer12=77.173, loss_interctc_layer15=71.131, loss_interctc_layer21=86.658, loss=82.237, backward_time=0.336, grad_norm=72.062, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.173e-04, train_time=1.857
+[gpua002:0/64] 2024-01-19 12:25:01,977 (trainer:753) INFO: 12epoch:train:9401-9500batch: iter_time=0.001, forward_time=0.303, loss_ctc=78.208, loss_interctc_layer6=85.136, loss_interctc_layer12=71.858, loss_interctc_layer15=66.539, loss_interctc_layer21=80.060, loss=76.360, backward_time=0.505, grad_norm=65.584, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.161, optim0_lr0=1.173e-04, train_time=2.175
+[gpua002:0/64] 2024-01-19 12:28:25,719 (trainer:753) INFO: 12epoch:train:9501-9600batch: iter_time=9.763e-05, forward_time=0.143, loss_ctc=69.505, loss_interctc_layer6=78.488, loss_interctc_layer12=65.337, loss_interctc_layer15=59.986, loss_interctc_layer21=71.289, loss=68.921, backward_time=0.398, grad_norm=52.888, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.173e-04, train_time=2.039
+[gpua002:0/64] 2024-01-19 12:31:01,005 (trainer:753) INFO: 12epoch:train:9601-9700batch: iter_time=8.452e-05, forward_time=0.142, loss_ctc=86.596, loss_interctc_layer6=96.510, loss_interctc_layer12=81.521, loss_interctc_layer15=75.683, loss_interctc_layer21=88.854, loss=85.833, backward_time=0.331, grad_norm=65.461, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.172e-04, train_time=1.552
+[gpua002:0/64] 2024-01-19 12:33:54,520 (trainer:753) INFO: 12epoch:train:9701-9800batch: iter_time=8.408e-05, forward_time=0.142, loss_ctc=78.061, loss_interctc_layer6=84.259, loss_interctc_layer12=70.200, loss_interctc_layer15=64.601, loss_interctc_layer21=80.040, loss=75.432, backward_time=0.382, grad_norm=61.247, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.172e-04, train_time=1.735
+[gpua002:0/64] 2024-01-19 12:37:17,048 (trainer:753) INFO: 12epoch:train:9801-9900batch: iter_time=8.878e-05, forward_time=0.143, loss_ctc=88.633, loss_interctc_layer6=99.557, loss_interctc_layer12=83.648, loss_interctc_layer15=77.343, loss_interctc_layer21=90.863, loss=88.009, backward_time=0.373, grad_norm=68.609, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.172e-04, train_time=2.025
+[gpua002:0/64] 2024-01-19 12:40:08,008 (trainer:753) INFO: 12epoch:train:9901-10000batch: iter_time=8.334e-05, forward_time=0.142, loss_ctc=90.873, loss_interctc_layer6=91.640, loss_interctc_layer12=76.648, loss_interctc_layer15=70.646, loss_interctc_layer21=93.552, loss=84.672, backward_time=0.337, grad_norm=63.464, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.171e-04, train_time=1.709
+[gpua002:0/64] 2024-01-19 12:40:28,047 (multiple_iter_factory:32) INFO: Building 8th iter-factory...
+[gpua002:0/64] 2024-01-19 12:40:48,093 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 12:40:51,698 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe6ef6279a0>)
+[gpua002:0/64] 2024-01-19 12:40:51,698 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, 
+[gpua002:0/64] 2024-01-19 12:40:51,701 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 12:50:00,256 (trainer:753) INFO: 12epoch:train:10001-10100batch: iter_time=4.097, forward_time=0.258, loss_ctc=84.904, loss_interctc_layer6=86.112, loss_interctc_layer12=72.182, loss_interctc_layer15=66.612, loss_interctc_layer21=87.243, loss=79.411, backward_time=0.313, grad_norm=64.669, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=1.171e-04, train_time=5.920
+[gpua002:0/64] 2024-01-19 12:52:11,298 (trainer:753) INFO: 12epoch:train:10101-10200batch: iter_time=8.120e-05, forward_time=0.142, loss_ctc=81.537, loss_interctc_layer6=92.988, loss_interctc_layer12=78.413, loss_interctc_layer15=72.223, loss_interctc_layer21=83.779, loss=81.788, backward_time=0.295, grad_norm=96.184, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.171e-04, train_time=1.312
+[gpua002:0/64] 2024-01-19 12:54:20,564 (trainer:753) INFO: 12epoch:train:10201-10300batch: iter_time=7.865e-05, forward_time=0.141, loss_ctc=85.579, loss_interctc_layer6=90.977, loss_interctc_layer12=76.222, loss_interctc_layer15=70.376, loss_interctc_layer21=87.712, loss=82.173, backward_time=0.309, grad_norm=62.955, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.170e-04, train_time=1.292
+[gpua002:0/64] 2024-01-19 12:56:51,094 (trainer:753) INFO: 12epoch:train:10301-10400batch: iter_time=9.334e-05, forward_time=0.142, loss_ctc=78.425, loss_interctc_layer6=92.927, loss_interctc_layer12=78.846, loss_interctc_layer15=73.019, loss_interctc_layer21=80.233, loss=80.690, backward_time=0.307, grad_norm=84.032, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.170e-04, train_time=1.505
+[gpua002:0/64] 2024-01-19 13:01:14,215 (trainer:753) INFO: 12epoch:train:10401-10500batch: iter_time=9.575e-05, forward_time=0.142, loss_ctc=86.145, loss_interctc_layer6=90.950, loss_interctc_layer12=76.514, loss_interctc_layer15=70.562, loss_interctc_layer21=88.370, loss=82.508, backward_time=0.475, grad_norm=60.154, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.170e-04, train_time=2.630
+[gpua002:0/64] 2024-01-19 13:05:48,622 (trainer:753) INFO: 12epoch:train:10501-10600batch: iter_time=9.514e-05, forward_time=0.260, loss_ctc=78.878, loss_interctc_layer6=90.353, loss_interctc_layer12=75.946, loss_interctc_layer15=69.903, loss_interctc_layer21=80.696, loss=79.155, backward_time=0.608, grad_norm=55.220, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.159, optim0_lr0=1.169e-04, train_time=2.740
+[gpua002:0/64] 2024-01-19 13:09:24,921 (trainer:753) INFO: 12epoch:train:10601-10700batch: iter_time=9.288e-05, forward_time=0.143, loss_ctc=86.602, loss_interctc_layer6=93.032, loss_interctc_layer12=78.492, loss_interctc_layer15=72.477, loss_interctc_layer21=88.785, loss=83.878, backward_time=0.427, grad_norm=75.880, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.169e-04, train_time=2.167
+[gpua002:0/64] 2024-01-19 13:12:34,242 (trainer:753) INFO: 12epoch:train:10701-10800batch: iter_time=9.700e-05, forward_time=0.142, loss_ctc=71.046, loss_interctc_layer6=78.641, loss_interctc_layer12=65.802, loss_interctc_layer15=60.546, loss_interctc_layer21=72.841, loss=69.775, backward_time=0.383, grad_norm=64.347, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.169e-04, train_time=1.893
+[gpua002:0/64] 2024-01-19 13:15:58,976 (trainer:753) INFO: 12epoch:train:10801-10900batch: iter_time=0.001, forward_time=0.314, loss_ctc=78.072, loss_interctc_layer6=85.751, loss_interctc_layer12=71.788, loss_interctc_layer15=66.071, loss_interctc_layer21=80.056, loss=76.347, backward_time=0.480, grad_norm=55.331, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.148, optim0_lr0=1.168e-04, train_time=2.047
+[gpua002:0/64] 2024-01-19 13:19:28,691 (trainer:753) INFO: 12epoch:train:10901-11000batch: iter_time=8.546e-05, forward_time=0.142, loss_ctc=92.154, loss_interctc_layer6=98.822, loss_interctc_layer12=82.872, loss_interctc_layer15=76.659, loss_interctc_layer21=94.585, loss=89.018, backward_time=0.454, grad_norm=59.407, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.168e-04, train_time=2.097
+[gpua002:0/64] 2024-01-19 13:22:42,654 (trainer:753) INFO: 12epoch:train:11001-11100batch: iter_time=8.059e-05, forward_time=0.141, loss_ctc=79.856, loss_interctc_layer6=86.824, loss_interctc_layer12=73.030, loss_interctc_layer15=67.441, loss_interctc_layer21=81.602, loss=77.751, backward_time=0.345, grad_norm=92.477, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.168e-04, train_time=1.939
+[gpua002:0/64] 2024-01-19 13:27:18,619 (trainer:753) INFO: 12epoch:train:11101-11200batch: iter_time=8.247e-05, forward_time=0.142, loss_ctc=82.037, loss_interctc_layer6=92.311, loss_interctc_layer12=77.219, loss_interctc_layer15=71.219, loss_interctc_layer21=84.095, loss=81.376, backward_time=0.471, grad_norm=65.113, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.167e-04, train_time=2.759
+[gpua002:0/64] 2024-01-19 13:29:05,058 (multiple_iter_factory:32) INFO: Building 9th iter-factory...
+[gpua002:0/64] 2024-01-19 13:29:24,715 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 13:29:28,373 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.11", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.11", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.11", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.11", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe759ffd0c0>)
+[gpua002:0/64] 2024-01-19 13:29:28,373 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.11, 
+[gpua002:0/64] 2024-01-19 13:29:28,391 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 13:37:20,730 (trainer:753) INFO: 12epoch:train:11201-11300batch: iter_time=4.304, forward_time=0.170, loss_ctc=89.054, loss_interctc_layer6=89.134, loss_interctc_layer12=74.370, loss_interctc_layer15=68.354, loss_interctc_layer21=91.820, loss=82.546, backward_time=0.307, grad_norm=53.910, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.167e-04, train_time=6.021
+[gpua002:0/64] 2024-01-19 13:39:28,614 (trainer:753) INFO: 12epoch:train:11301-11400batch: iter_time=8.179e-05, forward_time=0.143, loss_ctc=82.091, loss_interctc_layer6=85.230, loss_interctc_layer12=72.211, loss_interctc_layer15=66.892, loss_interctc_layer21=84.407, loss=78.166, backward_time=0.303, grad_norm=58.936, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.167e-04, train_time=1.279
+[gpua002:0/64] 2024-01-19 13:42:36,204 (trainer:753) INFO: 12epoch:train:11401-11500batch: iter_time=8.582e-05, forward_time=0.143, loss_ctc=89.639, loss_interctc_layer6=96.586, loss_interctc_layer12=81.019, loss_interctc_layer15=74.852, loss_interctc_layer21=92.081, loss=86.836, backward_time=0.415, grad_norm=61.890, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.166e-04, train_time=1.876
+[gpua002:0/64] 2024-01-19 13:45:27,756 (trainer:753) INFO: 12epoch:train:11501-11600batch: iter_time=9.163e-05, forward_time=0.145, loss_ctc=84.731, loss_interctc_layer6=97.091, loss_interctc_layer12=82.053, loss_interctc_layer15=76.195, loss_interctc_layer21=86.243, loss=85.263, backward_time=0.355, grad_norm=107.700, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.166e-04, train_time=1.715
+[gpua002:0/64] 2024-01-19 13:48:07,306 (trainer:753) INFO: 12epoch:train:11601-11700batch: iter_time=9.035e-05, forward_time=0.147, loss_ctc=82.325, loss_interctc_layer6=91.130, loss_interctc_layer12=76.280, loss_interctc_layer15=70.074, loss_interctc_layer21=84.498, loss=80.861, backward_time=0.379, grad_norm=65.090, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.166e-04, train_time=1.595
+[gpua002:0/64] 2024-01-19 13:52:16,296 (trainer:753) INFO: 12epoch:train:11701-11800batch: iter_time=0.016, forward_time=0.252, loss_ctc=80.426, loss_interctc_layer6=89.394, loss_interctc_layer12=75.212, loss_interctc_layer15=69.146, loss_interctc_layer21=82.751, loss=79.386, backward_time=0.445, grad_norm=70.275, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=1.165e-04, train_time=2.488
+[gpua002:0/64] 2024-01-19 13:54:46,823 (trainer:753) INFO: 12epoch:train:11801-11900batch: iter_time=9.365e-05, forward_time=0.143, loss_ctc=83.645, loss_interctc_layer6=90.720, loss_interctc_layer12=76.314, loss_interctc_layer15=70.269, loss_interctc_layer21=85.611, loss=81.312, backward_time=0.304, grad_norm=52.993, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.165e-04, train_time=1.507
+[gpua002:0/64] 2024-01-19 13:58:51,955 (trainer:753) INFO: 12epoch:train:11901-12000batch: iter_time=9.205e-05, forward_time=0.142, loss_ctc=78.172, loss_interctc_layer6=85.241, loss_interctc_layer12=71.888, loss_interctc_layer15=66.461, loss_interctc_layer21=80.167, loss=76.386, backward_time=0.416, grad_norm=63.802, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.165e-04, train_time=2.451
+[gpua002:0/64] 2024-01-19 14:01:43,917 (trainer:753) INFO: 12epoch:train:12001-12100batch: iter_time=9.145e-05, forward_time=0.142, loss_ctc=69.142, loss_interctc_layer6=77.983, loss_interctc_layer12=64.882, loss_interctc_layer15=59.693, loss_interctc_layer21=70.896, loss=68.519, backward_time=0.375, grad_norm=56.021, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.164e-04, train_time=1.719
+[gpua002:0/64] 2024-01-19 14:05:07,275 (trainer:753) INFO: 12epoch:train:12101-12200batch: iter_time=9.679e-05, forward_time=0.178, loss_ctc=85.970, loss_interctc_layer6=95.943, loss_interctc_layer12=80.890, loss_interctc_layer15=75.131, loss_interctc_layer21=88.087, loss=85.204, backward_time=0.401, grad_norm=66.464, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.145, optim0_lr0=1.164e-04, train_time=2.033
+[gpua002:0/64] 2024-01-19 14:09:18,287 (trainer:753) INFO: 12epoch:train:12201-12300batch: iter_time=8.269e-05, forward_time=0.141, loss_ctc=77.270, loss_interctc_layer6=83.796, loss_interctc_layer12=69.736, loss_interctc_layer15=64.065, loss_interctc_layer21=79.415, loss=74.856, backward_time=0.546, grad_norm=50.239, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.164e-04, train_time=2.510
+[gpua002:0/64] 2024-01-19 14:13:14,744 (trainer:753) INFO: 12epoch:train:12301-12400batch: iter_time=9.222e-05, forward_time=0.143, loss_ctc=86.827, loss_interctc_layer6=97.792, loss_interctc_layer12=82.337, loss_interctc_layer15=76.156, loss_interctc_layer21=89.084, loss=86.439, backward_time=0.399, grad_norm=60.375, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.163e-04, train_time=2.364
+[gpua002:0/64] 2024-01-19 14:16:39,540 (trainer:684) WARNING: The grad norm is nan. Skipping updating the model.
+[gpua002:0/64] 2024-01-19 14:16:43,347 (trainer:753) INFO: 12epoch:train:12401-12500batch: iter_time=1.037e-04, forward_time=0.143, loss_ctc=89.510, loss_interctc_layer6=90.651, loss_interctc_layer12=75.928, loss_interctc_layer15=69.981, loss_interctc_layer21=92.100, loss=83.634, backward_time=0.495, grad_norm=55.416, clip=100.000, loss_scale=1.998e+31, optim_step_time=0.138, optim0_lr0=1.163e-04, train_time=2.086
+[gpua002:0/64] 2024-01-19 14:17:03,382 (multiple_iter_factory:32) INFO: Building 10th iter-factory...
+[gpua002:0/64] 2024-01-19 14:17:22,514 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 14:17:26,004 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.2", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.2", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.2", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.2", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe71b74a710>)
+[gpua002:0/64] 2024-01-19 14:17:26,004 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.2, 
+[gpua002:0/64] 2024-01-19 14:17:26,133 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 14:27:58,925 (trainer:753) INFO: 12epoch:train:12501-12600batch: iter_time=5.330, forward_time=0.202, loss_ctc=88.700, loss_interctc_layer6=86.505, loss_interctc_layer12=72.751, loss_interctc_layer15=66.869, loss_interctc_layer21=91.107, loss=81.186, backward_time=0.306, grad_norm=63.134, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.163e-04, train_time=6.755
+[gpua002:0/64] 2024-01-19 14:30:36,742 (trainer:753) INFO: 12epoch:train:12601-12700batch: iter_time=8.350e-05, forward_time=0.142, loss_ctc=85.957, loss_interctc_layer6=91.777, loss_interctc_layer12=77.391, loss_interctc_layer15=71.254, loss_interctc_layer21=88.100, loss=82.896, backward_time=0.337, grad_norm=59.398, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.162e-04, train_time=1.578
+[gpua002:0/64] 2024-01-19 14:33:21,907 (trainer:753) INFO: 12epoch:train:12701-12800batch: iter_time=8.508e-05, forward_time=0.142, loss_ctc=88.042, loss_interctc_layer6=90.105, loss_interctc_layer12=75.362, loss_interctc_layer15=69.431, loss_interctc_layer21=90.326, loss=82.653, backward_time=0.369, grad_norm=58.842, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.162e-04, train_time=1.651
+[gpua002:0/64] 2024-01-19 14:36:17,037 (trainer:753) INFO: 12epoch:train:12801-12900batch: iter_time=8.514e-05, forward_time=0.231, loss_ctc=79.642, loss_interctc_layer6=92.274, loss_interctc_layer12=77.795, loss_interctc_layer15=72.211, loss_interctc_layer21=81.594, loss=80.703, backward_time=0.400, grad_norm=67.399, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.140, optim0_lr0=1.162e-04, train_time=1.749
+[gpua002:0/64] 2024-01-19 14:39:59,644 (trainer:753) INFO: 12epoch:train:12901-13000batch: iter_time=9.477e-05, forward_time=0.188, loss_ctc=93.612, loss_interctc_layer6=90.807, loss_interctc_layer12=76.124, loss_interctc_layer15=70.402, loss_interctc_layer21=96.375, loss=85.464, backward_time=0.395, grad_norm=80.148, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.161e-04, train_time=2.228
+[gpua002:0/64] 2024-01-19 14:44:08,106 (trainer:753) INFO: 12epoch:train:13001-13100batch: iter_time=9.998e-05, forward_time=0.142, loss_ctc=81.732, loss_interctc_layer6=89.837, loss_interctc_layer12=75.263, loss_interctc_layer15=69.212, loss_interctc_layer21=83.721, loss=79.953, backward_time=0.597, grad_norm=58.170, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.161e-04, train_time=2.483
+[gpua002:0/64] 2024-01-19 14:47:21,295 (trainer:753) INFO: 12epoch:train:13101-13200batch: iter_time=9.024e-05, forward_time=0.143, loss_ctc=95.559, loss_interctc_layer6=93.215, loss_interctc_layer12=78.641, loss_interctc_layer15=72.688, loss_interctc_layer21=98.152, loss=87.651, backward_time=0.416, grad_norm=80.467, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.161e-04, train_time=1.933
+[gpua002:0/64] 2024-01-19 14:51:42,435 (trainer:753) INFO: 12epoch:train:13201-13300batch: iter_time=9.675e-05, forward_time=0.143, loss_ctc=72.891, loss_interctc_layer6=78.554, loss_interctc_layer12=65.671, loss_interctc_layer15=60.466, loss_interctc_layer21=74.714, loss=70.459, backward_time=0.614, grad_norm=54.089, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.160e-04, train_time=2.611
+[gpua002:0/64] 2024-01-19 14:55:45,444 (trainer:753) INFO: 12epoch:train:13301-13400batch: iter_time=9.838e-05, forward_time=0.142, loss_ctc=81.490, loss_interctc_layer6=85.593, loss_interctc_layer12=71.639, loss_interctc_layer15=65.985, loss_interctc_layer21=83.600, loss=77.661, backward_time=0.436, grad_norm=72.115, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.160e-04, train_time=2.430
+[gpua002:0/64] 2024-01-19 14:58:35,008 (trainer:753) INFO: 12epoch:train:13401-13500batch: iter_time=8.629e-05, forward_time=0.142, loss_ctc=91.708, loss_interctc_layer6=96.834, loss_interctc_layer12=81.206, loss_interctc_layer15=75.057, loss_interctc_layer21=94.016, loss=87.764, backward_time=0.328, grad_norm=57.299, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.160e-04, train_time=1.695
+[gpua002:0/64] 2024-01-19 15:02:03,262 (trainer:753) INFO: 12epoch:train:13501-13600batch: iter_time=9.268e-05, forward_time=0.234, loss_ctc=86.419, loss_interctc_layer6=86.925, loss_interctc_layer12=73.124, loss_interctc_layer15=67.535, loss_interctc_layer21=88.460, loss=80.493, backward_time=0.441, grad_norm=61.221, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.147, optim0_lr0=1.159e-04, train_time=2.082
+[gpua002:0/64] 2024-01-19 15:04:28,562 (trainer:753) INFO: 12epoch:train:13601-13700batch: iter_time=8.472e-05, forward_time=0.143, loss_ctc=86.280, loss_interctc_layer6=92.041, loss_interctc_layer12=76.926, loss_interctc_layer15=70.876, loss_interctc_layer21=88.433, loss=82.911, backward_time=0.314, grad_norm=71.247, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.159e-04, train_time=1.452
+[gpua002:0/64] 2024-01-19 15:06:27,410 (multiple_iter_factory:32) INFO: Building 11th iter-factory...
+[gpua002:0/64] 2024-01-19 15:06:47,002 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 15:06:50,630 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe774ee2b90>)
+[gpua002:0/64] 2024-01-19 15:06:50,630 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, 
+[gpua002:0/64] 2024-01-19 15:06:50,634 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 15:17:33,226 (trainer:753) INFO: 12epoch:train:13701-13800batch: iter_time=5.496, forward_time=0.194, loss_ctc=94.964, loss_interctc_layer6=88.802, loss_interctc_layer12=73.759, loss_interctc_layer15=67.696, loss_interctc_layer21=98.056, loss=84.655, backward_time=0.437, grad_norm=60.931, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.141, optim0_lr0=1.159e-04, train_time=7.847
+[gpua002:0/64] 2024-01-19 15:19:46,678 (trainer:753) INFO: 12epoch:train:13801-13900batch: iter_time=8.744e-05, forward_time=0.144, loss_ctc=85.045, loss_interctc_layer6=84.351, loss_interctc_layer12=71.378, loss_interctc_layer15=65.926, loss_interctc_layer21=87.451, loss=78.830, backward_time=0.297, grad_norm=67.287, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.158e-04, train_time=1.334
+[gpua002:0/64] 2024-01-19 15:22:45,895 (trainer:753) INFO: 12epoch:train:13901-14000batch: iter_time=8.881e-05, forward_time=0.143, loss_ctc=92.269, loss_interctc_layer6=96.447, loss_interctc_layer12=80.768, loss_interctc_layer15=74.633, loss_interctc_layer21=94.746, loss=87.773, backward_time=0.344, grad_norm=125.853, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.139, optim0_lr0=1.158e-04, train_time=1.792
+[gpua002:0/64] 2024-01-19 15:26:21,781 (trainer:753) INFO: 12epoch:train:14001-14100batch: iter_time=1.001e-04, forward_time=0.142, loss_ctc=87.171, loss_interctc_layer6=97.481, loss_interctc_layer12=81.953, loss_interctc_layer15=78.033, loss_interctc_layer21=88.272, loss=86.582, backward_time=0.359, grad_norm=187.051, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.158e-04, train_time=2.159
+[gpua002:0/64] 2024-01-19 15:30:29,082 (trainer:753) INFO: 12epoch:train:14101-14200batch: iter_time=9.579e-05, forward_time=0.143, loss_ctc=87.497, loss_interctc_layer6=90.432, loss_interctc_layer12=75.589, loss_interctc_layer15=69.588, loss_interctc_layer21=89.801, loss=82.581, backward_time=0.556, grad_norm=67.524, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.157e-04, train_time=2.473
+[gpua002:0/64] 2024-01-19 15:34:56,504 (trainer:753) INFO: 12epoch:train:14201-14300batch: iter_time=9.100e-05, forward_time=0.207, loss_ctc=85.277, loss_interctc_layer6=88.696, loss_interctc_layer12=74.162, loss_interctc_layer15=68.225, loss_interctc_layer21=87.583, loss=80.789, backward_time=0.552, grad_norm=60.235, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.149, optim0_lr0=1.157e-04, train_time=2.674
+[gpua002:0/64] 2024-01-19 15:38:08,181 (trainer:753) INFO: 12epoch:train:14301-14400batch: iter_time=8.907e-05, forward_time=0.142, loss_ctc=85.722, loss_interctc_layer6=90.980, loss_interctc_layer12=76.198, loss_interctc_layer15=70.181, loss_interctc_layer21=88.057, loss=82.228, backward_time=0.325, grad_norm=90.493, clip=100.000, loss_scale=1.014e+31, optim_step_time=0.138, optim0_lr0=1.157e-04, train_time=1.915
+[gpua002:0/64] 2024-01-19 15:41:18,337 (trainer:753) INFO: 12epoch:train:14401-14500batch: iter_time=9.041e-05, forward_time=0.240, loss_ctc=87.168, loss_interctc_layer6=84.467, loss_interctc_layer12=71.193, loss_interctc_layer15=65.759, loss_interctc_layer21=89.539, loss=79.625, backward_time=0.385, grad_norm=66.896, clip=100.000, loss_scale=1.045e+31, optim_step_time=0.149, optim0_lr0=1.156e-04, train_time=1.902
+[gpua002:0/64] 2024-01-19 15:45:08,110 (trainer:753) INFO: 12epoch:train:14501-14600batch: iter_time=9.220e-05, forward_time=0.144, loss_ctc=70.571, loss_interctc_layer6=78.116, loss_interctc_layer12=64.877, loss_interctc_layer15=59.467, loss_interctc_layer21=72.330, loss=69.072, backward_time=0.429, grad_norm=100.868, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.156e-04, train_time=2.298
+[gpua002:0/64] 2024-01-19 15:49:28,406 (trainer:753) INFO: 12epoch:train:14601-14700batch: iter_time=9.735e-05, forward_time=0.143, loss_ctc=89.476, loss_interctc_layer6=95.564, loss_interctc_layer12=80.470, loss_interctc_layer15=74.519, loss_interctc_layer21=91.708, loss=86.347, backward_time=0.523, grad_norm=60.581, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.156e-04, train_time=2.603
+[gpua002:0/64] 2024-01-19 15:52:49,728 (trainer:753) INFO: 12epoch:train:14701-14800batch: iter_time=9.307e-05, forward_time=0.142, loss_ctc=81.798, loss_interctc_layer6=84.163, loss_interctc_layer12=70.122, loss_interctc_layer15=64.348, loss_interctc_layer21=83.875, loss=76.861, backward_time=0.365, grad_norm=100.221, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.156e-04, train_time=2.013
+[gpua002:0/64] 2024-01-19 15:55:34,222 (trainer:753) INFO: 12epoch:train:14801-14900batch: iter_time=9.032e-05, forward_time=0.143, loss_ctc=94.217, loss_interctc_layer6=98.362, loss_interctc_layer12=82.581, loss_interctc_layer15=76.291, loss_interctc_layer21=96.647, loss=89.619, backward_time=0.437, grad_norm=64.788, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.155e-04, train_time=1.645
+[gpua002:0/64] 2024-01-19 15:57:57,223 (trainer:753) INFO: 12epoch:train:14901-15000batch: iter_time=8.946e-05, forward_time=0.143, loss_ctc=94.706, loss_interctc_layer6=91.302, loss_interctc_layer12=76.314, loss_interctc_layer15=70.219, loss_interctc_layer21=97.149, loss=85.938, backward_time=0.312, grad_norm=78.997, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.155e-04, train_time=1.430
+[gpua002:0/64] 2024-01-19 16:33:39,933 (trainer:352) INFO: 12epoch results: [train] iter_time=0.327, forward_time=0.165, loss_ctc=85.771, loss_interctc_layer6=90.880, loss_interctc_layer12=76.481, loss_interctc_layer15=70.672, loss_interctc_layer21=87.917, loss=82.344, backward_time=0.382, grad_norm=75.664, clip=100.000, loss_scale=6.697e+30, optim_step_time=0.140, optim0_lr0=1.180e-04, train_time=2.181, time=9 hours, 5 minutes and 50.57 seconds, total_count=180000, gpu_max_cached_mem_GB=34.396, [valid] loss_ctc=56.826, cer_ctc=0.253, loss_interctc_layer6=59.354, cer_interctc_layer6=0.264, loss_interctc_layer12=46.832, cer_interctc_layer12=0.197, loss_interctc_layer15=42.420, cer_interctc_layer15=0.170, loss_interctc_layer21=58.679, cer_interctc_layer21=0.264, loss=52.822, time=35 minutes and 18.17 seconds, total_count=56052, gpu_max_cached_mem_GB=34.396
+[gpua002:0/64] 2024-01-19 16:34:04,514 (trainer:407) INFO: The best model has been updated: valid.cer_ctc, valid.total_count
+[gpua002:0/64] 2024-01-19 16:34:04,792 (trainer:461) INFO: The model files were removed: exp/s2t_train_s2t_multitask-ctc_ebf27_conv2d8_size1024_raw_bpe50000/7epoch.pth
+[gpua002:0/64] 2024-01-19 16:34:04,853 (trainer:286) INFO: 13/45epoch started. Estimated time to finish: 2 weeks, 12 hours and 24 minutes
+[gpua002:0/64] 2024-01-19 16:34:05,576 (multiple_iter_factory:32) INFO: Building 0th iter-factory...
+[gpua002:0/64] 2024-01-19 16:34:23,893 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 16:34:27,486 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.0", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.0", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.0", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.0", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe71257f490>)
+[gpua002:0/64] 2024-01-19 16:34:27,486 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.0, 
+[gpua002:0/64] 2024-01-19 16:34:27,489 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 16:43:01,293 (trainer:753) INFO: 13epoch:train:1-100batch: iter_time=3.974, forward_time=0.193, loss_ctc=103.538, loss_interctc_layer6=100.965, loss_interctc_layer12=86.221, loss_interctc_layer15=80.534, loss_interctc_layer21=105.665, loss=95.384, backward_time=0.303, grad_norm=74.865, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=1.155e-04, train_time=5.355
+[gpua002:0/64] 2024-01-19 16:45:09,791 (trainer:753) INFO: 13epoch:train:101-200batch: iter_time=9.890e-05, forward_time=0.143, loss_ctc=110.546, loss_interctc_layer6=101.407, loss_interctc_layer12=85.481, loss_interctc_layer15=79.262, loss_interctc_layer21=113.288, loss=97.997, backward_time=0.296, grad_norm=69.664, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=1.154e-04, train_time=1.287
+[gpua002:0/64] 2024-01-19 16:47:37,586 (trainer:753) INFO: 13epoch:train:201-300batch: iter_time=1.063e-04, forward_time=0.144, loss_ctc=103.839, loss_interctc_layer6=99.200, loss_interctc_layer12=85.138, loss_interctc_layer15=79.510, loss_interctc_layer21=106.285, loss=94.795, backward_time=0.327, grad_norm=101.199, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.154e-04, train_time=1.478
+[gpua002:0/64] 2024-01-19 16:50:21,512 (trainer:753) INFO: 13epoch:train:301-400batch: iter_time=1.005e-04, forward_time=0.223, loss_ctc=100.351, loss_interctc_layer6=105.816, loss_interctc_layer12=89.344, loss_interctc_layer15=82.773, loss_interctc_layer21=102.818, loss=96.221, backward_time=0.363, grad_norm=71.417, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.144, optim0_lr0=1.154e-04, train_time=1.639
+[gpua002:0/64] 2024-01-19 16:53:16,574 (trainer:753) INFO: 13epoch:train:401-500batch: iter_time=1.008e-04, forward_time=0.193, loss_ctc=96.144, loss_interctc_layer6=89.974, loss_interctc_layer12=75.801, loss_interctc_layer15=70.072, loss_interctc_layer21=98.862, loss=86.171, backward_time=0.380, grad_norm=78.318, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=1.153e-04, train_time=1.749
+[gpua002:0/64] 2024-01-19 16:56:28,995 (trainer:753) INFO: 13epoch:train:501-600batch: iter_time=1.150e-04, forward_time=0.143, loss_ctc=95.144, loss_interctc_layer6=92.157, loss_interctc_layer12=78.129, loss_interctc_layer15=72.622, loss_interctc_layer21=97.657, loss=87.142, backward_time=0.402, grad_norm=74.562, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.153e-04, train_time=1.925
+[gpua002:0/64] 2024-01-19 16:59:45,351 (trainer:753) INFO: 13epoch:train:601-700batch: iter_time=1.241e-04, forward_time=0.149, loss_ctc=89.914, loss_interctc_layer6=94.207, loss_interctc_layer12=79.656, loss_interctc_layer15=73.989, loss_interctc_layer21=91.998, loss=85.953, backward_time=0.355, grad_norm=76.699, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.153e-04, train_time=1.963
+[gpua002:0/64] 2024-01-19 17:02:40,792 (trainer:753) INFO: 13epoch:train:701-800batch: iter_time=4.130e-04, forward_time=0.243, loss_ctc=96.333, loss_interctc_layer6=98.418, loss_interctc_layer12=83.217, loss_interctc_layer15=77.489, loss_interctc_layer21=98.599, loss=90.811, backward_time=0.386, grad_norm=71.160, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=1.152e-04, train_time=1.749
+[gpua002:0/64] 2024-01-19 17:05:05,691 (trainer:753) INFO: 13epoch:train:801-900batch: iter_time=1.080e-04, forward_time=0.166, loss_ctc=100.922, loss_interctc_layer6=98.006, loss_interctc_layer12=82.977, loss_interctc_layer15=77.120, loss_interctc_layer21=103.061, loss=92.417, backward_time=0.311, grad_norm=59.783, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=1.152e-04, train_time=1.454
+[gpua002:0/64] 2024-01-19 17:07:59,918 (trainer:753) INFO: 13epoch:train:901-1000batch: iter_time=1.175e-04, forward_time=0.199, loss_ctc=103.041, loss_interctc_layer6=106.594, loss_interctc_layer12=91.567, loss_interctc_layer15=85.572, loss_interctc_layer21=106.208, loss=98.596, backward_time=0.390, grad_norm=86.146, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.152e-04, train_time=1.741
+[gpua002:0/64] 2024-01-19 17:10:51,202 (trainer:753) INFO: 13epoch:train:1001-1100batch: iter_time=8.678e-05, forward_time=0.142, loss_ctc=90.561, loss_interctc_layer6=91.800, loss_interctc_layer12=77.026, loss_interctc_layer15=70.950, loss_interctc_layer21=93.204, loss=84.708, backward_time=0.396, grad_norm=77.226, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.151e-04, train_time=1.713
+[gpua002:0/64] 2024-01-19 17:14:34,481 (trainer:753) INFO: 13epoch:train:1101-1200batch: iter_time=9.817e-05, forward_time=0.145, loss_ctc=97.005, loss_interctc_layer6=92.000, loss_interctc_layer12=78.735, loss_interctc_layer15=73.575, loss_interctc_layer21=99.425, loss=88.148, backward_time=0.439, grad_norm=64.423, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.151e-04, train_time=2.233
+[gpua002:0/64] 2024-01-19 17:16:42,078 (multiple_iter_factory:32) INFO: Building 1th iter-factory...
+[gpua002:0/64] 2024-01-19 17:17:01,661 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 17:17:05,247 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.9", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.9", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.9", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.9", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe718d64040>)
+[gpua002:0/64] 2024-01-19 17:17:05,247 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.9, 
+[gpua002:0/64] 2024-01-19 17:17:05,251 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 17:23:54,114 (trainer:753) INFO: 13epoch:train:1201-1300batch: iter_time=3.387, forward_time=0.177, loss_ctc=101.080, loss_interctc_layer6=98.769, loss_interctc_layer12=83.255, loss_interctc_layer15=77.180, loss_interctc_layer21=103.593, loss=92.775, backward_time=0.376, grad_norm=87.949, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=1.151e-04, train_time=5.596
+[gpua002:0/64] 2024-01-19 17:26:53,163 (trainer:753) INFO: 13epoch:train:1301-1400batch: iter_time=8.806e-05, forward_time=0.192, loss_ctc=97.788, loss_interctc_layer6=99.368, loss_interctc_layer12=83.817, loss_interctc_layer15=77.726, loss_interctc_layer21=100.426, loss=91.825, backward_time=0.337, grad_norm=57.596, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=1.150e-04, train_time=1.790
+[gpua002:0/64] 2024-01-19 17:29:10,702 (trainer:753) INFO: 13epoch:train:1401-1500batch: iter_time=8.293e-05, forward_time=0.158, loss_ctc=99.102, loss_interctc_layer6=95.754, loss_interctc_layer12=80.676, loss_interctc_layer15=74.773, loss_interctc_layer21=101.581, loss=90.377, backward_time=0.307, grad_norm=103.049, clip=100.000, loss_scale=2.089e+31, optim_step_time=0.141, optim0_lr0=1.150e-04, train_time=1.374
+[gpua002:0/64] 2024-01-19 17:31:37,141 (trainer:753) INFO: 13epoch:train:1501-1600batch: iter_time=7.970e-05, forward_time=0.176, loss_ctc=98.712, loss_interctc_layer6=109.508, loss_interctc_layer12=93.432, loss_interctc_layer15=87.310, loss_interctc_layer21=101.212, loss=98.035, backward_time=0.314, grad_norm=72.858, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.147, optim0_lr0=1.150e-04, train_time=1.465
+[gpua002:0/64] 2024-01-19 17:34:41,268 (trainer:753) INFO: 13epoch:train:1601-1700batch: iter_time=8.745e-05, forward_time=0.157, loss_ctc=88.748, loss_interctc_layer6=92.106, loss_interctc_layer12=77.187, loss_interctc_layer15=71.306, loss_interctc_layer21=91.178, loss=84.105, backward_time=0.333, grad_norm=59.980, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=1.149e-04, train_time=1.840
+[gpua002:0/64] 2024-01-19 17:37:27,523 (trainer:753) INFO: 13epoch:train:1701-1800batch: iter_time=8.182e-05, forward_time=0.144, loss_ctc=82.215, loss_interctc_layer6=86.849, loss_interctc_layer12=72.833, loss_interctc_layer15=67.314, loss_interctc_layer21=84.151, loss=78.673, backward_time=0.354, grad_norm=63.653, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=1.149e-04, train_time=1.663
+[gpua002:0/64] 2024-01-19 17:39:49,781 (trainer:753) INFO: 13epoch:train:1801-1900batch: iter_time=1.979e-04, forward_time=0.143, loss_ctc=92.535, loss_interctc_layer6=93.975, loss_interctc_layer12=79.604, loss_interctc_layer15=73.781, loss_interctc_layer21=95.277, loss=87.035, backward_time=0.315, grad_norm=93.789, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.137, optim0_lr0=1.149e-04, train_time=1.422
+[gpua002:0/64] 2024-01-19 17:42:59,248 (trainer:753) INFO: 13epoch:train:1901-2000batch: iter_time=8.504e-05, forward_time=0.164, loss_ctc=91.792, loss_interctc_layer6=99.993, loss_interctc_layer12=84.589, loss_interctc_layer15=78.844, loss_interctc_layer21=94.053, loss=89.854, backward_time=0.423, grad_norm=74.701, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.140, optim0_lr0=1.148e-04, train_time=1.894
+[gpua002:0/64] 2024-01-19 17:45:44,672 (trainer:753) INFO: 13epoch:train:2001-2100batch: iter_time=8.437e-05, forward_time=0.142, loss_ctc=86.288, loss_interctc_layer6=92.282, loss_interctc_layer12=77.478, loss_interctc_layer15=71.638, loss_interctc_layer21=88.342, loss=83.206, backward_time=0.328, grad_norm=66.552, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=1.148e-04, train_time=1.654
+[gpua002:0/64] 2024-01-19 17:49:36,636 (trainer:753) INFO: 13epoch:train:2101-2200batch: iter_time=9.079e-05, forward_time=0.160, loss_ctc=95.439, loss_interctc_layer6=97.313, loss_interctc_layer12=82.255, loss_interctc_layer15=76.147, loss_interctc_layer21=97.767, loss=89.784, backward_time=0.390, grad_norm=129.979, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=1.148e-04, train_time=2.320
+[gpua002:0/64] 2024-01-19 17:53:29,308 (trainer:753) INFO: 13epoch:train:2201-2300batch: iter_time=5.578e-04, forward_time=0.175, loss_ctc=108.387, loss_interctc_layer6=110.987, loss_interctc_layer12=93.460, loss_interctc_layer15=86.615, loss_interctc_layer21=111.399, loss=102.170, backward_time=0.440, grad_norm=79.209, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=1.148e-04, train_time=2.326
+[gpua002:0/64] 2024-01-19 17:56:06,740 (trainer:753) INFO: 13epoch:train:2301-2400batch: iter_time=9.125e-05, forward_time=0.143, loss_ctc=80.614, loss_interctc_layer6=86.921, loss_interctc_layer12=73.567, loss_interctc_layer15=68.413, loss_interctc_layer21=82.706, loss=78.444, backward_time=0.309, grad_norm=127.109, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.138, optim0_lr0=1.147e-04, train_time=1.573
+[gpua002:0/64] 2024-01-19 17:58:39,051 (trainer:753) INFO: 13epoch:train:2401-2500batch: iter_time=0.001, forward_time=0.168, loss_ctc=88.826, loss_interctc_layer6=92.070, loss_interctc_layer12=77.739, loss_interctc_layer15=71.594, loss_interctc_layer21=90.671, loss=84.180, backward_time=0.320, grad_norm=66.767, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.142, optim0_lr0=1.147e-04, train_time=1.524
+[gpua002:0/64] 2024-01-19 17:58:59,081 (multiple_iter_factory:32) INFO: Building 2th iter-factory...
+[gpua002:0/64] 2024-01-19 17:59:17,931 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 17:59:21,417 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.10", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.10", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.10", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.10", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea383d43a0>)
+[gpua002:0/64] 2024-01-19 17:59:21,417 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.10, 
+[gpua002:0/64] 2024-01-19 17:59:21,421 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 18:06:09,461 (trainer:684) WARNING: The grad norm is nan. Skipping updating the model.
+[gpua002:0/64] 2024-01-19 18:07:43,057 (trainer:753) INFO: 13epoch:train:2501-2600batch: iter_time=3.996, forward_time=0.179, loss_ctc=101.602, loss_interctc_layer6=98.853, loss_interctc_layer12=83.687, loss_interctc_layer15=77.750, loss_interctc_layer21=104.463, loss=93.271, backward_time=0.304, grad_norm=67.405, clip=100.000, loss_scale=2.561e+31, optim_step_time=0.140, optim0_lr0=1.147e-04, train_time=5.439
+[gpua002:0/64] 2024-01-19 18:10:02,270 (trainer:753) INFO: 13epoch:train:2601-2700batch: iter_time=8.311e-05, forward_time=0.146, loss_ctc=108.601, loss_interctc_layer6=100.109, loss_interctc_layer12=83.935, loss_interctc_layer15=77.497, loss_interctc_layer21=111.679, loss=96.364, backward_time=0.307, grad_norm=69.289, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.146e-04, train_time=1.393
+[gpua002:0/64] 2024-01-19 18:12:59,995 (trainer:753) INFO: 13epoch:train:2701-2800batch: iter_time=8.210e-05, forward_time=0.143, loss_ctc=100.254, loss_interctc_layer6=96.982, loss_interctc_layer12=82.247, loss_interctc_layer15=76.594, loss_interctc_layer21=102.995, loss=91.814, backward_time=0.357, grad_norm=64.430, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.146e-04, train_time=1.777
+[gpua002:0/64] 2024-01-19 18:15:52,441 (trainer:753) INFO: 13epoch:train:2801-2900batch: iter_time=8.758e-05, forward_time=0.143, loss_ctc=98.713, loss_interctc_layer6=104.036, loss_interctc_layer12=87.427, loss_interctc_layer15=80.654, loss_interctc_layer21=101.263, loss=94.418, backward_time=0.340, grad_norm=59.266, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.146e-04, train_time=1.724
+[gpua002:0/64] 2024-01-19 18:19:17,021 (trainer:753) INFO: 13epoch:train:2901-3000batch: iter_time=9.740e-05, forward_time=0.143, loss_ctc=94.043, loss_interctc_layer6=89.252, loss_interctc_layer12=74.813, loss_interctc_layer15=68.948, loss_interctc_layer21=96.603, loss=84.732, backward_time=0.465, grad_norm=56.352, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.145e-04, train_time=2.046
+[gpua002:0/64] 2024-01-19 18:22:39,022 (trainer:753) INFO: 13epoch:train:3001-3100batch: iter_time=9.161e-05, forward_time=0.194, loss_ctc=93.636, loss_interctc_layer6=90.435, loss_interctc_layer12=76.094, loss_interctc_layer15=70.522, loss_interctc_layer21=95.993, loss=85.336, backward_time=0.402, grad_norm=84.756, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=1.145e-04, train_time=2.020
+[gpua002:0/64] 2024-01-19 18:26:05,613 (trainer:753) INFO: 13epoch:train:3101-3200batch: iter_time=9.070e-05, forward_time=0.170, loss_ctc=88.206, loss_interctc_layer6=92.475, loss_interctc_layer12=78.052, loss_interctc_layer15=72.285, loss_interctc_layer21=90.391, loss=84.282, backward_time=0.461, grad_norm=80.346, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.145e-04, train_time=2.066
+[gpua002:0/64] 2024-01-19 18:29:07,679 (trainer:753) INFO: 13epoch:train:3201-3300batch: iter_time=9.655e-05, forward_time=0.183, loss_ctc=96.117, loss_interctc_layer6=97.905, loss_interctc_layer12=82.506, loss_interctc_layer15=76.659, loss_interctc_layer21=98.502, loss=90.338, backward_time=0.343, grad_norm=71.388, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.141, optim0_lr0=1.144e-04, train_time=1.819
+[gpua002:0/64] 2024-01-19 18:32:22,922 (trainer:753) INFO: 13epoch:train:3301-3400batch: iter_time=9.431e-05, forward_time=0.178, loss_ctc=99.489, loss_interctc_layer6=97.282, loss_interctc_layer12=81.830, loss_interctc_layer15=75.831, loss_interctc_layer21=101.941, loss=91.274, backward_time=0.411, grad_norm=72.957, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.143, optim0_lr0=1.144e-04, train_time=1.953
+[gpua002:0/64] 2024-01-19 18:36:12,898 (trainer:753) INFO: 13epoch:train:3401-3500batch: iter_time=8.536e-05, forward_time=0.171, loss_ctc=98.461, loss_interctc_layer6=103.590, loss_interctc_layer12=87.589, loss_interctc_layer15=81.661, loss_interctc_layer21=100.979, loss=94.456, backward_time=0.605, grad_norm=81.015, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.140, optim0_lr0=1.144e-04, train_time=2.299
+[gpua002:0/64] 2024-01-19 18:39:39,981 (trainer:753) INFO: 13epoch:train:3501-3600batch: iter_time=8.540e-05, forward_time=0.143, loss_ctc=89.017, loss_interctc_layer6=90.431, loss_interctc_layer12=75.528, loss_interctc_layer15=69.482, loss_interctc_layer21=91.444, loss=83.180, backward_time=0.431, grad_norm=93.127, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.143e-04, train_time=2.072
+[gpua002:0/64] 2024-01-19 18:42:12,594 (trainer:753) INFO: 13epoch:train:3601-3700batch: iter_time=9.266e-05, forward_time=0.142, loss_ctc=93.407, loss_interctc_layer6=89.469, loss_interctc_layer12=75.815, loss_interctc_layer15=70.436, loss_interctc_layer21=95.870, loss=84.999, backward_time=0.305, grad_norm=56.758, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.143e-04, train_time=1.526
+[gpua002:0/64] 2024-01-19 18:44:01,174 (multiple_iter_factory:32) INFO: Building 3th iter-factory...
+[gpua002:0/64] 2024-01-19 18:44:20,577 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 18:44:24,199 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.1", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.1", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.1", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.1", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fe724081990>)
+[gpua002:0/64] 2024-01-19 18:44:24,199 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.1, 
+[gpua002:0/64] 2024-01-19 18:44:24,245 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 18:53:31,101 (trainer:753) INFO: 13epoch:train:3701-3800batch: iter_time=4.998, forward_time=0.145, loss_ctc=99.209, loss_interctc_layer6=97.125, loss_interctc_layer12=81.578, loss_interctc_layer15=75.581, loss_interctc_layer21=101.661, loss=91.031, backward_time=0.342, grad_norm=61.772, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.143e-04, train_time=6.785
+[gpua002:0/64] 2024-01-19 18:56:38,809 (trainer:753) INFO: 13epoch:train:3801-3900batch: iter_time=2.693e-04, forward_time=0.208, loss_ctc=96.948, loss_interctc_layer6=98.376, loss_interctc_layer12=82.621, loss_interctc_layer15=76.337, loss_interctc_layer21=99.434, loss=90.743, backward_time=0.439, grad_norm=65.018, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.142, optim0_lr0=1.143e-04, train_time=1.877
+[gpua002:0/64] 2024-01-19 18:59:19,444 (trainer:753) INFO: 13epoch:train:3901-4000batch: iter_time=8.442e-04, forward_time=0.170, loss_ctc=97.179, loss_interctc_layer6=94.277, loss_interctc_layer12=79.273, loss_interctc_layer15=73.301, loss_interctc_layer21=99.851, loss=88.776, backward_time=0.318, grad_norm=61.853, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.142e-04, train_time=1.606
+[gpua002:0/64] 2024-01-19 19:02:09,625 (trainer:753) INFO: 13epoch:train:4001-4100batch: iter_time=2.656e-04, forward_time=0.144, loss_ctc=97.926, loss_interctc_layer6=109.388, loss_interctc_layer12=93.256, loss_interctc_layer15=86.859, loss_interctc_layer21=100.333, loss=97.552, backward_time=0.362, grad_norm=68.878, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.138, optim0_lr0=1.142e-04, train_time=1.702
+[gpua002:0/64] 2024-01-19 19:06:26,882 (trainer:753) INFO: 13epoch:train:4101-4200batch: iter_time=1.046e-04, forward_time=0.202, loss_ctc=87.460, loss_interctc_layer6=91.111, loss_interctc_layer12=76.184, loss_interctc_layer15=70.142, loss_interctc_layer21=89.939, loss=82.967, backward_time=0.419, grad_norm=62.970, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.159, optim0_lr0=1.142e-04, train_time=2.572
+[gpua002:0/64] 2024-01-19 19:09:46,219 (trainer:753) INFO: 13epoch:train:4201-4300batch: iter_time=0.006, forward_time=0.160, loss_ctc=80.917, loss_interctc_layer6=86.115, loss_interctc_layer12=72.216, loss_interctc_layer15=66.510, loss_interctc_layer21=83.310, loss=77.814, backward_time=0.445, grad_norm=62.555, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.141e-04, train_time=1.993
+[gpua002:0/64] 2024-01-19 19:13:36,408 (trainer:753) INFO: 13epoch:train:4301-4400batch: iter_time=1.093e-04, forward_time=0.189, loss_ctc=90.845, loss_interctc_layer6=92.942, loss_interctc_layer12=78.271, loss_interctc_layer15=72.774, loss_interctc_layer21=93.302, loss=85.627, backward_time=0.400, grad_norm=73.428, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.141e-04, train_time=2.301
+[gpua002:0/64] 2024-01-19 19:16:54,199 (trainer:753) INFO: 13epoch:train:4401-4500batch: iter_time=1.127e-04, forward_time=0.143, loss_ctc=90.198, loss_interctc_layer6=99.591, loss_interctc_layer12=84.159, loss_interctc_layer15=77.789, loss_interctc_layer21=92.697, loss=88.887, backward_time=0.474, grad_norm=71.333, clip=100.000, loss_scale=2.028e+31, optim_step_time=0.139, optim0_lr0=1.141e-04, train_time=1.978
+[gpua002:0/64] 2024-01-19 19:20:17,033 (trainer:753) INFO: 13epoch:train:4501-4600batch: iter_time=9.829e-05, forward_time=0.144, loss_ctc=85.047, loss_interctc_layer6=91.472, loss_interctc_layer12=76.708, loss_interctc_layer15=70.774, loss_interctc_layer21=87.182, loss=82.236, backward_time=0.342, grad_norm=75.625, clip=100.000, loss_scale=3.509e+31, optim_step_time=0.139, optim0_lr0=1.140e-04, train_time=2.028
+[gpua002:0/64] 2024-01-19 19:23:37,111 (trainer:753) INFO: 13epoch:train:4601-4700batch: iter_time=1.013e-04, forward_time=0.143, loss_ctc=93.552, loss_interctc_layer6=95.811, loss_interctc_layer12=80.839, loss_interctc_layer15=74.783, loss_interctc_layer21=96.122, loss=88.221, backward_time=0.378, grad_norm=68.534, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=1.140e-04, train_time=2.001
+[gpua002:0/64] 2024-01-19 19:26:34,052 (trainer:753) INFO: 13epoch:train:4701-4800batch: iter_time=1.010e-04, forward_time=0.143, loss_ctc=104.570, loss_interctc_layer6=106.222, loss_interctc_layer12=89.611, loss_interctc_layer15=83.536, loss_interctc_layer21=107.139, loss=98.216, backward_time=0.404, grad_norm=87.470, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=1.140e-04, train_time=1.769
+[gpua002:0/64] 2024-01-19 19:29:59,779 (trainer:753) INFO: 13epoch:train:4801-4900batch: iter_time=9.972e-04, forward_time=0.203, loss_ctc=79.527, loss_interctc_layer6=86.227, loss_interctc_layer12=72.733, loss_interctc_layer15=67.486, loss_interctc_layer21=81.756, loss=77.546, backward_time=0.385, grad_norm=87.127, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.141, optim0_lr0=1.139e-04, train_time=2.057
+[gpua002:0/64] 2024-01-19 19:33:39,891 (trainer:753) INFO: 13epoch:train:4901-5000batch: iter_time=1.011e-04, forward_time=0.146, loss_ctc=88.043, loss_interctc_layer6=91.186, loss_interctc_layer12=76.659, loss_interctc_layer15=70.810, loss_interctc_layer21=90.158, loss=83.371, backward_time=0.396, grad_norm=70.121, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=1.139e-04, train_time=2.201
+[gpua002:0/64] 2024-01-19 19:33:59,921 (multiple_iter_factory:32) INFO: Building 4th iter-factory...
+[gpua002:0/64] 2024-01-19 19:34:19,552 (s2t:401) INFO: Optional Data Names: ('text_prev', 'text_ctc', 'text_spk2', 'text_spk3', 'text_spk4')
+[gpua002:0/64] 2024-01-19 19:34:23,091 (abs_task:1660) INFO: [train] dataset:
+ESPnetDataset(
+  speech: {"path": "exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8", "type": "kaldi_ark"}
+  text_prev: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.prev/split.8", "type": "text"}
+  text_ctc: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text.ctc/split.8", "type": "text"}
+  text: {"path": "exp/s2t_stats_raw_bpe50000/splits12/text/split.8", "type": "text"}
+  preprocess: <espnet2.train.preprocessor.S2TCTCPreprocessor object at 0x7fea5227d210>)
+[gpua002:0/64] 2024-01-19 19:34:23,091 (abs_task:1661) INFO: [train] Batch sampler: UnsortedBatchSampler(N-batch=19027, batch_size=256, key_file=exp/s2t_stats_raw_bpe50000/splits12/speech_shape/split.8, 
+[gpua002:0/64] 2024-01-19 19:34:23,094 (abs_task:1662) INFO: [train] mini-batch sizes summary: N-batch=19027, mean=256.0, min=256, max=257
+[gpua002:0/64] 2024-01-19 19:44:22,130 (trainer:753) INFO: 13epoch:train:5001-5100batch: iter_time=5.027, forward_time=0.223, loss_ctc=100.544, loss_interctc_layer6=98.269, loss_interctc_layer12=82.932, loss_interctc_layer15=77.201, loss_interctc_layer21=102.981, loss=92.386, backward_time=0.311, grad_norm=59.953, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.143, optim0_lr0=1.139e-04, train_time=6.420
+[gpua002:0/64] 2024-01-19 19:46:59,318 (trainer:753) INFO: 13epoch:train:5101-5200batch: iter_time=8.232e-05, forward_time=0.144, loss_ctc=108.721, loss_interctc_layer6=99.500, loss_interctc_layer12=83.312, loss_interctc_layer15=76.929, loss_interctc_layer21=111.729, loss=96.038, backward_time=0.320, grad_norm=77.594, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=1.139e-04, train_time=1.574
+[gpua002:0/64] 2024-01-19 19:49:42,595 (trainer:753) INFO: 13epoch:train:5201-5300batch: iter_time=8.656e-05, forward_time=0.143, loss_ctc=101.169, loss_interctc_layer6=96.006, loss_interctc_layer12=81.422, loss_interctc_layer15=75.398, loss_interctc_layer21=103.944, loss=91.588, backward_time=0.320, grad_norm=75.048, clip=100.000, loss_scale=4.056e+31, optim_step_time=0.139, optim0_lr0=1.138e-04, train_time=1.633
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:2774854"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:3199906"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:2063258"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:1121004"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:1624521"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:594356"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:834168"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:2372178"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:2421100"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:3620411"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:4003646"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:1251793"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:1696225"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:136082"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:503183"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:2080623"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:213278"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:4444999"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:4901453"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:5371509"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:6182369"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:5732429"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:6671256"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:12424906"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:20854704"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:21946915"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:21360868"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:12842798"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:14723647"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:15834262"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:15241626"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:13289192"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:14207195"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:13740753"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:16304776"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:18659601"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:19214778"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:20087509"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:19628932"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:16834561"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:17237297"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:18206453"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:17728240"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:20425713"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.3/data_wav.ark:2850359"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:7538838"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:7139167"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:11527947"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:9641508"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:10379511"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:7892244"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:8322547"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:9228899"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:8815166"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:26942489"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:27475687"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:22422722"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:23219808"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:24425350"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:22899744"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:24002951"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:24965284"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:25803743"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py:481: UserWarning: An error happens at loading "dump/raw/org/vctk/tr_no_dev_whisper/data/format.2/data_wav.ark:23632009"
+  warnings.warn('An error happens at loading "{}"'.format(ark_name))
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_187_000000625_000002515_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_199_000001660_000003245_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_137_000000590_000002735_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_161_000000090_000000955_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_149_000000910_000003530_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_125_000000855_000002210_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_073_000000750_000001480_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_173_000000775_000003160_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_122_000000765_000002030_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_212_000001335_000002430_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_227_000000955_000002335_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_085_000000860_000002620_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_097_000000845_000003150_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_048_000000875_000002105_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_061_000000980_000002000_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_110_000000720_000001800_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_112_000001190_000002810_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_252_000000620_000002530_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_240_000000585_000002760_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_264_000001370_000002870_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_288_000001165_000002825_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_276_000001215_000002375_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_300_000000610_000002685_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_033_000000480_000003675_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_251_000001185_000004110_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_276_000001450_000004215_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_263_000001415_000005725_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_045_000000975_000004125_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_118_000000780_000002750_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_094_000000690_000003640_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_106_000000880_000004030_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_082_000000925_000002460_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_057_000001110_000003770_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_069_000001085_000002860_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_130_000001070_000003640_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_190_000000850_000002570_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_227_000001230_000002710_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_203_000000850_000004325_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_215_000001270_000004580_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_142_000001690_000003230_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_154_000000550_000002430_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_178_000001030_000003265_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_166_000001350_000003165_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_239_000001095_000003025_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_134_000001015_000003670_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_324_000000980_000002550_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_312_000000655_000003685_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_021_000000785_000009885_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_388_000001105_000002405_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_008_000001340_000008335_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_337_000001370_000002430_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_350_000001660_000004010_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_375_000001270_000002780_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p227_363_000000475_000002585_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_024_000001470_000007255_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_036_000000940_000002435_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_288_000001140_000002840_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_349_000001280_000003990_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_312_000000980_000001815_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_337_000001405_000004555_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_300_000001570_000003030_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_361_000001520_000003630_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p229_012_000001195_000004375_eng_asr
+ERROR:root:Error happened with path=exp/s2t_stats_raw_bpe50000/splits12/wav.scp/split.8, type=kaldi_ark, id=VCTK_p228_325_000001810_000004190_eng_asr
+Process SpawnProcess-4:
+Process SpawnProcess-1:
+Process SpawnProcess-3:
+Process SpawnProcess-1:
+Process SpawnProcess-1:
+Process SpawnProcess-3:
+Process SpawnProcess-3:
+Process SpawnProcess-3:
+Process SpawnProcess-4:
+Process SpawnProcess-2:
+Process SpawnProcess-1:
+Process SpawnProcess-2:
+Process SpawnProcess-1:
+Process SpawnProcess-2:
+Process SpawnProcess-3:
+Process SpawnProcess-4:
+Process SpawnProcess-3:
+Process SpawnProcess-4:
+Process SpawnProcess-2:
+Process SpawnProcess-2:
+Process SpawnProcess-3:
+Process SpawnProcess-4:
+Process SpawnProcess-3:
+Process SpawnProcess-3:
+Process SpawnProcess-1:
+Process SpawnProcess-3:
+Process SpawnProcess-4:
+Process SpawnProcess-4:
+Process SpawnProcess-1:
+Process SpawnProcess-2:
+Process SpawnProcess-3:
+Process SpawnProcess-3:
+Process SpawnProcess-2:
+Process SpawnProcess-3:
+Process SpawnProcess-1:
+Process SpawnProcess-4:
+Process SpawnProcess-4:
+Process SpawnProcess-2:
+Process SpawnProcess-1:
+Process SpawnProcess-3:
+Traceback (most recent call last):
+Traceback (most recent call last):
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+Traceback (most recent call last):
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcd in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+Traceback (most recent call last):
+Traceback (most recent call last):
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf6 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf9 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbd in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+Traceback (most recent call last):
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcd in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf9 in position 0: invalid start byte
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb8 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x83 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa8 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf6 in position 0: invalid start byte
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe6 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbd in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+Traceback (most recent call last):
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe8 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc9 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa0 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb8 in position 0: invalid start byte
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x83 in position 0: invalid start byte
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8b in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xac in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe6 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x84 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcf in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+Traceback (most recent call last):
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa8 in position 0: invalid start byte
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xac in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe8 in position 0: unexpected end of data
+
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc9 in position 0: unexpected end of data
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa0 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd8 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+Traceback (most recent call last):
+Traceback (most recent call last):
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf4 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x84 in position 0: invalid start byte
+
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcf in position 0: unexpected end of data
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x92 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8d in position 0: invalid start byte
+
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd8 in position 0: unexpected end of data
+
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x89 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf6 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x92 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf5 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8d in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa3 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf4 in position 0: unexpected end of data
+
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb4 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x89 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8e in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf6 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf2 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe6 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa3 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd8 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb4 in position 0: invalid start byte
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xce in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf5 in position 0: invalid start byte
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8e in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa3 in position 0: invalid start byte
+
+Traceback (most recent call last):
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf2 in position 0: unexpected end of data
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x99 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe6 in position 0: unexpected end of data
+
+Traceback (most recent call last):
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe2 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa3 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd8 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xce in position 0: unexpected end of data
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb1 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x99 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd8 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb1 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x9d in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb9 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd8 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd3 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x9d in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb9 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd3 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+Traceback (most recent call last):
+Traceback (most recent call last):
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe3 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb2 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xe3 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb2 in position 0: invalid start byte
+
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-2:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+Process SpawnProcess-1:
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb5 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb5 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-4:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x89 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x89 in position 0: invalid start byte
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xdd in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xdd in position 0: unexpected end of data
+
+Process SpawnProcess-1:
+Traceback (most recent call last):
+Process SpawnProcess-4:
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x9a in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x9a in position 0: invalid start byte
+
+Process SpawnProcess-4:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+Traceback (most recent call last):
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x9a in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x9a in position 0: invalid start byte
+
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf6 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf6 in position 0: invalid start byte
+
+Process SpawnProcess-2:
+Process SpawnProcess-2:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa4 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa4 in position 0: invalid start byte
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc5 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xc5 in position 0: unexpected end of data
+
+Process SpawnProcess-3:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x90 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x90 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-3:
+Process SpawnProcess-2:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x97 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x97 in position 0: invalid start byte
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x85 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+Process SpawnProcess-4:
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x85 in position 0: invalid start byte
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x83 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x83 in position 0: invalid start byte
+
+Process SpawnProcess-1:
+Process SpawnProcess-2:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x92 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x92 in position 0: invalid start byte
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbc in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xbc in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-4:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf2 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xf2 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-2:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xdc in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xdc in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-1:
+Process SpawnProcess-1:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x93 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x93 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-4:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa0 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xa0 in position 0: invalid start byte
+
+Process SpawnProcess-2:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8c in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x8c in position 0: invalid start byte
+
+Process SpawnProcess-2:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x96 in position 0: invalid start byte
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0x96 in position 0: invalid start byte
+
+Process SpawnProcess-1:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 543, in reraise
+    raise exception
+ValueError: Caught ValueError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xdd in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 572, in read_ascii_mat
+    char = b.decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xdd in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 574, in read_ascii_mat
+    raise ValueError("File format is wrong?")
+ValueError: File format is wrong?
+
+Process SpawnProcess-1:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcc in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xcc in position 0: unexpected end of data
+
+Process SpawnProcess-4:
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 314, in _bootstrap
+    self.run()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/multiprocessing/process.py", line 108, in run
+    self._target(*self._args, **self._kwargs)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1432, in main_worker
+    cls.trainer.run(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 304, in run
+    all_steps_are_invalid = cls.train_one_epoch(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/trainer.py", line 531, in train_one_epoch
+    for iiter, (utt_id, batch) in enumerate(
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/reporter.py", line 267, in measure_iter_time
+    retval = next(iterator)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/iterators/multiple_iter_factory.py", line 35, in build_iter
+    yield from iter_factory.build_iter(epoch, shuffle)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 628, in __next__
+    data = self._next_data()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1333, in _next_data
+    return self._process_data(data)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1359, in _process_data
+    data.reraise()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/_utils.py", line 542, in reraise
+    raise RuntimeError(msg) from None
+RuntimeError: Caught UnicodeDecodeError in DataLoader worker process 1.
+Original Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 555, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd0 in position 0: unexpected end of data
+
+During handling of the above exception, another exception occurred:
+
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/worker.py", line 302, in _worker_loop
+    data = fetcher.fetch(index)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in fetch
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/utils/data/_utils/fetch.py", line 58, in <listcomp>
+    data = [self.dataset[idx] for idx in possibly_batched_index]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 558, in __getitem__
+    value = loader[uid]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/train/dataset.py", line 54, in __getitem__
+    retval = self.loader[key]
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/utils.py", line 479, in __getitem__
+    return self._loader(ark_name)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 237, in load_mat
+    return _load_mat(fd, offset, slices, endian=endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 330, in _load_mat
+    array = read_kaldi(fd, endian)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 442, in read_kaldi
+    array = read_ascii_mat(fd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/kaldiio/matio.py", line 589, in read_ascii_mat
+    char = fd.read(1).decode(encoding=default_encoding)
+UnicodeDecodeError: 'utf-8' codec can't decode byte 0xd0 in position 0: unexpected end of data
+
+gpua013:395173:395247 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua013:395173:395173 [1] NCCL INFO comm 0x55c396d17fe0 rank 5 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua083:196705:196783 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua017:2815424:2815507 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua083:196705:196705 [2] NCCL INFO comm 0x564924364b60 rank 58 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua017:2815424:2815424 [1] NCCL INFO comm 0x555efacf0500 rank 13 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua079:2406228:2406312 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua079:2406228:2406228 [2] NCCL INFO comm 0x55aebc056f20 rank 50 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua017:2815425:2815504 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua017:2815425:2815425 [2] NCCL INFO comm 0x55bc91f10020 rank 14 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua087:102966:103036 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua087:102966:102966 [2] NCCL INFO comm 0x555c44b6efc0 rank 62 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua017:2815423:2815505 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua016:311570:311689 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua079:2406226:2406313 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua016:311568:311688 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua079:2406229:2406311 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua017:2815423:2815423 [0] NCCL INFO comm 0x558c31a2ef20 rank 12 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua055:2316454:2316525 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua055:2316455:2316526 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua035:260521:260628 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua024:349659:349755 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua024:349661:349757 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua083:196706:196784 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua016:311567:311690 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua053:179585:179694 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua053:179586:179692 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua083:196704:196781 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua035:260522:260629 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua055:2316455:2316455 [2] NCCL INFO comm 0x5568d932cac0 rank 42 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua079:2406226:2406226 [0] NCCL INFO comm 0x55cf75509b10 rank 48 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua035:260521:260521 [2] NCCL INFO comm 0x555bf1b64b20 rank 26 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua024:349661:349661 [3] NCCL INFO comm 0x55e3a5740670 rank 19 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua058:102161:102260 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua087:102964:103037 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua079:2406229:2406229 [3] NCCL INFO comm 0x56029a9c6950 rank 51 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua016:311570:311570 [3] NCCL INFO comm 0x55c00019cee0 rank 11 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua016:311568:311568 [1] NCCL INFO comm 0x55c0bac9dda0 rank 9 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua055:2316454:2316454 [1] NCCL INFO comm 0x55cdce03aee0 rank 41 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua058:102161:102161 [2] NCCL INFO comm 0x55705da34420 rank 46 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua055:2316453:2316527 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua080:1427103:1427176 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua083:196704:196704 [1] NCCL INFO comm 0x55ce729cb670 rank 57 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua024:349659:349659 [1] NCCL INFO comm 0x5602f3ebe4e0 rank 17 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua079:2406227:2406314 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua083:196706:196706 [3] NCCL INFO comm 0x55e11249eda0 rank 59 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua053:179584:179691 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua053:179585:179585 [1] NCCL INFO comm 0x55baf655d6f0 rank 37 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua053:179586:179586 [2] NCCL INFO comm 0x559e082b31e0 rank 38 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua055:2316456:2316528 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua080:1427106:1427177 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua016:311567:311567 [0] NCCL INFO comm 0x55da7ca44890 rank 8 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua058:102159:102262 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua087:102967:103035 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua080:1427106:1427106 [3] NCCL INFO comm 0x555942f648e0 rank 55 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua058:102160:102259 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua055:2316453:2316453 [0] NCCL INFO comm 0x56406b6e9aa0 rank 40 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua053:179587:179693 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua087:102964:102964 [0] NCCL INFO comm 0x5565fd0a0580 rank 60 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua035:260520:260631 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua024:349658:349756 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua035:260522:260522 [3] NCCL INFO comm 0x561a303144d0 rank 27 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua087:102965:103034 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua016:311569:311687 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua058:102162:102261 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua055:2316456:2316456 [3] NCCL INFO comm 0x563e974bd250 rank 43 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua079:2406227:2406227 [1] NCCL INFO comm 0x5643b77bfd50 rank 49 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua058:102160:102160 [1] NCCL INFO comm 0x5608e2f758d0 rank 45 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua024:349660:349758 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua087:102965:102965 [1] NCCL INFO comm 0x55a57cf7b4f0 rank 61 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua080:1427104:1427174 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua080:1427105:1427175 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua024:349660:349660 [2] NCCL INFO comm 0x5581f3f66310 rank 18 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua087:102967:102967 [3] NCCL INFO comm 0x555c31f87ca0 rank 63 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua017:2815426:2815506 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua053:179584:179584 [0] NCCL INFO comm 0x55d72416a360 rank 36 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua016:311569:311569 [2] NCCL INFO comm 0x55726fb33ee0 rank 10 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua013:395174:395249 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua080:1427103:1427103 [0] NCCL INFO comm 0x5603ba03ce60 rank 52 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua058:102159:102159 [0] NCCL INFO comm 0x5642f3bcd790 rank 44 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua035:260520:260520 [1] NCCL INFO comm 0x55ad88e815a0 rank 25 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua053:179587:179587 [3] NCCL INFO comm 0x564395cc1f10 rank 39 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua013:395174:395174 [2] NCCL INFO comm 0x55a365b5aa60 rank 6 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua058:102162:102162 [3] NCCL INFO comm 0x55a590e20350 rank 47 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua024:349658:349658 [0] NCCL INFO comm 0x5637d5b1cf10 rank 16 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua080:1427105:1427105 [2] NCCL INFO comm 0x561931f3d1d0 rank 54 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua080:1427104:1427104 [1] NCCL INFO comm 0x55604e9d43e0 rank 53 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua017:2815426:2815426 [3] NCCL INFO comm 0x55e3633b10e0 rank 15 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua013:395172:395246 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua083:196703:196782 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua013:395172:395172 [0] NCCL INFO comm 0x55f77ab49490 rank 4 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua083:196703:196703 [0] NCCL INFO comm 0x55f08d116e60 rank 56 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua036:985765:985835 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua036:985762:985833 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua036:985763:985834 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua036:985764:985832 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua036:985764:985764 [2] NCCL INFO comm 0x55c4a4c8da20 rank 30 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua036:985762:985762 [0] NCCL INFO comm 0x55c2a75b5130 rank 28 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua036:985765:985765 [3] NCCL INFO comm 0x560c55567fe0 rank 31 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua036:985763:985763 [1] NCCL INFO comm 0x5558a13b1a50 rank 29 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua013:395175:395248 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua013:395175:395175 [3] NCCL INFO comm 0x55debbd55590 rank 7 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua035:260519:260630 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua035:260519:260519 [0] NCCL INFO comm 0x55da44f13b30 rank 24 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua051:387979:388079 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua051:387978:388078 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua051:387977:388081 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua051:387976:388080 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua051:387979:387979 [3] NCCL INFO comm 0x55dcff877f00 rank 35 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua051:387978:387978 [2] NCCL INFO comm 0x55b2b8bf5dc0 rank 34 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua051:387977:387977 [1] NCCL INFO comm 0x561d489334e0 rank 33 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua051:387976:387976 [0] NCCL INFO comm 0x555713d9a190 rank 32 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua034:186776:186874 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua034:186777:186873 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua034:186779:186872 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua034:186778:186875 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua034:186777:186777 [1] NCCL INFO comm 0x55e7d5b957e0 rank 21 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua034:186779:186779 [3] NCCL INFO comm 0x5616b81acd10 rank 23 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua034:186776:186776 [0] NCCL INFO comm 0x562076438ba0 rank 20 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+gpua034:186778:186778 [2] NCCL INFO comm 0x56381768d830 rank 22 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua002:51612:51716 [3] NCCL INFO [Service thread] Connection closed by localRank 3
+gpua002:51611:51713 [2] NCCL INFO [Service thread] Connection closed by localRank 2
+gpua002:51610:51715 [1] NCCL INFO [Service thread] Connection closed by localRank 1
+gpua002:51610:51610 [1] NCCL INFO comm 0x55dc7087fd20 rank 1 nranks 64 cudaDev 1 busId 46000 - Abort COMPLETE
+gpua002:51611:51611 [2] NCCL INFO comm 0x55acdf676190 rank 2 nranks 64 cudaDev 2 busId 85000 - Abort COMPLETE
+gpua002:51612:51612 [3] NCCL INFO comm 0x5594523978c0 rank 3 nranks 64 cudaDev 3 busId c7000 - Abort COMPLETE
+gpua002:51609:51714 [0] NCCL INFO [Service thread] Connection closed by localRank 0
+gpua002:51609:51609 [0] NCCL INFO comm 0x5654a5ed42e0 rank 0 nranks 64 cudaDev 0 busId 7000 - Abort COMPLETE
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    raise ProcessExitedException(
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 2 terminated with exit code 1
+torch.multiprocessing.spawn.ProcessExitedException: process 0 terminated with exit code 1
+    raise ProcessExitedException(
+    raise ProcessExitedException(
+    raise ProcessExitedException(
+    raise ProcessExitedException(
+    raise ProcessExitedException(
+    raise ProcessExitedException(
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 3 terminated with exit code 1
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+torch.multiprocessing.spawn.ProcessExitedException: process 0 terminated with exit code 1
+torch.multiprocessing.spawn.ProcessExitedException: process 0 terminated with exit code 1
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 0 terminated with exit code 1
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 3 terminated with exit code 1
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+Traceback (most recent call last):
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 196, in _run_module_as_main
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    return _run_code(code, main_globals, None,
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/runpy.py", line 86, in _run_code
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    exec(code, run_globals)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 23, in <module>
+    main()
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/bin/s2t_train.py", line 19, in main
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 2 terminated with exit code 1
+    S2TTask.main(cmd=cmd)
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/espnet2/tasks/abs_task.py", line 1169, in main
+    while not ProcessContext(processes, error_queues).join():
+  File "/scratch/bbjs/peng6/espnet-owsm-ctc/tools/miniconda/envs/espnet/lib/python3.10/site-packages/torch/multiprocessing/spawn.py", line 149, in join
+    raise ProcessExitedException(
+torch.multiprocessing.spawn.ProcessExitedException: process 1 terminated with exit code 1
+srun: error: gpua079: task 12: Exited with exit code 1
+srun: error: gpua087: task 15: Exited with exit code 1
+srun: error: gpua035: task 6: Exited with exit code 1
+srun: error: gpua017: task 3: Exited with exit code 1
+srun: error: gpua080: task 13: Exited with exit code 1
+srun: error: gpua053: task 9: Exited with exit code 1
+srun: error: gpua013: task 1: Exited with exit code 1
+srun: error: gpua024: task 4: Exited with exit code 1
+srun: error: gpua051: task 8: Exited with exit code 1
+srun: error: gpua034: task 5: Exited with exit code 1
+srun: error: gpua016: task 2: Exited with exit code 1
+srun: error: gpua002: task 0: Exited with exit code 1
+srun: error: gpua036: task 7: Exited with exit code 1
+srun: error: gpua055: task 10: Exited with exit code 1
+srun: error: gpua083: task 14: Exited with exit code 1
+srun: error: gpua058: task 11: Exited with exit code 1
+# Accounting: begin_time=1705589298
+# Accounting: end_time=1705715475
+# Accounting: time=126177 threads=1
+# Finished at Fri Jan 19 19:51:15 CST 2024 with status 1