diff --git "a/run.log" "b/run.log" --- "a/run.log" +++ "b/run.log" @@ -1,13 +1,13 @@ -[2022-12-20 04:28:45,093] [WARNING] [runner.py:179:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only. -[2022-12-20 04:28:45,104] [INFO] [runner.py:508:main] cmd = /home/milan/hf_env/bin/python3 -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF19 --master_addr=127.0.0.1 --master_port=29560 run_speech_recognition_seq2seq_streaming.py --deepspeed=ds_config.json --model_name_or_path=openai/whisper-large-v2 --dataset_name=mozilla-foundation/common_voice_11_0 --dataset_config_name=cs --language=czech --train_split_name=train+validation --eval_split_name=test --model_index_name=Whisper Large-v2 Czech CV11 v2 --max_steps=5000 --output_dir=./ --per_device_train_batch_size=32 --per_device_eval_batch_size=8 --gradient_accumulation_steps=2 --logging_steps=25 --learning_rate=1e-5 --warmup_steps=500 --evaluation_strategy=steps --eval_steps=1000 --save_strategy=steps --save_steps=1000 --generation_max_length=225 --length_column_name=input_length --max_duration_in_seconds=30 --text_column_name=sentence --freeze_feature_encoder=False --report_to=tensorboard --metric_for_best_model=wer --greater_is_better=False --load_best_model_at_end --gradient_checkpointing --fp16 --overwrite_output_dir --do_train --do_eval --predict_with_generate --do_normalize_eval --streaming=False --use_auth_token --push_to_hub -[2022-12-20 04:28:46,705] [INFO] [launch.py:142:main] WORLD INFO DICT: {'localhost': [0]} -[2022-12-20 04:28:46,706] [INFO] [launch.py:148:main] nnodes=1, num_local_procs=1, node_rank=0 -[2022-12-20 04:28:46,706] [INFO] [launch.py:161:main] global_rank_mapping=defaultdict(, {'localhost': [0]}) -[2022-12-20 04:28:46,706] [INFO] [launch.py:162:main] dist_world_size=1 -[2022-12-20 04:28:46,706] [INFO] [launch.py:164:main] Setting CUDA_VISIBLE_DEVICES=0 -[2022-12-20 04:28:50,920] [INFO] [comm.py:654:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl -12/20/2022 04:28:51 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: True -12/20/2022 04:28:51 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments( +[2022-12-20 10:11:56,294] [WARNING] [runner.py:179:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only. +[2022-12-20 10:11:56,305] [INFO] [runner.py:508:main] cmd = /home/milan/hf_env/bin/python3 -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF19 --master_addr=127.0.0.1 --master_port=29560 run_speech_recognition_seq2seq_streaming.py --deepspeed=ds_config.json --model_name_or_path=openai/whisper-large-v2 --dataset_name=mozilla-foundation/common_voice_11_0 --dataset_config_name=cs --language=czech --train_split_name=train+validation --eval_split_name=test --model_index_name=Whisper Large-v2 Czech CV11 v2 --max_steps=5000 --output_dir=./ --per_device_train_batch_size=32 --per_device_eval_batch_size=8 --gradient_accumulation_steps=2 --logging_steps=25 --learning_rate=1e-5 --warmup_steps=500 --evaluation_strategy=steps --eval_steps=1000 --save_strategy=steps --save_steps=1000 --generation_max_length=225 --length_column_name=input_length --max_duration_in_seconds=30 --text_column_name=sentence --freeze_feature_encoder=False --report_to=tensorboard --metric_for_best_model=wer --greater_is_better=False --load_best_model_at_end --gradient_checkpointing --fp16 --overwrite_output_dir --do_train --do_eval --predict_with_generate --do_normalize_eval --streaming=False --use_auth_token --push_to_hub +[2022-12-20 10:11:57,861] [INFO] [launch.py:142:main] WORLD INFO DICT: {'localhost': [0]} +[2022-12-20 10:11:57,861] [INFO] [launch.py:148:main] nnodes=1, num_local_procs=1, node_rank=0 +[2022-12-20 10:11:57,861] [INFO] [launch.py:161:main] global_rank_mapping=defaultdict(, {'localhost': [0]}) +[2022-12-20 10:11:57,861] [INFO] [launch.py:162:main] dist_world_size=1 +[2022-12-20 10:11:57,861] [INFO] [launch.py:164:main] Setting CUDA_VISIBLE_DEVICES=0 +[2022-12-20 10:12:02,018] [INFO] [comm.py:654:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +12/20/2022 10:12:02 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: True +12/20/2022 10:12:02 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments( _n_gpu=1, adafactor=False, adam_beta1=0.9, @@ -64,7 +64,7 @@ local_rank=0, log_level=passive, log_level_replica=passive, log_on_each_node=True, -logging_dir=./runs/Dec20_04-28-50_129-146-123-136, +logging_dir=./runs/Dec20_10-12-02_129-146-123-136, logging_first_step=False, logging_nan_inf_filter=True, logging_steps=25, @@ -117,7 +117,7 @@ warmup_steps=500, weight_decay=0.0, xpu_backend=None, ) -12/20/2022 04:28:51 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments( +12/20/2022 10:12:02 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments( _n_gpu=1, adafactor=False, adam_beta1=0.9, @@ -174,7 +174,7 @@ local_rank=0, log_level=passive, log_level_replica=passive, log_on_each_node=True, -logging_dir=./runs/Dec20_04-28-50_129-146-123-136, +logging_dir=./runs/Dec20_10-12-02_129-146-123-136, logging_first_step=False, logging_nan_inf_filter=True, logging_steps=25, @@ -227,23 +227,23 @@ warmup_steps=500, weight_decay=0.0, xpu_backend=None, ) -12/20/2022 04:28:53 - INFO - datasets.info - Loading Dataset Infos from /home/milan/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:53 - INFO - datasets.builder - Overwrite dataset info from restored data version. -12/20/2022 04:28:53 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:53 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) -12/20/2022 04:28:53 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:54 - INFO - datasets.info - Loading Dataset Infos from /home/milan/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:54 - INFO - datasets.builder - Overwrite dataset info from restored data version. -12/20/2022 04:28:54 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:54 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) -12/20/2022 04:28:54 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:56 - INFO - datasets.info - Loading Dataset Infos from /home/milan/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:56 - INFO - datasets.builder - Overwrite dataset info from restored data version. -12/20/2022 04:28:56 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -12/20/2022 04:28:56 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) -12/20/2022 04:28:56 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f -[INFO|configuration_utils.py:654] 2022-12-20 04:28:57,066 >> loading configuration file config.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/config.json -[INFO|configuration_utils.py:706] 2022-12-20 04:28:57,070 >> Model config WhisperConfig { +12/20/2022 10:12:04 - INFO - datasets.info - Loading Dataset Infos from /home/milan/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:04 - INFO - datasets.builder - Overwrite dataset info from restored data version. +12/20/2022 10:12:04 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:04 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) +12/20/2022 10:12:04 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:06 - INFO - datasets.info - Loading Dataset Infos from /home/milan/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:06 - INFO - datasets.builder - Overwrite dataset info from restored data version. +12/20/2022 10:12:06 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:06 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) +12/20/2022 10:12:06 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:08 - INFO - datasets.info - Loading Dataset Infos from /home/milan/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:08 - INFO - datasets.builder - Overwrite dataset info from restored data version. +12/20/2022 10:12:08 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +12/20/2022 10:12:08 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) +12/20/2022 10:12:08 - INFO - datasets.info - Loading Dataset info from /home/milan/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/cs/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f +[INFO|configuration_utils.py:654] 2022-12-20 10:12:08,655 >> loading configuration file config.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/config.json +[INFO|configuration_utils.py:706] 2022-12-20 10:12:08,659 >> Model config WhisperConfig { "_name_or_path": "openai/whisper-large-v2", "activation_dropout": 0.0, "activation_function": "gelu", @@ -386,8 +386,8 @@ xpu_backend=None, "vocab_size": 51865 } -[INFO|feature_extraction_utils.py:464] 2022-12-20 04:28:57,307 >> loading configuration file preprocessor_config.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/preprocessor_config.json -[INFO|feature_extraction_utils.py:501] 2022-12-20 04:28:57,323 >> Feature extractor WhisperFeatureExtractor { +[INFO|feature_extraction_utils.py:464] 2022-12-20 10:12:08,896 >> loading configuration file preprocessor_config.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/preprocessor_config.json +[INFO|feature_extraction_utils.py:501] 2022-12-20 10:12:08,912 >> Feature extractor WhisperFeatureExtractor { "chunk_length": 30, "feature_extractor_type": "WhisperFeatureExtractor", "feature_size": 80, @@ -16644,139 +16644,137 @@ xpu_backend=None, "sampling_rate": 16000 } -[INFO|tokenization_utils_base.py:1799] 2022-12-20 04:28:57,604 >> loading file vocab.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/vocab.json -[INFO|tokenization_utils_base.py:1799] 2022-12-20 04:28:57,604 >> loading file tokenizer.json from cache at None -[INFO|tokenization_utils_base.py:1799] 2022-12-20 04:28:57,604 >> loading file merges.txt from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/merges.txt -[INFO|tokenization_utils_base.py:1799] 2022-12-20 04:28:57,604 >> loading file normalizer.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/normalizer.json -[INFO|tokenization_utils_base.py:1799] 2022-12-20 04:28:57,604 >> loading file added_tokens.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/added_tokens.json -[INFO|tokenization_utils_base.py:1799] 2022-12-20 04:28:57,604 >> loading file special_tokens_map.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/special_tokens_map.json -[INFO|tokenization_utils_base.py:1799] 2022-12-20 04:28:57,604 >> loading file tokenizer_config.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/tokenizer_config.json -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|endoftext|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|startoftranscript|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|en|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|zh|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|de|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|es|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|ru|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|ko|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|fr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|ja|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|pt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|tr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|pl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|ca|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|nl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|ar|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|sv|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|it|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|id|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|hi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|fi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|vi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|iw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|uk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|el|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|ms|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|cs|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|ro|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|da|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,666 >> Adding <|hu|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|ta|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|no|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|th|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|ur|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|hr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|bg|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|lt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|la|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|mi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|ml|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|cy|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|sk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|te|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|fa|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|lv|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|bn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|sr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|az|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|sl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|kn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|et|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|mk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|br|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|eu|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|is|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|hy|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|ne|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|mn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|bs|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|kk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|sq|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|sw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|gl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|mr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|pa|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|si|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,667 >> Adding <|km|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|sn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|yo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|so|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|af|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|oc|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|ka|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|be|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|tg|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|sd|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|gu|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|am|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|yi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|lo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|uz|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|fo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|ht|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|ps|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|tk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|nn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|mt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|sa|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|lb|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|my|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|bo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|tl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|mg|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|as|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|tt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|haw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|ln|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|ha|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|ba|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|jw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|su|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,668 >> Adding <|translate|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,669 >> Adding <|transcribe|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,669 >> Adding <|startoflm|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,669 >> Adding <|startofprev|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,669 >> Adding <|nocaptions|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 04:28:57,669 >> Adding <|notimestamps|> to the vocabulary -[INFO|modeling_utils.py:2251] 2022-12-20 04:28:57,673 >> loading weights file pytorch_model.bin from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/pytorch_model.bin -[INFO|modeling_utils.py:2806] 2022-12-20 04:29:14,265 >> All model checkpoint weights were used when initializing WhisperForConditionalGeneration. +[INFO|tokenization_utils_base.py:1799] 2022-12-20 10:12:09,157 >> loading file vocab.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/vocab.json +[INFO|tokenization_utils_base.py:1799] 2022-12-20 10:12:09,157 >> loading file tokenizer.json from cache at None +[INFO|tokenization_utils_base.py:1799] 2022-12-20 10:12:09,157 >> loading file merges.txt from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/merges.txt +[INFO|tokenization_utils_base.py:1799] 2022-12-20 10:12:09,157 >> loading file normalizer.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/normalizer.json +[INFO|tokenization_utils_base.py:1799] 2022-12-20 10:12:09,157 >> loading file added_tokens.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/added_tokens.json +[INFO|tokenization_utils_base.py:1799] 2022-12-20 10:12:09,157 >> loading file special_tokens_map.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/special_tokens_map.json +[INFO|tokenization_utils_base.py:1799] 2022-12-20 10:12:09,157 >> loading file tokenizer_config.json from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/tokenizer_config.json +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|endoftext|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|startoftranscript|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|en|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|zh|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|de|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|es|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|ru|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|ko|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|fr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|ja|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|pt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|tr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|pl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|ca|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|nl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|ar|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|sv|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|it|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|id|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|hi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|fi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|vi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,220 >> Adding <|iw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|uk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|el|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|ms|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|cs|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|ro|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|da|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|hu|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|ta|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|no|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|th|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|ur|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|hr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|bg|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|lt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|la|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|mi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|ml|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|cy|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|sk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|te|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|fa|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|lv|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|bn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|sr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|az|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|sl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|kn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|et|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|mk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|br|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|eu|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|is|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|hy|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|ne|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|mn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,221 >> Adding <|bs|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|kk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|sq|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|sw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|gl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|mr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|pa|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|si|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|km|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|sn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|yo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|so|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|af|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|oc|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|ka|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|be|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|tg|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|sd|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|gu|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|am|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|yi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|lo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|uz|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|fo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|ht|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|ps|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|tk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|nn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|mt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|sa|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|lb|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|my|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|bo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,222 >> Adding <|tl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|mg|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|as|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|tt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|haw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|ln|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|ha|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|ba|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|jw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|su|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|translate|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|transcribe|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|startoflm|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|startofprev|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|nocaptions|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:12:09,223 >> Adding <|notimestamps|> to the vocabulary +[INFO|modeling_utils.py:2251] 2022-12-20 10:12:09,227 >> loading weights file pytorch_model.bin from cache at /home/milan/.cache/huggingface/hub/models--openai--whisper-large-v2/snapshots/e823955b7861a1d66fef509b8601ada6d7762c03/pytorch_model.bin +[INFO|modeling_utils.py:2806] 2022-12-20 10:12:25,819 >> All model checkpoint weights were used when initializing WhisperForConditionalGeneration. -[INFO|modeling_utils.py:2814] 2022-12-20 04:29:14,265 >> All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at openai/whisper-large-v2. +[INFO|modeling_utils.py:2814] 2022-12-20 10:12:25,819 >> All the weights of WhisperForConditionalGeneration were initialized from the model checkpoint at openai/whisper-large-v2. If your task is similar to the task the model of the checkpoint was trained on, you can already use WhisperForConditionalGeneration for predictions without further training. - 0%| | 0/15086 [00:00> Feature extractor saved in ./preprocessor_config.json -[INFO|tokenization_utils_base.py:2157] 2022-12-20 05:27:32,397 >> tokenizer config file saved in ./tokenizer_config.json -[INFO|tokenization_utils_base.py:2164] 2022-12-20 05:27:32,397 >> Special tokens file saved in ./special_tokens_map.json -[INFO|tokenization_utils_base.py:2210] 2022-12-20 05:27:32,397 >> added tokens file saved in ./added_tokens.json -[INFO|configuration_utils.py:447] 2022-12-20 05:27:32,475 >> Configuration saved in ./config.json -[INFO|image_processing_utils.py:294] 2022-12-20 05:27:32,477 >> loading configuration file ./preprocessor_config.json -[INFO|feature_extraction_utils.py:462] 2022-12-20 05:27:32,482 >> loading configuration file ./preprocessor_config.json -[INFO|feature_extraction_utils.py:501] 2022-12-20 05:27:32,496 >> Feature extractor WhisperFeatureExtractor { + 0%| | 0/236 [00:00> Feature extractor saved in ./preprocessor_config.json +[INFO|tokenization_utils_base.py:2157] 2022-12-20 10:22:02,747 >> tokenizer config file saved in ./tokenizer_config.json +[INFO|tokenization_utils_base.py:2164] 2022-12-20 10:22:02,747 >> Special tokens file saved in ./special_tokens_map.json +[INFO|tokenization_utils_base.py:2210] 2022-12-20 10:22:02,747 >> added tokens file saved in ./added_tokens.json +[INFO|configuration_utils.py:447] 2022-12-20 10:22:02,828 >> Configuration saved in ./config.json +[INFO|image_processing_utils.py:294] 2022-12-20 10:22:02,830 >> loading configuration file ./preprocessor_config.json +[INFO|feature_extraction_utils.py:462] 2022-12-20 10:22:02,835 >> loading configuration file ./preprocessor_config.json +[INFO|feature_extraction_utils.py:501] 2022-12-20 10:22:02,850 >> Feature extractor WhisperFeatureExtractor { "chunk_length": 30, "feature_extractor_type": "WhisperFeatureExtractor", "feature_size": 80, @@ -33033,128 +33031,127 @@ If your task is similar to the task the model of the checkpoint was trained on, "sampling_rate": 16000 } -[INFO|tokenization_utils_base.py:1797] 2022-12-20 05:27:32,530 >> loading file vocab.json -[INFO|tokenization_utils_base.py:1797] 2022-12-20 05:27:32,530 >> loading file tokenizer.json -[INFO|tokenization_utils_base.py:1797] 2022-12-20 05:27:32,530 >> loading file merges.txt -[INFO|tokenization_utils_base.py:1797] 2022-12-20 05:27:32,530 >> loading file normalizer.json -[INFO|tokenization_utils_base.py:1797] 2022-12-20 05:27:32,530 >> loading file added_tokens.json -[INFO|tokenization_utils_base.py:1797] 2022-12-20 05:27:32,530 >> loading file special_tokens_map.json -[INFO|tokenization_utils_base.py:1797] 2022-12-20 05:27:32,530 >> loading file tokenizer_config.json -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|endoftext|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|startoftranscript|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|en|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|zh|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|de|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|es|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|ru|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|ko|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|fr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|ja|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|pt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|tr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|pl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|ca|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|nl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|ar|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|sv|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|it|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|id|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|hi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,581 >> Adding <|fi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|vi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|iw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|uk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|el|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|ms|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|cs|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|ro|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|da|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|hu|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|ta|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|no|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|th|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|ur|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|hr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|bg|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|lt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|la|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|mi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|ml|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|cy|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|sk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|te|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|fa|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|lv|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|bn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|sr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|az|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|sl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|kn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|et|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|mk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|br|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|eu|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|is|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,582 >> Adding <|hy|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|ne|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|mn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|bs|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|kk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|sq|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|sw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|gl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|mr|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|pa|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|si|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|km|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|sn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|yo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|so|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|af|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|oc|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|ka|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|be|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|tg|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|sd|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|gu|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|am|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|yi|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|lo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|uz|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|fo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|ht|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|ps|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|tk|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|nn|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|mt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|sa|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|lb|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|my|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,583 >> Adding <|bo|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|tl|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|mg|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|as|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|tt|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|haw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|ln|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|ha|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|ba|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|jw|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|su|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|translate|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|transcribe|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|startoflm|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|startofprev|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|nocaptions|> to the vocabulary -[INFO|tokenization_utils.py:426] 2022-12-20 05:27:32,584 >> Adding <|notimestamps|> to the vocabulary +[INFO|tokenization_utils_base.py:1797] 2022-12-20 10:22:02,861 >> loading file vocab.json +[INFO|tokenization_utils_base.py:1797] 2022-12-20 10:22:02,861 >> loading file tokenizer.json +[INFO|tokenization_utils_base.py:1797] 2022-12-20 10:22:02,861 >> loading file merges.txt +[INFO|tokenization_utils_base.py:1797] 2022-12-20 10:22:02,861 >> loading file normalizer.json +[INFO|tokenization_utils_base.py:1797] 2022-12-20 10:22:02,861 >> loading file added_tokens.json +[INFO|tokenization_utils_base.py:1797] 2022-12-20 10:22:02,861 >> loading file special_tokens_map.json +[INFO|tokenization_utils_base.py:1797] 2022-12-20 10:22:02,861 >> loading file tokenizer_config.json +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,917 >> Adding <|endoftext|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,917 >> Adding <|startoftranscript|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,917 >> Adding <|en|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,917 >> Adding <|zh|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,917 >> Adding <|de|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,917 >> Adding <|es|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,917 >> Adding <|ru|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|ko|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|fr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|ja|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|pt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|tr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|pl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|ca|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|nl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|ar|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|sv|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|it|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|id|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|hi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|fi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|vi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|iw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|uk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|el|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|ms|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,918 >> Adding <|cs|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|ro|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|da|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|hu|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|ta|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|no|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|th|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|ur|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|hr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|bg|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|lt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|la|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|mi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|ml|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|cy|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|sk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|te|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|fa|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|lv|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|bn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|sr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|az|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,919 >> Adding <|sl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|kn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|et|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|mk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|br|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|eu|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|is|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|hy|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|ne|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|mn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|bs|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|kk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|sq|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|sw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|gl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|mr|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|pa|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|si|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|km|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|sn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|yo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|so|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,920 >> Adding <|af|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|oc|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|ka|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|be|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|tg|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|sd|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|gu|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|am|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|yi|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|lo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|uz|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|fo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|ht|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|ps|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|tk|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|nn|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|mt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|sa|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|lb|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|my|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,921 >> Adding <|bo|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|tl|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|mg|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|as|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|tt|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|haw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|ln|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|ha|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|ba|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|jw|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|su|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|translate|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|transcribe|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|startoflm|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|startofprev|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|nocaptions|> to the vocabulary +[INFO|tokenization_utils.py:426] 2022-12-20 10:22:02,922 >> Adding <|notimestamps|> to the vocabulary /home/milan/whisper-audio-concat-test/./ is already a clone of https://huggingface.co/mikr/whisper-audio-concat-test. Make sure you pull the latest changes with `repo.git_pull()`. -12/20/2022 05:27:34 - WARNING - huggingface_hub.repository - /home/milan/whisper-audio-concat-test/./ is already a clone of https://huggingface.co/mikr/whisper-audio-concat-test. Make sure you pull the latest changes with `repo.git_pull()`. -[INFO|trainer.py:511] 2022-12-20 05:27:38,129 >> max_steps is given, it will override any value given in num_train_epochs -[INFO|trainer.py:565] 2022-12-20 05:27:38,129 >> Using cuda_amp half precision backend -[INFO|trainer.py:710] 2022-12-20 05:27:38,130 >> The following columns in the training set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: audio, input_length, sentence. If audio, input_length, sentence are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message. -[2022-12-20 05:27:38,137] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.7, git-hash=unknown, git-branch=unknown -[2022-12-20 05:27:39,430] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False -[2022-12-20 05:27:40,636] [WARNING] [cpu_adam.py:83:__init__] FP16 params for CPUAdam may not work on AMD CPUs +12/20/2022 10:22:04 - WARNING - huggingface_hub.repository - /home/milan/whisper-audio-concat-test/./ is already a clone of https://huggingface.co/mikr/whisper-audio-concat-test. Make sure you pull the latest changes with `repo.git_pull()`. +[INFO|trainer.py:511] 2022-12-20 10:22:08,793 >> max_steps is given, it will override any value given in num_train_epochs +[INFO|trainer.py:565] 2022-12-20 10:22:08,794 >> Using cuda_amp half precision backend +[2022-12-20 10:22:08,801] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.7.7, git-hash=unknown, git-branch=unknown +[2022-12-20 10:22:09,703] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False +[2022-12-20 10:22:10,934] [WARNING] [cpu_adam.py:83:__init__] FP16 params for CPUAdam may not work on AMD CPUs Installed CUDA version 11.6 does not match the version torch was compiled with 11.7 but since the APIs are compatible, accepting this combination Using /home/milan/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... Detected CUDA files, patching ldflags @@ -33163,41 +33160,41 @@ Building extension module cpu_adam... Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) ninja: no work to do. Loading extension module cpu_adam... -Time to load cpu_adam op: 3.1376593112945557 seconds +Time to load cpu_adam op: 3.2882912158966064 seconds Adam Optimizer #0 is created with AVX2 arithmetic capability. Config: alpha=0.000010, betas=(0.900000, 0.999000), weight_decay=0.000000, adam_w=1 -[2022-12-20 05:27:45,942] [INFO] [logging.py:68:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adamw as basic optimizer -[2022-12-20 05:27:46,247] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam -[2022-12-20 05:27:46,247] [INFO] [utils.py:52:is_zero_supported_optimizer] Checking ZeRO support for optimizer=DeepSpeedCPUAdam type= -[2022-12-20 05:27:46,247] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 ZeRO stage 2 optimizer -[2022-12-20 05:27:46,247] [INFO] [stage_1_and_2.py:140:__init__] Reduce bucket size 200000000 -[2022-12-20 05:27:46,247] [INFO] [stage_1_and_2.py:141:__init__] Allgather bucket size 200000000 -[2022-12-20 05:27:46,247] [INFO] [stage_1_and_2.py:142:__init__] CPU Offload: True -[2022-12-20 05:27:46,247] [INFO] [stage_1_and_2.py:143:__init__] Round robin gradient partitioning: False +[2022-12-20 10:22:16,248] [INFO] [logging.py:68:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adamw as basic optimizer +[2022-12-20 10:22:16,551] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam +[2022-12-20 10:22:16,551] [INFO] [utils.py:52:is_zero_supported_optimizer] Checking ZeRO support for optimizer=DeepSpeedCPUAdam type= +[2022-12-20 10:22:16,551] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 ZeRO stage 2 optimizer +[2022-12-20 10:22:16,551] [INFO] [stage_1_and_2.py:140:__init__] Reduce bucket size 200000000 +[2022-12-20 10:22:16,552] [INFO] [stage_1_and_2.py:141:__init__] Allgather bucket size 200000000 +[2022-12-20 10:22:16,552] [INFO] [stage_1_and_2.py:142:__init__] CPU Offload: True +[2022-12-20 10:22:16,552] [INFO] [stage_1_and_2.py:143:__init__] Round robin gradient partitioning: False Using /home/milan/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... Emitting ninja build file /home/milan/.cache/torch_extensions/py38_cu117/utils/build.ninja... Building extension module utils... Allowing ninja to set a default number of workers... (overridable by setting the environment variable MAX_JOBS=N) ninja: no work to do. Loading extension module utils... -Time to load utils op: 0.5715920925140381 seconds +Time to load utils op: 0.6276412010192871 seconds Rank: 0 partition count [1] and sizes[(1543304960, False)] -[2022-12-20 05:27:50,286] [INFO] [utils.py:827:see_memory_usage] Before initializing optimizer states -[2022-12-20 05:27:50,287] [INFO] [utils.py:828:see_memory_usage] MA 3.0 GB Max_MA 3.0 GB CA 5.99 GB Max_CA 6 GB -[2022-12-20 05:27:50,287] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 15.52 GB, percent = 7.9% -[2022-12-20 05:27:54,592] [INFO] [utils.py:827:see_memory_usage] After initializing optimizer states -[2022-12-20 05:27:54,593] [INFO] [utils.py:828:see_memory_usage] MA 3.0 GB Max_MA 3.0 GB CA 5.99 GB Max_CA 6 GB -[2022-12-20 05:27:54,594] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 35.27 GB, percent = 17.9% -[2022-12-20 05:27:54,594] [INFO] [stage_1_and_2.py:525:__init__] optimizer state initialized -[2022-12-20 05:27:54,719] [INFO] [utils.py:827:see_memory_usage] After initializing ZeRO optimizer -[2022-12-20 05:27:54,719] [INFO] [utils.py:828:see_memory_usage] MA 3.0 GB Max_MA 3.0 GB CA 5.99 GB Max_CA 6 GB -[2022-12-20 05:27:54,720] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 35.19 GB, percent = 17.9% -[2022-12-20 05:27:54,747] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = adamw -[2022-12-20 05:27:54,747] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using configured LR scheduler = WarmupDecayLR -[2022-12-20 05:27:54,747] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = -[2022-12-20 05:27:54,747] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[1e-05], mom=[[0.9, 0.999]] -[2022-12-20 05:27:54,749] [INFO] [config.py:1020:print] DeepSpeedEngine configuration: -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] activation_checkpointing_config { +[2022-12-20 10:22:20,332] [INFO] [utils.py:827:see_memory_usage] Before initializing optimizer states +[2022-12-20 10:22:20,333] [INFO] [utils.py:828:see_memory_usage] MA 3.0 GB Max_MA 3.0 GB CA 5.99 GB Max_CA 6 GB +[2022-12-20 10:22:20,333] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 15.43 GB, percent = 7.8% +[2022-12-20 10:22:24,263] [INFO] [utils.py:827:see_memory_usage] After initializing optimizer states +[2022-12-20 10:22:24,264] [INFO] [utils.py:828:see_memory_usage] MA 3.0 GB Max_MA 3.0 GB CA 5.99 GB Max_CA 6 GB +[2022-12-20 10:22:24,264] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 35.08 GB, percent = 17.8% +[2022-12-20 10:22:24,264] [INFO] [stage_1_and_2.py:525:__init__] optimizer state initialized +[2022-12-20 10:22:24,345] [INFO] [utils.py:827:see_memory_usage] After initializing ZeRO optimizer +[2022-12-20 10:22:24,346] [INFO] [utils.py:828:see_memory_usage] MA 3.0 GB Max_MA 3.0 GB CA 5.99 GB Max_CA 6 GB +[2022-12-20 10:22:24,346] [INFO] [utils.py:836:see_memory_usage] CPU Virtual Memory: used = 35.08 GB, percent = 17.8% +[2022-12-20 10:22:24,379] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = adamw +[2022-12-20 10:22:24,380] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using configured LR scheduler = WarmupDecayLR +[2022-12-20 10:22:24,380] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = +[2022-12-20 10:22:24,380] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[1e-05], mom=[[0.9, 0.999]] +[2022-12-20 10:22:24,382] [INFO] [config.py:1020:print] DeepSpeedEngine configuration: +[2022-12-20 10:22:24,382] [INFO] [config.py:1024:print] activation_checkpointing_config { "partition_activations": false, "contiguous_memory_optimization": false, "cpu_checkpointing": false, @@ -33205,10 +33202,10 @@ Rank: 0 partition count [1] and sizes[(1543304960, False)] "synchronize_checkpoint_boundary": false, "profile": false } -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] amp_enabled .................. False -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] amp_params ................... False -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] autotuning_config ............ { +[2022-12-20 10:22:24,382] [INFO] [config.py:1024:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +[2022-12-20 10:22:24,382] [INFO] [config.py:1024:print] amp_enabled .................. False +[2022-12-20 10:22:24,382] [INFO] [config.py:1024:print] amp_params ................... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] autotuning_config ............ { "enabled": false, "start_step": null, "end_step": null, @@ -33233,29 +33230,29 @@ Rank: 0 partition count [1] and sizes[(1543304960, False)] "min_train_micro_batch_size_per_gpu": 1, "num_tuning_micro_batch_sizes": 3 } -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] bfloat16_enabled ............. False -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] checkpoint_parallel_write_pipeline False -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] checkpoint_tag_validation_enabled True -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] checkpoint_tag_validation_fail False -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] comms_config ................. -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] communication_data_type ...... None -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} -[2022-12-20 05:27:54,750] [INFO] [config.py:1024:print] curriculum_enabled ........... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] curriculum_params ............ False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] dataloader_drop_last ......... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] disable_allgather ............ False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] dump_state ................... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] dynamic_loss_scale_args ...... {'init_scale': 65536, 'scale_window': 1000, 'delayed_shift': 2, 'min_scale': 1} -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_enabled ........... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_gas_boundary_resolution 1 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_layer_name ........ bert.encoder.layer -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_layer_num ......... 0 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_max_iter .......... 100 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_stability ......... 1e-06 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_tol ............... 0.01 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] eigenvalue_verbose ........... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] elasticity_enabled ........... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] flops_profiler_config ........ { +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] bfloat16_enabled ............. False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] checkpoint_parallel_write_pipeline False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] checkpoint_tag_validation_enabled True +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] checkpoint_tag_validation_fail False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] comms_config ................. +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] communication_data_type ...... None +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] curriculum_enabled ........... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] curriculum_params ............ False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] dataloader_drop_last ......... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] disable_allgather ............ False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] dump_state ................... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] dynamic_loss_scale_args ...... {'init_scale': 65536, 'scale_window': 1000, 'delayed_shift': 2, 'min_scale': 1} +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_enabled ........... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_gas_boundary_resolution 1 +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_layer_name ........ bert.encoder.layer +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_layer_num ......... 0 +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_max_iter .......... 100 +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_stability ......... 1e-06 +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_tol ............... 0.01 +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] eigenvalue_verbose ........... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] elasticity_enabled ........... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] flops_profiler_config ........ { "enabled": false, "profile_step": 1, "module_depth": -1, @@ -33263,20 +33260,20 @@ Rank: 0 partition count [1] and sizes[(1543304960, False)] "detailed": true, "output_file": null } -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] fp16_auto_cast ............... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] fp16_enabled ................. True -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] fp16_master_weights_and_gradients False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] global_rank .................. 0 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] grad_accum_dtype ............. None -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] gradient_accumulation_steps .. 2 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] gradient_clipping ............ 1.0 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] gradient_predivide_factor .... 1.0 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] initial_dynamic_scale ........ 65536 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] load_universal_checkpoint .... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] loss_scale ................... 0 -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] memory_breakdown ............. False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] monitor_config ............... -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] nebula_config ................ { +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] fp16_auto_cast ............... False +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] fp16_enabled ................. True +[2022-12-20 10:22:24,383] [INFO] [config.py:1024:print] fp16_master_weights_and_gradients False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] global_rank .................. 0 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] grad_accum_dtype ............. None +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] gradient_accumulation_steps .. 2 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] gradient_clipping ............ 1.0 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] gradient_predivide_factor .... 1.0 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] initial_dynamic_scale ........ 65536 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] load_universal_checkpoint .... False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] loss_scale ................... 0 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] memory_breakdown ............. False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] monitor_config ............... +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] nebula_config ................ { "enabled": false, "persistent_storage_path": null, "persistent_time_interval": 100, @@ -33284,28 +33281,28 @@ Rank: 0 partition count [1] and sizes[(1543304960, False)] "enable_nebula_load": true, "load_path": null } -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] optimizer_legacy_fusion ...... False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] optimizer_name ............... adamw -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] optimizer_params ............. {'lr': 1e-05, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0} -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] pld_enabled .................. False -[2022-12-20 05:27:54,751] [INFO] [config.py:1024:print] pld_params ................... False -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] prescale_gradients ........... False -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] scheduler_name ............... WarmupDecayLR -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] scheduler_params ............. {'last_batch_iteration': -1, 'total_num_steps': 5000, 'warmup_min_lr': 0, 'warmup_max_lr': 1e-05, 'warmup_num_steps': 500} -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] sparse_attention ............. None -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] sparse_gradients_enabled ..... False -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] steps_per_print .............. 10 -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] train_batch_size ............. 64 -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] train_micro_batch_size_per_gpu 32 -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] use_node_local_storage ....... False -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] wall_clock_breakdown ......... False -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] world_size ................... 1 -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] zero_allow_untested_optimizer False -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=200000000 allgather_partitions=True allgather_bucket_size=200000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=True, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] zero_enabled ................. True -[2022-12-20 05:27:54,752] [INFO] [config.py:1024:print] zero_optimization_stage ...... 2 -[2022-12-20 05:27:54,752] [INFO] [config.py:1009:print_user_config] json = { +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] optimizer_legacy_fusion ...... False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] optimizer_name ............... adamw +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] optimizer_params ............. {'lr': 1e-05, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0} +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] pld_enabled .................. False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] pld_params ................... False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] prescale_gradients ........... False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] scheduler_name ............... WarmupDecayLR +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] scheduler_params ............. {'last_batch_iteration': -1, 'total_num_steps': 5000, 'warmup_min_lr': 0, 'warmup_max_lr': 1e-05, 'warmup_num_steps': 500} +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] sparse_attention ............. None +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] sparse_gradients_enabled ..... False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] steps_per_print .............. 10 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] train_batch_size ............. 64 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] train_micro_batch_size_per_gpu 32 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] use_node_local_storage ....... False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] wall_clock_breakdown ......... False +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] world_size ................... 1 +[2022-12-20 10:22:24,384] [INFO] [config.py:1024:print] zero_allow_untested_optimizer False +[2022-12-20 10:22:24,385] [INFO] [config.py:1024:print] zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=200000000 allgather_partitions=True allgather_bucket_size=200000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=True, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False +[2022-12-20 10:22:24,385] [INFO] [config.py:1024:print] zero_enabled ................. True +[2022-12-20 10:22:24,385] [INFO] [config.py:1024:print] zero_optimization_stage ...... 2 +[2022-12-20 10:22:24,385] [INFO] [config.py:1009:print_user_config] json = { "fp16": { "enabled": true, "loss_scale": 0, @@ -33354,2144 +33351,1355 @@ Rank: 0 partition count [1] and sizes[(1543304960, False)] Using /home/milan/.cache/torch_extensions/py38_cu117 as PyTorch extensions root... No modifications detected for re-loaded extension module utils, skipping build step... Loading extension module utils... -Time to load utils op: 0.00042319297790527344 seconds -[INFO|trainer.py:1641] 2022-12-20 05:27:54,755 >> ***** Running training ***** -[INFO|trainer.py:1642] 2022-12-20 05:27:54,755 >> Num examples = 15086 -[INFO|trainer.py:1643] 2022-12-20 05:27:54,755 >> Num Epochs = 22 -[INFO|trainer.py:1644] 2022-12-20 05:27:54,755 >> Instantaneous batch size per device = 32 -[INFO|trainer.py:1645] 2022-12-20 05:27:54,755 >> Total train batch size (w. parallel, distributed & accumulation) = 64 -[INFO|trainer.py:1646] 2022-12-20 05:27:54,755 >> Gradient Accumulation steps = 2 -[INFO|trainer.py:1647] 2022-12-20 05:27:54,755 >> Total optimization steps = 5000 -[INFO|trainer.py:1648] 2022-12-20 05:27:54,760 >> Number of trainable parameters = 1543304960 - 0%| | 0/5000 [00:00> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: audio, input_length, sentence. If audio, input_length, sentence are not expected by `WhisperForConditionalGeneration.forward`, you can safely ignore this message. -[INFO|trainer.py:2955] 2022-12-20 08:38:10,026 >> ***** Running Evaluation ***** -[INFO|trainer.py:2957] 2022-12-20 08:38:10,026 >> Num examples = 7714 -[INFO|trainer.py:2960] 2022-12-20 08:38:10,026 >> Batch size = 8 +Time to load utils op: 0.00040650367736816406 seconds +[INFO|trainer.py:1641] 2022-12-20 10:22:24,388 >> ***** Running training ***** +[INFO|trainer.py:1642] 2022-12-20 10:22:24,388 >> Num examples = 2650 +[INFO|trainer.py:1643] 2022-12-20 10:22:24,388 >> Num Epochs = 122 +[INFO|trainer.py:1644] 2022-12-20 10:22:24,388 >> Instantaneous batch size per device = 32 +[INFO|trainer.py:1645] 2022-12-20 10:22:24,388 >> Total train batch size (w. parallel, distributed & accumulation) = 64 +[INFO|trainer.py:1646] 2022-12-20 10:22:24,388 >> Gradient Accumulation steps = 2 +[INFO|trainer.py:1647] 2022-12-20 10:22:24,388 >> Total optimization steps = 5000 +[INFO|trainer.py:1648] 2022-12-20 10:22:24,392 >> Number of trainable parameters = 1543304960 + 0%| | 0/5000 [00:00> ***** Running Evaluation ***** +[INFO|trainer.py:2957] 2022-12-20 13:28:44,584 >> Num examples = 1325 +[INFO|trainer.py:2960] 2022-12-20 13:28:44,584 >> Batch size = 8 - 0%| | 0/965 [00:00> Saving model checkpoint to ./checkpoint-1000 -[INFO|configuration_utils.py:447] 2022-12-20 09:13:49,236 >> Configuration saved in ./checkpoint-1000/config.json -[INFO|modeling_utils.py:1680] 2022-12-20 09:13:52,715 >> Model weights saved in ./checkpoint-1000/pytorch_model.bin -[INFO|feature_extraction_utils.py:368] 2022-12-20 09:13:52,729 >> Feature extractor saved in ./checkpoint-1000/preprocessor_config.json -[2022-12-20 09:13:52,730] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is begin to save! + 0%| | 0/166 [00:00> Saving model checkpoint to ./checkpoint-1000 +[INFO|configuration_utils.py:447] 2022-12-20 13:50:11,172 >> Configuration saved in ./checkpoint-1000/config.json +[INFO|modeling_utils.py:1680] 2022-12-20 13:50:15,406 >> Model weights saved in ./checkpoint-1000/pytorch_model.bin +[INFO|feature_extraction_utils.py:368] 2022-12-20 13:50:15,421 >> Feature extractor saved in ./checkpoint-1000/preprocessor_config.json +[2022-12-20 13:50:15,422] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1012 is begin to save! /home/milan/hf_env/lib/python3.8/site-packages/torch/nn/modules/module.py:1428: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details. warnings.warn( -[2022-12-20 09:13:52,740] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-1000/global_step1000/mp_rank_00_model_states.pt -[2022-12-20 09:13:52,740] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-1000/global_step1000/mp_rank_00_model_states.pt... -[2022-12-20 09:13:56,319] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-1000/global_step1000/mp_rank_00_model_states.pt. -[2022-12-20 09:13:56,320] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt... -[2022-12-20 09:14:11,751] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt. -[2022-12-20 09:14:11,751] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt -[2022-12-20 09:14:11,751] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! -[INFO|feature_extraction_utils.py:368] 2022-12-20 09:14:13,300 >> Feature extractor saved in ./preprocessor_config.json +[2022-12-20 13:50:15,432] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-1000/global_step1012/mp_rank_00_model_states.pt +[2022-12-20 13:50:15,432] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-1000/global_step1012/mp_rank_00_model_states.pt... +[2022-12-20 13:50:18,927] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-1000/global_step1012/mp_rank_00_model_states.pt. +[2022-12-20 13:50:18,929] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-1000/global_step1012/zero_pp_rank_0_mp_rank_00_optim_states.pt... +[2022-12-20 13:50:34,343] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-1000/global_step1012/zero_pp_rank_0_mp_rank_00_optim_states.pt. +[2022-12-20 13:50:34,343] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-1000/global_step1012/zero_pp_rank_0_mp_rank_00_optim_states.pt +[2022-12-20 13:50:34,343] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1012 is ready now! +[INFO|feature_extraction_utils.py:368] 2022-12-20 13:50:36,874 >> Feature extractor saved in ./preprocessor_config.json