| [2022-12-19 11:14:18,662] [WARNING] [runner.py:179:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only. | |
| [2022-12-19 11:14:18,671] [INFO] [runner.py:508:main] cmd = /usr/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF19 --master_addr=127.0.0.1 --master_port=29500 run_speech_recognition_seq2seq_streaming.py --deepspeed=ds_config.json --model_name_or_path=openai/whisper-medium --dataset_name=mozilla-foundation/common_voice_11_0 --dataset_config_name=sk --language=slovak --train_split_name=train+validation --eval_split_name=test --model_index_name=Whisper Medium Slovak CV11 --max_steps=5000 --output_dir=./ --per_device_train_batch_size=64 --per_device_eval_batch_size=32 --logging_steps=25 --learning_rate=1e-5 --warmup_steps=500 --evaluation_strategy=steps --eval_steps=1000 --save_strategy=steps --save_steps=1000 --generation_max_length=225 --length_column_name=input_length --max_duration_in_seconds=30 --text_column_name=sentence --freeze_feature_encoder=False --report_to=tensorboard --metric_for_best_model=wer --greater_is_better=False --load_best_model_at_end --gradient_checkpointing --fp16 --overwrite_output_dir --do_train --do_eval --predict_with_generate --do_normalize_eval --streaming=False --use_auth_token --push_to_hub | |
| [2022-12-19 11:14:21,921] [INFO] [launch.py:135:main] 0 NV_LIBNCCL_DEV_PACKAGE=libnccl-dev=2.13.4-1+cuda11.7 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:135:main] 0 NV_LIBNCCL_DEV_PACKAGE_VERSION=2.13.4-1 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:135:main] 0 NCCL_VERSION=2.13.4-1 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:135:main] 0 NV_LIBNCCL_DEV_PACKAGE_NAME=libnccl-dev | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:135:main] 0 NV_LIBNCCL_PACKAGE=libnccl2=2.13.4-1+cuda11.7 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:135:main] 0 NV_LIBNCCL_PACKAGE_NAME=libnccl2 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:135:main] 0 NV_LIBNCCL_PACKAGE_VERSION=2.13.4-1 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:142:main] WORLD INFO DICT: {'localhost': [0]} | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:148:main] nnodes=1, num_local_procs=1, node_rank=0 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:161:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0]}) | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:162:main] dist_world_size=1 | |
| [2022-12-19 11:14:21,922] [INFO] [launch.py:164:main] Setting CUDA_VISIBLE_DEVICES=0 | |
| [2022-12-19 11:14:29,454] [INFO] [comm.py:654:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl | |
| 12/19/2022 11:14:29 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: True | |
| 12/19/2022 11:14:29 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments( | |
| _n_gpu=1, | |
| adafactor=False, | |
| adam_beta1=0.9, | |
| adam_beta2=0.999, | |
| adam_epsilon=1e-08, | |
| auto_find_batch_size=False, | |
| bf16=False, | |
| bf16_full_eval=False, | |
| data_seed=None, | |
| dataloader_drop_last=False, | |
| dataloader_num_workers=0, | |
| dataloader_pin_memory=True, | |
| ddp_bucket_cap_mb=None, | |
| ddp_find_unused_parameters=None, | |
| ddp_timeout=1800, | |
| debug=[], | |
| deepspeed=ds_config.json, | |
| disable_tqdm=False, | |
| do_eval=True, | |
| do_predict=False, | |
| do_train=True, | |
| eval_accumulation_steps=None, | |
| eval_delay=0, | |
| eval_steps=1000, | |
| evaluation_strategy=steps, | |
| fp16=True, | |
| fp16_backend=auto, | |
| fp16_full_eval=False, | |
| fp16_opt_level=O1, | |
| fsdp=[], | |
| fsdp_min_num_params=0, | |
| fsdp_transformer_layer_cls_to_wrap=None, | |
| full_determinism=False, | |
| generation_max_length=225, | |
| generation_num_beams=None, | |
| gradient_accumulation_steps=1, | |
| gradient_checkpointing=True, | |
| greater_is_better=False, | |
| group_by_length=False, | |
| half_precision_backend=auto, | |
| hub_model_id=None, | |
| hub_private_repo=False, | |
| hub_strategy=every_save, | |
| hub_token=<HUB_TOKEN>, | |
| ignore_data_skip=False, | |
| include_inputs_for_metrics=False, | |
| jit_mode_eval=False, | |
| label_names=None, | |
| label_smoothing_factor=0.0, | |
| learning_rate=1e-05, | |
| length_column_name=input_length, | |
| load_best_model_at_end=True, | |
| local_rank=0, | |
| log_level=passive, | |
| log_level_replica=passive, | |
| log_on_each_node=True, | |
| logging_dir=./runs/Dec19_11-14-29_fe2747a042f0, | |
| logging_first_step=False, | |
| logging_nan_inf_filter=True, | |
| logging_steps=25, | |
| logging_strategy=steps, | |
| lr_scheduler_type=linear, | |
| max_grad_norm=1.0, | |
| max_steps=5000, | |
| metric_for_best_model=wer, | |
| mp_parameters=, | |
| no_cuda=False, | |
| num_train_epochs=3.0, | |
| optim=adamw_hf, | |
| optim_args=None, | |
| output_dir=./, | |
| overwrite_output_dir=True, | |
| past_index=-1, | |
| per_device_eval_batch_size=32, | |
| per_device_train_batch_size=64, | |
| predict_with_generate=True, | |
| prediction_loss_only=False, | |
| push_to_hub=True, | |
| push_to_hub_model_id=None, | |
| push_to_hub_organization=None, | |
| push_to_hub_token=<PUSH_TO_HUB_TOKEN>, | |
| ray_scope=last, | |
| remove_unused_columns=True, | |
| report_to=['tensorboard'], | |
| resume_from_checkpoint=None, | |
| run_name=./, | |
| save_on_each_node=False, | |
| save_steps=1000, | |
| save_strategy=steps, | |
| save_total_limit=None, | |
| seed=42, | |
| sharded_ddp=[], | |
| skip_memory_metrics=True, | |
| sortish_sampler=False, | |
| tf32=None, | |
| torch_compile=False, | |
| torch_compile_backend=None, | |
| torch_compile_mode=None, | |
| torchdynamo=None, | |
| tpu_metrics_debug=False, | |
| tpu_num_cores=None, | |
| use_ipex=False, | |
| use_legacy_prediction_loop=False, | |
| use_mps_device=False, | |
| warmup_ratio=0.0, | |
| warmup_steps=500, | |
| weight_decay=0.0, | |
| xpu_backend=None, | |
| ) | |
| 12/19/2022 11:14:29 - INFO - __main__ - Training/evaluation parameters Seq2SeqTrainingArguments( | |
| _n_gpu=1, | |
| adafactor=False, | |
| adam_beta1=0.9, | |
| adam_beta2=0.999, | |
| adam_epsilon=1e-08, | |
| auto_find_batch_size=False, | |
| bf16=False, | |
| bf16_full_eval=False, | |
| data_seed=None, | |
| dataloader_drop_last=False, | |
| dataloader_num_workers=0, | |
| dataloader_pin_memory=True, | |
| ddp_bucket_cap_mb=None, | |
| ddp_find_unused_parameters=None, | |
| ddp_timeout=1800, | |
| debug=[], | |
| deepspeed=ds_config.json, | |
| disable_tqdm=False, | |
| do_eval=True, | |
| do_predict=False, | |
| do_train=True, | |
| eval_accumulation_steps=None, | |
| eval_delay=0, | |
| eval_steps=1000, | |
| evaluation_strategy=steps, | |
| fp16=True, | |
| fp16_backend=auto, | |
| fp16_full_eval=False, | |
| fp16_opt_level=O1, | |
| fsdp=[], | |
| fsdp_min_num_params=0, | |
| fsdp_transformer_layer_cls_to_wrap=None, | |
| full_determinism=False, | |
| generation_max_length=225, | |
| generation_num_beams=None, | |
| gradient_accumulation_steps=1, | |
| gradient_checkpointing=True, | |
| greater_is_better=False, | |
| group_by_length=False, | |
| half_precision_backend=auto, | |
| hub_model_id=None, | |
| hub_private_repo=False, | |
| hub_strategy=every_save, | |
| hub_token=<HUB_TOKEN>, | |
| ignore_data_skip=False, | |
| include_inputs_for_metrics=False, | |
| jit_mode_eval=False, | |
| label_names=None, | |
| label_smoothing_factor=0.0, | |
| learning_rate=1e-05, | |
| length_column_name=input_length, | |
| load_best_model_at_end=True, | |
| local_rank=0, | |
| log_level=passive, | |
| log_level_replica=passive, | |
| log_on_each_node=True, | |
| logging_dir=./runs/Dec19_11-14-29_fe2747a042f0, | |
| logging_first_step=False, | |
| logging_nan_inf_filter=True, | |
| logging_steps=25, | |
| logging_strategy=steps, | |
| lr_scheduler_type=linear, | |
| max_grad_norm=1.0, | |
| max_steps=5000, | |
| metric_for_best_model=wer, | |
| mp_parameters=, | |
| no_cuda=False, | |
| num_train_epochs=3.0, | |
| optim=adamw_hf, | |
| optim_args=None, | |
| output_dir=./, | |
| overwrite_output_dir=True, | |
| past_index=-1, | |
| per_device_eval_batch_size=32, | |
| per_device_train_batch_size=64, | |
| predict_with_generate=True, | |
| prediction_loss_only=False, | |
| push_to_hub=True, | |
| push_to_hub_model_id=None, | |
| push_to_hub_organization=None, | |
| push_to_hub_token=<PUSH_TO_HUB_TOKEN>, | |
| ray_scope=last, | |
| remove_unused_columns=True, | |
| report_to=['tensorboard'], | |
| resume_from_checkpoint=None, | |
| run_name=./, | |
| save_on_each_node=False, | |
| save_steps=1000, | |
| save_strategy=steps, | |
| save_total_limit=None, | |
| seed=42, | |
| sharded_ddp=[], | |
| skip_memory_metrics=True, | |
| sortish_sampler=False, | |
| tf32=None, | |
| torch_compile=False, | |
| torch_compile_backend=None, | |
| torch_compile_mode=None, | |
| torchdynamo=None, | |
| tpu_metrics_debug=False, | |
| tpu_num_cores=None, | |
| use_ipex=False, | |
| use_legacy_prediction_loop=False, | |
| use_mps_device=False, | |
| warmup_ratio=0.0, | |
| warmup_steps=500, | |
| weight_decay=0.0, | |
| xpu_backend=None, | |
| ) | |
| 12/19/2022 11:14:33 - INFO - datasets.info - Loading Dataset Infos from /root/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f | |
| 12/19/2022 11:14:33 - INFO - datasets.builder - Generating dataset common_voice_11_0 (/root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) | |
| Downloading and preparing dataset common_voice_11_0/sk to /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f... | |
| 12/19/2022 11:14:33 - INFO - datasets.builder - Dataset not on Hf google storage. Downloading and preparing it from source | |
| 12/19/2022 11:14:33 - INFO - datasets.download.download_manager - Downloading took 0.0 min | |
| 12/19/2022 11:14:33 - INFO - datasets.download.download_manager - Checksum Computation took 0.0 min | |
| 12/19/2022 11:14:34 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/train/sk_train_0.tar not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpfcxe8xjf | |
| 12/19/2022 11:14:37 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/train/sk_train_0.tar in cache at /root/.cache/huggingface/datasets/downloads/dc60bece3babd1ec34c0d46bc16ae66faf1b9cbcbfb15aec62d81e4be6fbf6bc | |
| 12/19/2022 11:14:37 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/dc60bece3babd1ec34c0d46bc16ae66faf1b9cbcbfb15aec62d81e4be6fbf6bc | |
| 12/19/2022 11:14:38 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/dev/sk_dev_0.tar not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpzhl6hdfs | |
| 12/19/2022 11:14:41 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/dev/sk_dev_0.tar in cache at /root/.cache/huggingface/datasets/downloads/5f1420f0120f31ad1f506ec49fb795f4cd827b99fb4eafebac44ff0f63caaed2 | |
| 12/19/2022 11:14:41 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/5f1420f0120f31ad1f506ec49fb795f4cd827b99fb4eafebac44ff0f63caaed2 | |
| 12/19/2022 11:14:42 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/test/sk_test_0.tar not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpniv8gs8j | |
| 12/19/2022 11:14:45 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/test/sk_test_0.tar in cache at /root/.cache/huggingface/datasets/downloads/0a53aff953dd59d0697ffaa461246f461d99f9fd7c9ce4ff851f6a78c6893565 | |
| 12/19/2022 11:14:45 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/0a53aff953dd59d0697ffaa461246f461d99f9fd7c9ce4ff851f6a78c6893565 | |
| 12/19/2022 11:14:45 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/other/sk_other_0.tar not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpxrkfsvrp | |
| 12/19/2022 11:14:47 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/other/sk_other_0.tar in cache at /root/.cache/huggingface/datasets/downloads/00bc49d96ccd766e8321fb91ac512d72c63efe96cc1b1f11ff32365bc96044a3 | |
| 12/19/2022 11:14:47 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/00bc49d96ccd766e8321fb91ac512d72c63efe96cc1b1f11ff32365bc96044a3 | |
| 12/19/2022 11:14:48 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/invalidated/sk_invalidated_0.tar not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmp6pi3yjxe | |
| 12/19/2022 11:14:50 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/audio/sk/invalidated/sk_invalidated_0.tar in cache at /root/.cache/huggingface/datasets/downloads/5b62d26cd68c7d9f66ad5c933b865db14e3db1aaaaf89e4968618835b5fed105 | |
| 12/19/2022 11:14:50 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/5b62d26cd68c7d9f66ad5c933b865db14e3db1aaaaf89e4968618835b5fed105 | |
| 12/19/2022 11:14:50 - INFO - datasets.download.download_manager - Downloading took 0.0 min | |
| 12/19/2022 11:14:51 - INFO - datasets.download.download_manager - Checksum Computation took 0.0 min | |
| 12/19/2022 11:14:51 - INFO - datasets.utils.py_utils - Spawning 5 processes for 5 objects in slices of [1, 1, 1, 1, 1] | |
| 12/19/2022 11:14:52 - INFO - datasets.utils.py_utils - Finished 5 processes | |
| 12/19/2022 11:14:52 - INFO - datasets.utils.py_utils - Unpacked 5 objects | |
| 12/19/2022 11:14:53 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/train.tsv not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpbsn_3w9z | |
| 12/19/2022 11:14:54 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/train.tsv in cache at /root/.cache/huggingface/datasets/downloads/84e9d6c3445faa56fe7f4baf001b75e1e9f81ffc486f049ffabe6202138831e2 | |
| 12/19/2022 11:14:54 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/84e9d6c3445faa56fe7f4baf001b75e1e9f81ffc486f049ffabe6202138831e2 | |
| 12/19/2022 11:14:55 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/dev.tsv not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpfyyx15dx | |
| 12/19/2022 11:14:56 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/dev.tsv in cache at /root/.cache/huggingface/datasets/downloads/269bfb57a2f9de09c6113bcc84a9b36d8df7c58611682c76c31d5314c7f8f689 | |
| 12/19/2022 11:14:56 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/269bfb57a2f9de09c6113bcc84a9b36d8df7c58611682c76c31d5314c7f8f689 | |
| 12/19/2022 11:14:57 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/test.tsv not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmp727ax739 | |
| 12/19/2022 11:14:58 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/test.tsv in cache at /root/.cache/huggingface/datasets/downloads/35cee125ac2f288e3813dfc999980ba2cfeffe6ac77770e4a2e9c95c72a15392 | |
| 12/19/2022 11:14:58 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/35cee125ac2f288e3813dfc999980ba2cfeffe6ac77770e4a2e9c95c72a15392 | |
| 12/19/2022 11:14:59 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/other.tsv not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpf04skevi | |
| 12/19/2022 11:15:00 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/other.tsv in cache at /root/.cache/huggingface/datasets/downloads/bda4e520dc25d592f67db8bc3080e6a86eaf9fafccfead8c9e1e43c244420aee | |
| 12/19/2022 11:15:00 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/bda4e520dc25d592f67db8bc3080e6a86eaf9fafccfead8c9e1e43c244420aee | |
| 12/19/2022 11:15:01 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/invalidated.tsv not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/tmpyvkivypo | |
| 12/19/2022 11:15:02 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/mozilla-foundation/common_voice_11_0/resolve/streaming/transcript/sk/invalidated.tsv in cache at /root/.cache/huggingface/datasets/downloads/cdb07f76580d10a78d2bc3374ef15f6c6dc93470f67c36007db980722d6606a8 | |
| 12/19/2022 11:15:02 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/cdb07f76580d10a78d2bc3374ef15f6c6dc93470f67c36007db980722d6606a8 | |
| 12/19/2022 11:15:02 - INFO - datasets.download.download_manager - Downloading took 0.0 min | |
| 12/19/2022 11:15:02 - INFO - datasets.download.download_manager - Checksum Computation took 0.0 min | |
| 12/19/2022 11:15:02 - INFO - datasets.utils.py_utils - Spawning 5 processes for 5 objects in slices of [1, 1, 1, 1, 1] | |
| 12/19/2022 11:15:03 - INFO - datasets.utils.py_utils - Finished 5 processes | |
| 12/19/2022 11:15:03 - INFO - datasets.utils.py_utils - Unpacked 5 objects | |
| 12/19/2022 11:15:03 - INFO - datasets.utils.info_utils - Unable to verify checksums. | |
| 12/19/2022 11:15:03 - INFO - datasets.builder - Generating train split | |
| 12/19/2022 11:15:04 - INFO - datasets.builder - Generating validation split | |
| 12/19/2022 11:15:05 - INFO - datasets.builder - Generating test split | |
| 12/19/2022 11:15:06 - INFO - datasets.builder - Generating other split | |
| 12/19/2022 11:15:06 - INFO - datasets.builder - Generating invalidated split | |
| 12/19/2022 11:15:07 - INFO - datasets.utils.info_utils - Unable to verify splits sizes. | |
| Dataset common_voice_11_0 downloaded and prepared to /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f. Subsequent calls will reuse this data. | |
| 12/19/2022 11:15:10 - INFO - datasets.info - Loading Dataset Infos from /root/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f | |
| 12/19/2022 11:15:10 - INFO - datasets.builder - Overwrite dataset info from restored data version. | |
| 12/19/2022 11:15:10 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f | |
| 12/19/2022 11:15:10 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) | |
| 12/19/2022 11:15:10 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f | |
| 12/19/2022 11:15:13 - INFO - datasets.info - Loading Dataset Infos from /root/.cache/huggingface/modules/datasets_modules/datasets/mozilla-foundation--common_voice_11_0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f | |
| 12/19/2022 11:15:13 - INFO - datasets.builder - Overwrite dataset info from restored data version. | |
| 12/19/2022 11:15:13 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f | |
| 12/19/2022 11:15:13 - WARNING - datasets.builder - Found cached dataset common_voice_11_0 (/root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f) | |
| 12/19/2022 11:15:13 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f | |
| 12/19/2022 11:15:29 - INFO - datasets.arrow_dataset - Caching processed dataset at /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f/cache-1198445f23fbce84.arrow | |
| 12/19/2022 11:32:01 - INFO - datasets.arrow_dataset - Caching processed dataset at /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f/cache-0b88e3fb4a4fb48f.arrow | |
| 12/19/2022 11:43:14 - INFO - datasets.arrow_dataset - Caching processed dataset at /root/.cache/huggingface/datasets/mozilla-foundation___common_voice_11_0/sk/11.0.0/f8e47235d9b4e68fa24ed71d63266a02018ccf7194b2a8c9c598a5f3ab304d9f/cache-843405339f5029e5.arrow | |
| 12/19/2022 11:53:27 - WARNING - huggingface_hub.repository - /usr/src/app/models/whisper-medium-sk-cv11/./ is already a clone of https://huggingface.co/mikr/whisper-medium-sk-cv11. Make sure you pull the latest changes with `repo.git_pull()`. | |
| [2022-12-19 11:53:32,051] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed info: version=0.8.0+a25c31b6, git-hash=a25c31b6, git-branch=master | |
| [2022-12-19 11:53:33,693] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Flops Profiler Enabled: False | |
| Adam Optimizer #0 is created with AVX2 arithmetic capability. | |
| Config: alpha=0.000010, betas=(0.900000, 0.999000), weight_decay=0.000000, adam_w=1 | |
| [2022-12-19 11:53:36,545] [INFO] [logging.py:68:log_dist] [Rank 0] Using DeepSpeed Optimizer param name adamw as basic optimizer | |
| [2022-12-19 11:53:36,707] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam | |
| [2022-12-19 11:53:36,707] [INFO] [utils.py:52:is_zero_supported_optimizer] Checking ZeRO support for optimizer=DeepSpeedCPUAdam type=<class 'deepspeed.ops.adam.cpu_adam.DeepSpeedCPUAdam'> | |
| [2022-12-19 11:53:36,708] [INFO] [logging.py:68:log_dist] [Rank 0] Creating fp16 ZeRO stage 2 optimizer | |
| [2022-12-19 11:53:36,708] [INFO] [stage_1_and_2.py:141:__init__] Reduce bucket size 200000000 | |
| [2022-12-19 11:53:36,708] [INFO] [stage_1_and_2.py:142:__init__] Allgather bucket size 200000000 | |
| [2022-12-19 11:53:36,708] [INFO] [stage_1_and_2.py:143:__init__] CPU Offload: True | |
| [2022-12-19 11:53:36,708] [INFO] [stage_1_and_2.py:144:__init__] Round robin gradient partitioning: False | |
| Rank: 0 partition count [1] and sizes[(763857920, False)] | |
| [2022-12-19 11:53:40,277] [INFO] [utils.py:831:see_memory_usage] Before initializing optimizer states | |
| [2022-12-19 11:53:40,278] [INFO] [utils.py:832:see_memory_usage] MA 1.52 GB Max_MA 1.52 GB CA 1.53 GB Max_CA 2 GB | |
| [2022-12-19 11:53:40,278] [INFO] [utils.py:840:see_memory_usage] CPU Virtual Memory: used = 191.59 GB, percent = 38.0% | |
| [2022-12-19 11:53:43,514] [INFO] [utils.py:831:see_memory_usage] After initializing optimizer states | |
| [2022-12-19 11:53:43,515] [INFO] [utils.py:832:see_memory_usage] MA 1.52 GB Max_MA 1.52 GB CA 1.53 GB Max_CA 2 GB | |
| [2022-12-19 11:53:43,515] [INFO] [utils.py:840:see_memory_usage] CPU Virtual Memory: used = 200.63 GB, percent = 39.8% | |
| [2022-12-19 11:53:43,515] [INFO] [stage_1_and_2.py:527:__init__] optimizer state initialized | |
| [2022-12-19 11:53:43,579] [INFO] [utils.py:831:see_memory_usage] After initializing ZeRO optimizer | |
| [2022-12-19 11:53:43,580] [INFO] [utils.py:832:see_memory_usage] MA 1.52 GB Max_MA 1.52 GB CA 1.53 GB Max_CA 2 GB | |
| [2022-12-19 11:53:43,580] [INFO] [utils.py:840:see_memory_usage] CPU Virtual Memory: used = 200.59 GB, percent = 39.8% | |
| [2022-12-19 11:53:43,615] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed Final Optimizer = adamw | |
| [2022-12-19 11:53:43,616] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed using configured LR scheduler = WarmupDecayLR | |
| [2022-12-19 11:53:43,616] [INFO] [logging.py:68:log_dist] [Rank 0] DeepSpeed LR Scheduler = <deepspeed.runtime.lr_schedules.WarmupDecayLR object at 0x7f50d005f670> | |
| [2022-12-19 11:53:43,616] [INFO] [logging.py:68:log_dist] [Rank 0] step=0, skipped=0, lr=[1e-05], mom=[[0.9, 0.999]] | |
| [2022-12-19 11:53:43,618] [INFO] [config.py:1008:print] DeepSpeedEngine configuration: | |
| [2022-12-19 11:53:43,618] [INFO] [config.py:1012:print] activation_checkpointing_config { | |
| "partition_activations": false, | |
| "contiguous_memory_optimization": false, | |
| "cpu_checkpointing": false, | |
| "number_checkpoints": null, | |
| "synchronize_checkpoint_boundary": false, | |
| "profile": false | |
| } | |
| [2022-12-19 11:53:43,618] [INFO] [config.py:1012:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} | |
| [2022-12-19 11:53:43,618] [INFO] [config.py:1012:print] amp_enabled .................. False | |
| [2022-12-19 11:53:43,618] [INFO] [config.py:1012:print] amp_params ................... False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] autotuning_config ............ { | |
| "enabled": false, | |
| "start_step": null, | |
| "end_step": null, | |
| "metric_path": null, | |
| "arg_mappings": null, | |
| "metric": "throughput", | |
| "model_info": null, | |
| "results_dir": "autotuning_results", | |
| "exps_dir": "autotuning_exps", | |
| "overwrite": true, | |
| "fast": true, | |
| "start_profile_step": 3, | |
| "end_profile_step": 5, | |
| "tuner_type": "gridsearch", | |
| "tuner_early_stopping": 5, | |
| "tuner_num_trials": 50, | |
| "model_info_path": null, | |
| "mp_size": 1, | |
| "max_train_batch_size": null, | |
| "min_train_batch_size": 1, | |
| "max_train_micro_batch_size_per_gpu": 1.024000e+03, | |
| "min_train_micro_batch_size_per_gpu": 1, | |
| "num_tuning_micro_batch_sizes": 3 | |
| } | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] bfloat16_enabled ............. False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] checkpoint_parallel_write_pipeline False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] checkpoint_tag_validation_enabled True | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] checkpoint_tag_validation_fail False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] comms_config ................. <deepspeed.comm.config.DeepSpeedCommsConfig object at 0x7f50d005f220> | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] communication_data_type ...... None | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] curriculum_enabled_legacy .... False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] curriculum_params_legacy ..... False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}} | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] data_efficiency_enabled ...... False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] dataloader_drop_last ......... False | |
| [2022-12-19 11:53:43,619] [INFO] [config.py:1012:print] disable_allgather ............ False | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] dump_state ................... False | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] dynamic_loss_scale_args ...... {'init_scale': 65536, 'scale_window': 1000, 'delayed_shift': 2, 'min_scale': 1} | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_enabled ........... False | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_gas_boundary_resolution 1 | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_layer_name ........ bert.encoder.layer | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_layer_num ......... 0 | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_max_iter .......... 100 | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_stability ......... 1e-06 | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_tol ............... 0.01 | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] eigenvalue_verbose ........... False | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] elasticity_enabled ........... False | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] flops_profiler_config ........ { | |
| "enabled": false, | |
| "profile_step": 1, | |
| "module_depth": -1, | |
| "top_modules": 1, | |
| "detailed": true, | |
| "output_file": null | |
| } | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] fp16_auto_cast ............... False | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] fp16_enabled ................. True | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] fp16_master_weights_and_gradients False | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] global_rank .................. 0 | |
| [2022-12-19 11:53:43,620] [INFO] [config.py:1012:print] grad_accum_dtype ............. None | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] gradient_accumulation_steps .. 1 | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] gradient_clipping ............ 1.0 | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] gradient_predivide_factor .... 1.0 | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] initial_dynamic_scale ........ 65536 | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] load_universal_checkpoint .... False | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] loss_scale ................... 0 | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] memory_breakdown ............. False | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] monitor_config ............... <deepspeed.monitor.config.DeepSpeedMonitorConfig object at 0x7f50d005f520> | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] nebula_config ................ { | |
| "enabled": false, | |
| "persistent_storage_path": null, | |
| "persistent_time_interval": 100, | |
| "num_of_version_in_retention": 2, | |
| "enable_nebula_load": true, | |
| "load_path": null | |
| } | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] optimizer_legacy_fusion ...... False | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] optimizer_name ............... adamw | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] optimizer_params ............. {'lr': 1e-05, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0} | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0} | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] pld_enabled .................. False | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] pld_params ................... False | |
| [2022-12-19 11:53:43,621] [INFO] [config.py:1012:print] prescale_gradients ........... False | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] scheduler_name ............... WarmupDecayLR | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] scheduler_params ............. {'last_batch_iteration': -1, 'total_num_steps': 5000, 'warmup_min_lr': 0, 'warmup_max_lr': 1e-05, 'warmup_num_steps': 500} | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] sparse_attention ............. None | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] sparse_gradients_enabled ..... False | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] steps_per_print .............. 10 | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] train_batch_size ............. 64 | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] train_micro_batch_size_per_gpu 64 | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] use_node_local_storage ....... False | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] wall_clock_breakdown ......... False | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] world_size ................... 1 | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] zero_allow_untested_optimizer False | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=200000000 allgather_partitions=True allgather_bucket_size=200000000 overlap_comm=True load_from_fp32_weights=True elastic_checkpoint=False offload_param=None offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='cpu', nvme_path=None, buffer_count=4, pin_memory=True, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] zero_enabled ................. True | |
| [2022-12-19 11:53:43,622] [INFO] [config.py:1012:print] zero_optimization_stage ...... 2 | |
| [2022-12-19 11:53:43,623] [INFO] [config.py:997:print_user_config] json = { | |
| "fp16": { | |
| "enabled": true, | |
| "loss_scale": 0, | |
| "loss_scale_window": 1000, | |
| "initial_scale_power": 16, | |
| "hysteresis": 2, | |
| "min_loss_scale": 1 | |
| }, | |
| "optimizer": { | |
| "type": "AdamW", | |
| "params": { | |
| "lr": 1e-05, | |
| "betas": [0.9, 0.999], | |
| "eps": 1e-08, | |
| "weight_decay": 0.0 | |
| } | |
| }, | |
| "scheduler": { | |
| "type": "WarmupDecayLR", | |
| "params": { | |
| "last_batch_iteration": -1, | |
| "total_num_steps": 5.000000e+03, | |
| "warmup_min_lr": 0, | |
| "warmup_max_lr": 1e-05, | |
| "warmup_num_steps": 500 | |
| } | |
| }, | |
| "zero_optimization": { | |
| "stage": 2, | |
| "offload_optimizer": { | |
| "device": "cpu", | |
| "pin_memory": true | |
| }, | |
| "allgather_partitions": true, | |
| "allgather_bucket_size": 2.000000e+08, | |
| "overlap_comm": true, | |
| "reduce_scatter": true, | |
| "reduce_bucket_size": 2.000000e+08, | |
| "contiguous_gradients": true | |
| }, | |
| "gradient_accumulation_steps": 1, | |
| "gradient_clipping": 1.0, | |
| "train_batch_size": 64, | |
| "train_micro_batch_size_per_gpu": 64 | |
| } | |
| [2022-12-19 11:53:57,008] [INFO] [stage_1_and_2.py:1767:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 65536, reducing to 65536 | |
| [2022-12-19 11:54:08,740] [INFO] [stage_1_and_2.py:1767:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 65536, reducing to 32768.0 | |
| [2022-12-19 11:54:20,566] [INFO] [stage_1_and_2.py:1767:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 32768.0, reducing to 16384.0 | |
| [2022-12-19 11:54:32,523] [INFO] [stage_1_and_2.py:1767:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 16384.0, reducing to 8192.0 | |
| [2022-12-19 11:55:49,396] [INFO] [logging.py:68:log_dist] [Rank 0] step=10, skipped=4, lr=[2.883141528559073e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 11:55:49,397] [INFO] [timer.py:196:stop] epoch=0/micro_step=10/global_step=10, RunningAvgSamplesPerSec=6.284724477029608, CurrSamplesPerSec=6.148180060714191, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 11:58:08,169] [INFO] [logging.py:68:log_dist] [Rank 0] step=20, skipped=4, lr=[4.461405575910259e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 11:58:08,171] [INFO] [timer.py:196:stop] epoch=0/micro_step=20/global_step=20, RunningAvgSamplesPerSec=5.884120194534651, CurrSamplesPerSec=5.054178494750393, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.8611, 'learning_rate': 4.898977360288234e-06, 'epoch': 0.36} | |
| [2022-12-19 12:00:49,857] [INFO] [logging.py:68:log_dist] [Rank 0] step=30, skipped=4, lr=[5.242641991936178e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:00:49,859] [INFO] [timer.py:196:stop] epoch=0/micro_step=30/global_step=30, RunningAvgSamplesPerSec=5.357395051686146, CurrSamplesPerSec=4.271363536074205, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:03:36,350] [INFO] [logging.py:68:log_dist] [Rank 0] step=40, skipped=4, lr=[5.766283057118146e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:03:36,352] [INFO] [timer.py:196:stop] epoch=0/micro_step=40/global_step=40, RunningAvgSamplesPerSec=5.089969895421541, CurrSamplesPerSec=4.327689824299493, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:06:23,404] [INFO] [logging.py:68:log_dist] [Rank 0] step=50, skipped=4, lr=[6.160712527409633e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:06:23,405] [INFO] [timer.py:196:stop] epoch=0/micro_step=50/global_step=50, RunningAvgSamplesPerSec=4.94469846197508, CurrSamplesPerSec=4.742895867520358, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.3224, 'learning_rate': 6.160712527409633e-06, 'epoch': 0.71} | |
| [2022-12-19 12:08:49,940] [INFO] [logging.py:68:log_dist] [Rank 0] step=60, skipped=4, lr=[6.4772414076394205e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:08:49,942] [INFO] [timer.py:196:stop] epoch=0/micro_step=60/global_step=60, RunningAvgSamplesPerSec=4.9854128285166475, CurrSamplesPerSec=5.299728280676971, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:11:25,236] [INFO] [logging.py:68:log_dist] [Rank 0] step=70, skipped=4, lr=[6.741623406776245e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:11:25,238] [INFO] [timer.py:196:stop] epoch=0/micro_step=70/global_step=70, RunningAvgSamplesPerSec=4.962771426482941, CurrSamplesPerSec=4.826170613844314, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.2375, 'learning_rate': 6.85912902234906e-06, 'epoch': 1.07} | |
| [2022-12-19 12:14:02,605] [INFO] [logging.py:68:log_dist] [Rank 0] step=80, skipped=4, lr=[6.968634661590082e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:14:02,606] [INFO] [timer.py:196:stop] epoch=0/micro_step=80/global_step=80, RunningAvgSamplesPerSec=4.937271773600281, CurrSamplesPerSec=4.78181846456958, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:16:33,552] [INFO] [logging.py:68:log_dist] [Rank 0] step=90, skipped=4, lr=[7.1675433522258775e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:16:33,554] [INFO] [timer.py:196:stop] epoch=0/micro_step=90/global_step=90, RunningAvgSamplesPerSec=4.945269383765002, CurrSamplesPerSec=5.042845013770653, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:19:09,093] [INFO] [logging.py:68:log_dist] [Rank 0] step=100, skipped=4, lr=[7.344547104469332e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:19:09,095] [INFO] [timer.py:196:stop] epoch=0/micro_step=100/global_step=100, RunningAvgSamplesPerSec=4.933974241037233, CurrSamplesPerSec=4.701477254589951, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.1304, 'learning_rate': 7.344547104469332e-06, 'epoch': 1.43} | |
| [2022-12-19 12:21:47,390] [INFO] [logging.py:68:log_dist] [Rank 0] step=110, skipped=4, lr=[7.503995457567235e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:21:47,392] [INFO] [timer.py:196:stop] epoch=0/micro_step=110/global_step=110, RunningAvgSamplesPerSec=4.91591709126971, CurrSamplesPerSec=4.688850220105803, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:24:23,449] [INFO] [logging.py:68:log_dist] [Rank 0] step=120, skipped=4, lr=[7.649058662787184e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:24:23,450] [INFO] [timer.py:196:stop] epoch=0/micro_step=120/global_step=120, RunningAvgSamplesPerSec=4.9070530462655775, CurrSamplesPerSec=5.1081928187039445, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.1404, 'learning_rate': 7.716963756434345e-06, 'epoch': 1.79} | |
| [2022-12-19 12:27:01,797] [INFO] [logging.py:68:log_dist] [Rank 0] step=130, skipped=4, lr=[7.782118888847307e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:27:01,799] [INFO] [timer.py:196:stop] epoch=0/micro_step=130/global_step=130, RunningAvgSamplesPerSec=4.8920194677244355, CurrSamplesPerSec=4.416138360200384, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:29:44,263] [INFO] [logging.py:68:log_dist] [Rank 0] step=140, skipped=4, lr=[7.905011559752758e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:29:44,264] [INFO] [timer.py:196:stop] epoch=0/micro_step=140/global_step=140, RunningAvgSamplesPerSec=4.868397273066603, CurrSamplesPerSec=4.920148346706677, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:32:21,595] [INFO] [logging.py:68:log_dist] [Rank 0] step=150, skipped=4, lr=[8.019180844200955e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:32:21,597] [INFO] [timer.py:196:stop] epoch=0/micro_step=150/global_step=150, RunningAvgSamplesPerSec=4.861667948366876, CurrSamplesPerSec=4.915494797679944, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.1005, 'learning_rate': 8.019180844200955e-06, 'epoch': 2.14} | |
| [2022-12-19 12:35:01,285] [INFO] [logging.py:68:log_dist] [Rank 0] step=160, skipped=4, lr=[8.125783520495252e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:35:01,287] [INFO] [timer.py:196:stop] epoch=0/micro_step=160/global_step=160, RunningAvgSamplesPerSec=4.850342817067083, CurrSamplesPerSec=4.7749939204247775, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:37:40,486] [INFO] [logging.py:68:log_dist] [Rank 0] step=170, skipped=4, lr=[8.225760510392298e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:37:40,488] [INFO] [timer.py:196:stop] epoch=0/micro_step=170/global_step=170, RunningAvgSamplesPerSec=4.84217822772632, CurrSamplesPerSec=4.639845609652397, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0496, 'learning_rate': 8.27351214279797e-06, 'epoch': 2.5} | |
| [2022-12-19 12:40:13,823] [INFO] [logging.py:68:log_dist] [Rank 0] step=180, skipped=4, lr=[8.31988745412743e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:40:13,825] [INFO] [timer.py:196:stop] epoch=0/micro_step=180/global_step=180, RunningAvgSamplesPerSec=4.846620755543373, CurrSamplesPerSec=4.94334514558199, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:42:51,582] [INFO] [logging.py:68:log_dist] [Rank 0] step=190, skipped=4, lr=[8.408811289387583e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:42:51,583] [INFO] [timer.py:196:stop] epoch=0/micro_step=190/global_step=190, RunningAvgSamplesPerSec=4.842148589841127, CurrSamplesPerSec=4.636333563213749, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:45:28,742] [INFO] [logging.py:68:log_dist] [Rank 0] step=200, skipped=4, lr=[8.49307723936858e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:45:28,744] [INFO] [timer.py:196:stop] epoch=0/micro_step=200/global_step=200, RunningAvgSamplesPerSec=4.838832903723075, CurrSamplesPerSec=4.873535141481062, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0468, 'learning_rate': 8.49307723936858e-06, 'epoch': 2.86} | |
| [2022-12-19 12:48:11,320] [INFO] [logging.py:68:log_dist] [Rank 0] step=210, skipped=4, lr=[8.573149077803088e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:48:11,322] [INFO] [timer.py:196:stop] epoch=0/micro_step=210/global_step=210, RunningAvgSamplesPerSec=4.825149349910002, CurrSamplesPerSec=4.864172355960737, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:50:45,911] [INFO] [logging.py:68:log_dist] [Rank 0] step=220, skipped=4, lr=[8.64942458567722e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:50:45,912] [INFO] [timer.py:196:stop] epoch=0/micro_step=220/global_step=220, RunningAvgSamplesPerSec=4.82800487670514, CurrSamplesPerSec=5.104461169974322, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0305, 'learning_rate': 8.686247975778677e-06, 'epoch': 3.21} | |
| [2022-12-19 12:53:15,841] [INFO] [logging.py:68:log_dist] [Rank 0] step=230, skipped=4, lr=[8.722247506883805e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:53:15,842] [INFO] [timer.py:196:stop] epoch=0/micro_step=230/global_step=230, RunningAvgSamplesPerSec=4.8362180032202176, CurrSamplesPerSec=4.783041219701146, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:55:55,413] [INFO] [logging.py:68:log_dist] [Rank 0] step=240, skipped=4, lr=[8.79191691333329e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:55:55,415] [INFO] [timer.py:196:stop] epoch=0/micro_step=240/global_step=240, RunningAvgSamplesPerSec=4.830415934243562, CurrSamplesPerSec=4.67587043487864, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 12:58:29,579] [INFO] [logging.py:68:log_dist] [Rank 0] step=250, skipped=4, lr=[8.858694625217149e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 12:58:29,581] [INFO] [timer.py:196:stop] epoch=0/micro_step=250/global_step=250, RunningAvgSamplesPerSec=4.833466213517386, CurrSamplesPerSec=5.486877476242189, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0245, 'learning_rate': 8.858694625217149e-06, 'epoch': 3.57} | |
| [2022-12-19 13:00:57,038] [INFO] [logging.py:68:log_dist] [Rank 0] step=260, skipped=4, lr=[8.922811151820517e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:00:57,039] [INFO] [timer.py:196:stop] epoch=0/micro_step=260/global_step=260, RunningAvgSamplesPerSec=4.845110965147443, CurrSamplesPerSec=4.9921115157357105, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:03:33,670] [INFO] [logging.py:68:log_dist] [Rank 0] step=270, skipped=4, lr=[8.984470493319244e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:03:33,672] [INFO] [timer.py:196:stop] epoch=0/micro_step=270/global_step=270, RunningAvgSamplesPerSec=4.8429444681448635, CurrSamplesPerSec=4.802661310533987, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0204, 'learning_rate': 9.014436199608479e-06, 'epoch': 3.93} | |
| [2022-12-19 13:06:10,407] [INFO] [logging.py:68:log_dist] [Rank 0] step=280, skipped=4, lr=[9.043854055968706e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:06:10,409] [INFO] [timer.py:196:stop] epoch=0/micro_step=280/global_step=280, RunningAvgSamplesPerSec=4.841414423642298, CurrSamplesPerSec=4.812496898464995, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:08:35,681] [INFO] [logging.py:68:log_dist] [Rank 0] step=290, skipped=4, lr=[9.10112387015335e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:08:35,682] [INFO] [timer.py:196:stop] epoch=0/micro_step=290/global_step=290, RunningAvgSamplesPerSec=4.855070183701984, CurrSamplesPerSec=5.301100474873604, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:11:04,702] [INFO] [logging.py:68:log_dist] [Rank 0] step=300, skipped=4, lr=[9.156425255148058e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:11:04,704] [INFO] [timer.py:196:stop] epoch=0/micro_step=300/global_step=300, RunningAvgSamplesPerSec=4.862483705117125, CurrSamplesPerSec=4.96419714698448, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0161, 'learning_rate': 9.156425255148058e-06, 'epoch': 4.29} | |
| [2022-12-19 13:13:41,785] [INFO] [logging.py:68:log_dist] [Rank 0] step=310, skipped=4, lr=[9.209889040960644e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:13:41,786] [INFO] [timer.py:196:stop] epoch=0/micro_step=310/global_step=310, RunningAvgSamplesPerSec=4.859764335974417, CurrSamplesPerSec=4.878675089060409, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:16:17,620] [INFO] [logging.py:68:log_dist] [Rank 0] step=320, skipped=4, lr=[9.261633432763397e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:16:17,621] [INFO] [timer.py:196:stop] epoch=0/micro_step=320/global_step=320, RunningAvgSamplesPerSec=4.858997002509444, CurrSamplesPerSec=4.80213043314379, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0129, 'learning_rate': 9.28689473531776e-06, 'epoch': 4.64} | |
| [2022-12-19 13:18:54,230] [INFO] [logging.py:68:log_dist] [Rank 0] step=330, skipped=4, lr=[9.311765584761373e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:18:54,231] [INFO] [timer.py:196:stop] epoch=0/micro_step=330/global_step=330, RunningAvgSamplesPerSec=4.857019464893833, CurrSamplesPerSec=4.742613643782026, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:21:32,500] [INFO] [logging.py:68:log_dist] [Rank 0] step=340, skipped=4, lr=[9.360382936198493e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:21:32,502] [INFO] [timer.py:196:stop] epoch=0/micro_step=340/global_step=340, RunningAvgSamplesPerSec=4.853928759897602, CurrSamplesPerSec=4.8359388617015355, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:24:08,047] [INFO] [logging.py:68:log_dist] [Rank 0] step=350, skipped=4, lr=[9.407574351377137e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:24:08,049] [INFO] [timer.py:196:stop] epoch=0/micro_step=350/global_step=350, RunningAvgSamplesPerSec=4.853820410388651, CurrSamplesPerSec=4.898408063422721, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.013, 'learning_rate': 9.407574351377137e-06, 'epoch': 5.0} | |
| [2022-12-19 13:26:45,328] [INFO] [logging.py:68:log_dist] [Rank 0] step=360, skipped=4, lr=[9.45342109721062e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:26:45,329] [INFO] [timer.py:196:stop] epoch=0/micro_step=360/global_step=360, RunningAvgSamplesPerSec=4.85178775805814, CurrSamplesPerSec=4.948161599842712, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:29:20,026] [INFO] [logging.py:68:log_dist] [Rank 0] step=370, skipped=4, lr=[9.497997685324628e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:29:20,027] [INFO] [timer.py:196:stop] epoch=0/micro_step=370/global_step=370, RunningAvgSamplesPerSec=4.852030422658968, CurrSamplesPerSec=4.587538166471939, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0084, 'learning_rate': 9.519831289296397e-06, 'epoch': 5.36} | |
| [2022-12-19 13:31:55,844] [INFO] [logging.py:68:log_dist] [Rank 0] step=380, skipped=4, lr=[9.541372600623587e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:31:55,845] [INFO] [timer.py:196:stop] epoch=0/micro_step=380/global_step=380, RunningAvgSamplesPerSec=4.851833732350051, CurrSamplesPerSec=4.7891279033152125, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:34:33,105] [INFO] [logging.py:68:log_dist] [Rank 0] step=390, skipped=4, lr=[9.583608934209288e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:34:33,106] [INFO] [timer.py:196:stop] epoch=0/micro_step=390/global_step=390, RunningAvgSamplesPerSec=4.850083370123769, CurrSamplesPerSec=4.644631745875583, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:37:12,002] [INFO] [logging.py:68:log_dist] [Rank 0] step=400, skipped=4, lr=[9.624764935335318e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:37:12,004] [INFO] [timer.py:196:stop] epoch=0/micro_step=400/global_step=400, RunningAvgSamplesPerSec=4.846477731334026, CurrSamplesPerSec=4.710384798486559, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0063, 'learning_rate': 9.624764935335318e-06, 'epoch': 5.71} | |
| [2022-12-19 13:39:47,883] [INFO] [logging.py:68:log_dist] [Rank 0] step=410, skipped=4, lr=[9.664894494516345e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:39:47,885] [INFO] [timer.py:196:stop] epoch=0/micro_step=410/global_step=410, RunningAvgSamplesPerSec=4.846520980004455, CurrSamplesPerSec=4.8562435318187935, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:42:18,602] [INFO] [logging.py:68:log_dist] [Rank 0] step=420, skipped=4, lr=[9.704047567846437e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:42:18,603] [INFO] [timer.py:196:stop] epoch=0/micro_step=420/global_step=420, RunningAvgSamplesPerSec=4.850877865185188, CurrSamplesPerSec=5.319254205108675, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0091, 'learning_rate': 9.723272550712454e-06, 'epoch': 6.07} | |
| [2022-12-19 13:44:50,834] [INFO] [logging.py:68:log_dist] [Rank 0] step=430, skipped=4, lr=[9.742270550908135e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:44:50,835] [INFO] [timer.py:196:stop] epoch=0/micro_step=430/global_step=430, RunningAvgSamplesPerSec=4.8535268333430155, CurrSamplesPerSec=4.883467831694119, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:47:26,590] [INFO] [logging.py:68:log_dist] [Rank 0] step=440, skipped=4, lr=[9.779606609292176e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:47:26,592] [INFO] [timer.py:196:stop] epoch=0/micro_step=440/global_step=440, RunningAvgSamplesPerSec=4.853048414412156, CurrSamplesPerSec=4.76805433357737, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:50:03,822] [INFO] [logging.py:68:log_dist] [Rank 0] step=450, skipped=4, lr=[9.816095971633122e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:50:03,823] [INFO] [timer.py:196:stop] epoch=0/micro_step=450/global_step=450, RunningAvgSamplesPerSec=4.851727248421968, CurrSamplesPerSec=4.862512962573305, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.008, 'learning_rate': 9.816095971633122e-06, 'epoch': 6.43} | |
| [2022-12-19 13:52:37,209] [INFO] [logging.py:68:log_dist] [Rank 0] step=460, skipped=4, lr=[9.851776190149156e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:52:37,211] [INFO] [timer.py:196:stop] epoch=0/micro_step=460/global_step=460, RunningAvgSamplesPerSec=4.854686255524522, CurrSamplesPerSec=5.006905800888788, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:54:58,883] [INFO] [logging.py:68:log_dist] [Rank 0] step=470, skipped=4, lr=[9.886682372916766e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:54:58,884] [INFO] [timer.py:196:stop] epoch=0/micro_step=470/global_step=470, RunningAvgSamplesPerSec=4.866808478943239, CurrSamplesPerSec=5.55124573746386, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0082, 'learning_rate': 9.90385555539545e-06, 'epoch': 6.79} | |
| [2022-12-19 13:57:24,670] [INFO] [logging.py:68:log_dist] [Rank 0] step=480, skipped=4, lr=[9.92084739148192e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:57:24,671] [INFO] [timer.py:196:stop] epoch=0/micro_step=480/global_step=480, RunningAvgSamplesPerSec=4.8735694202012425, CurrSamplesPerSec=5.105844323967845, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 13:59:51,505] [INFO] [logging.py:68:log_dist] [Rank 0] step=490, skipped=4, lr=[9.954302066885107e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 13:59:51,507] [INFO] [timer.py:196:stop] epoch=0/micro_step=490/global_step=490, RunningAvgSamplesPerSec=4.879328693961105, CurrSamplesPerSec=5.361670640483412, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:02:17,782] [INFO] [logging.py:68:log_dist] [Rank 0] step=500, skipped=4, lr=[9.987075336738768e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:02:17,783] [INFO] [timer.py:196:stop] epoch=0/micro_step=500/global_step=500, RunningAvgSamplesPerSec=4.887102593599529, CurrSamplesPerSec=5.400279695125675, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0085, 'learning_rate': 9.987075336738768e-06, 'epoch': 7.14} | |
| [2022-12-19 14:04:38,368] [INFO] [logging.py:68:log_dist] [Rank 0] step=510, skipped=4, lr=[9.98888888888889e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:04:38,369] [INFO] [timer.py:196:stop] epoch=0/micro_step=510/global_step=510, RunningAvgSamplesPerSec=4.898997366676477, CurrSamplesPerSec=5.396514227837796, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:07:03,835] [INFO] [logging.py:68:log_dist] [Rank 0] step=520, skipped=4, lr=[9.966666666666667e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:07:03,837] [INFO] [timer.py:196:stop] epoch=0/micro_step=520/global_step=520, RunningAvgSamplesPerSec=4.905130946159965, CurrSamplesPerSec=4.988483885181161, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0063, 'learning_rate': 9.955555555555556e-06, 'epoch': 7.5} | |
| [2022-12-19 14:09:35,891] [INFO] [logging.py:68:log_dist] [Rank 0] step=530, skipped=4, lr=[9.944444444444445e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:09:35,893] [INFO] [timer.py:196:stop] epoch=0/micro_step=530/global_step=530, RunningAvgSamplesPerSec=4.906098705272462, CurrSamplesPerSec=4.758214160450357, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:12:11,852] [INFO] [logging.py:68:log_dist] [Rank 0] step=540, skipped=4, lr=[9.922222222222222e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:12:11,853] [INFO] [timer.py:196:stop] epoch=0/micro_step=540/global_step=540, RunningAvgSamplesPerSec=4.904466433323427, CurrSamplesPerSec=4.969838803352754, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:14:47,792] [INFO] [logging.py:68:log_dist] [Rank 0] step=550, skipped=4, lr=[9.9e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:14:47,794] [INFO] [timer.py:196:stop] epoch=0/micro_step=550/global_step=550, RunningAvgSamplesPerSec=4.902692656494618, CurrSamplesPerSec=4.781843337754648, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0084, 'learning_rate': 9.9e-06, 'epoch': 7.86} | |
| [2022-12-19 14:17:23,420] [INFO] [logging.py:68:log_dist] [Rank 0] step=560, skipped=4, lr=[9.877777777777778e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:17:23,421] [INFO] [timer.py:196:stop] epoch=0/micro_step=560/global_step=560, RunningAvgSamplesPerSec=4.901205410548811, CurrSamplesPerSec=5.010666437943654, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:20:01,753] [INFO] [logging.py:68:log_dist] [Rank 0] step=570, skipped=4, lr=[9.855555555555555e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:20:01,754] [INFO] [timer.py:196:stop] epoch=0/micro_step=570/global_step=570, RunningAvgSamplesPerSec=4.898100972927672, CurrSamplesPerSec=4.759615844767654, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0049, 'learning_rate': 9.844444444444446e-06, 'epoch': 8.21} | |
| [2022-12-19 14:22:35,621] [INFO] [logging.py:68:log_dist] [Rank 0] step=580, skipped=4, lr=[9.833333333333333e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:22:35,622] [INFO] [timer.py:196:stop] epoch=0/micro_step=580/global_step=580, RunningAvgSamplesPerSec=4.898424414949789, CurrSamplesPerSec=5.1699776603263325, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:25:02,778] [INFO] [logging.py:68:log_dist] [Rank 0] step=590, skipped=4, lr=[9.811111111111112e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:25:02,779] [INFO] [timer.py:196:stop] epoch=0/micro_step=590/global_step=590, RunningAvgSamplesPerSec=4.902626294312846, CurrSamplesPerSec=5.092728847325434, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:27:29,246] [INFO] [logging.py:68:log_dist] [Rank 0] step=600, skipped=4, lr=[9.78888888888889e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:27:29,248] [INFO] [timer.py:196:stop] epoch=0/micro_step=600/global_step=600, RunningAvgSamplesPerSec=4.907046849888979, CurrSamplesPerSec=5.223062659736944, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0059, 'learning_rate': 9.78888888888889e-06, 'epoch': 8.57} | |
| [2022-12-19 14:29:57,726] [INFO] [logging.py:68:log_dist] [Rank 0] step=610, skipped=4, lr=[9.766666666666667e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:29:57,727] [INFO] [timer.py:196:stop] epoch=0/micro_step=610/global_step=610, RunningAvgSamplesPerSec=4.910471417559111, CurrSamplesPerSec=5.122817956349585, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:32:21,053] [INFO] [logging.py:68:log_dist] [Rank 0] step=620, skipped=4, lr=[9.744444444444445e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:32:21,054] [INFO] [timer.py:196:stop] epoch=0/micro_step=620/global_step=620, RunningAvgSamplesPerSec=4.917031539120992, CurrSamplesPerSec=5.2714502306701085, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0066, 'learning_rate': 9.733333333333334e-06, 'epoch': 8.93} | |
| [2022-12-19 14:34:53,075] [INFO] [logging.py:68:log_dist] [Rank 0] step=630, skipped=4, lr=[9.722222222222223e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:34:53,077] [INFO] [timer.py:196:stop] epoch=0/micro_step=630/global_step=630, RunningAvgSamplesPerSec=4.917835817961153, CurrSamplesPerSec=4.917546549724145, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:37:28,079] [INFO] [logging.py:68:log_dist] [Rank 0] step=640, skipped=4, lr=[9.7e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:37:28,080] [INFO] [timer.py:196:stop] epoch=0/micro_step=640/global_step=640, RunningAvgSamplesPerSec=4.917070836153263, CurrSamplesPerSec=4.88024030445332, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:39:52,985] [INFO] [logging.py:68:log_dist] [Rank 0] step=650, skipped=4, lr=[9.677777777777778e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:39:52,987] [INFO] [timer.py:196:stop] epoch=0/micro_step=650/global_step=650, RunningAvgSamplesPerSec=4.922261688223727, CurrSamplesPerSec=5.31764146920894, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0053, 'learning_rate': 9.677777777777778e-06, 'epoch': 9.29} | |
| [2022-12-19 14:42:20,988] [INFO] [logging.py:68:log_dist] [Rank 0] step=660, skipped=4, lr=[9.655555555555556e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:42:20,990] [INFO] [timer.py:196:stop] epoch=0/micro_step=660/global_step=660, RunningAvgSamplesPerSec=4.925255429831317, CurrSamplesPerSec=5.157295982637809, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:44:48,284] [INFO] [logging.py:68:log_dist] [Rank 0] step=670, skipped=4, lr=[9.633333333333335e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:44:48,286] [INFO] [timer.py:196:stop] epoch=0/micro_step=670/global_step=670, RunningAvgSamplesPerSec=4.928434904008725, CurrSamplesPerSec=5.061157079205904, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.004, 'learning_rate': 9.622222222222222e-06, 'epoch': 9.64} | |
| [2022-12-19 14:47:15,119] [INFO] [logging.py:68:log_dist] [Rank 0] step=680, skipped=4, lr=[9.611111111111112e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:47:15,121] [INFO] [timer.py:196:stop] epoch=0/micro_step=680/global_step=680, RunningAvgSamplesPerSec=4.932332719017831, CurrSamplesPerSec=5.07334611342772, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:49:42,987] [INFO] [logging.py:68:log_dist] [Rank 0] step=690, skipped=4, lr=[9.58888888888889e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:49:42,989] [INFO] [timer.py:196:stop] epoch=0/micro_step=690/global_step=690, RunningAvgSamplesPerSec=4.934952106280914, CurrSamplesPerSec=5.23162515983437, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:52:09,781] [INFO] [logging.py:68:log_dist] [Rank 0] step=700, skipped=4, lr=[9.566666666666668e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:52:09,782] [INFO] [timer.py:196:stop] epoch=0/micro_step=700/global_step=700, RunningAvgSamplesPerSec=4.9380357034046325, CurrSamplesPerSec=5.24687746476847, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0054, 'learning_rate': 9.566666666666668e-06, 'epoch': 10.0} | |
| [2022-12-19 14:54:39,366] [INFO] [logging.py:68:log_dist] [Rank 0] step=710, skipped=4, lr=[9.544444444444445e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:54:39,367] [INFO] [timer.py:196:stop] epoch=0/micro_step=710/global_step=710, RunningAvgSamplesPerSec=4.939692191956853, CurrSamplesPerSec=5.010938063895246, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 14:57:11,603] [INFO] [logging.py:68:log_dist] [Rank 0] step=720, skipped=4, lr=[9.522222222222223e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:57:11,604] [INFO] [timer.py:196:stop] epoch=0/micro_step=720/global_step=720, RunningAvgSamplesPerSec=4.9399408655745685, CurrSamplesPerSec=5.087045225116837, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0041, 'learning_rate': 9.511111111111112e-06, 'epoch': 10.36} | |
| [2022-12-19 14:59:38,451] [INFO] [logging.py:68:log_dist] [Rank 0] step=730, skipped=4, lr=[9.5e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 14:59:38,452] [INFO] [timer.py:196:stop] epoch=0/micro_step=730/global_step=730, RunningAvgSamplesPerSec=4.943429470946694, CurrSamplesPerSec=5.086961162941001, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:02:14,198] [INFO] [logging.py:68:log_dist] [Rank 0] step=740, skipped=4, lr=[9.47777777777778e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:02:14,200] [INFO] [timer.py:196:stop] epoch=0/micro_step=740/global_step=740, RunningAvgSamplesPerSec=4.941945164153034, CurrSamplesPerSec=4.616645396900982, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:04:50,347] [INFO] [logging.py:68:log_dist] [Rank 0] step=750, skipped=4, lr=[9.455555555555557e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:04:50,349] [INFO] [timer.py:196:stop] epoch=0/micro_step=750/global_step=750, RunningAvgSamplesPerSec=4.940280554893238, CurrSamplesPerSec=4.9899512607127114, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0042, 'learning_rate': 9.455555555555557e-06, 'epoch': 10.71} | |
| [2022-12-19 15:07:23,856] [INFO] [logging.py:68:log_dist] [Rank 0] step=760, skipped=4, lr=[9.433333333333335e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:07:23,857] [INFO] [timer.py:196:stop] epoch=0/micro_step=760/global_step=760, RunningAvgSamplesPerSec=4.939938762139419, CurrSamplesPerSec=5.072232465913488, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:09:55,140] [INFO] [logging.py:68:log_dist] [Rank 0] step=770, skipped=4, lr=[9.411111111111113e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:09:55,141] [INFO] [timer.py:196:stop] epoch=0/micro_step=770/global_step=770, RunningAvgSamplesPerSec=4.940492222311285, CurrSamplesPerSec=4.816172566434398, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0031, 'learning_rate': 9.4e-06, 'epoch': 11.07} | |
| [2022-12-19 15:12:30,313] [INFO] [logging.py:68:log_dist] [Rank 0] step=780, skipped=4, lr=[9.38888888888889e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:12:30,314] [INFO] [timer.py:196:stop] epoch=0/micro_step=780/global_step=780, RunningAvgSamplesPerSec=4.939305584796219, CurrSamplesPerSec=4.9126494042424165, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:15:04,911] [INFO] [logging.py:68:log_dist] [Rank 0] step=790, skipped=4, lr=[9.366666666666668e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:15:04,913] [INFO] [timer.py:196:stop] epoch=0/micro_step=790/global_step=790, RunningAvgSamplesPerSec=4.938492549839854, CurrSamplesPerSec=5.011928756665134, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:17:34,843] [INFO] [logging.py:68:log_dist] [Rank 0] step=800, skipped=4, lr=[9.344444444444446e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:17:34,844] [INFO] [timer.py:196:stop] epoch=0/micro_step=800/global_step=800, RunningAvgSamplesPerSec=4.939797377547636, CurrSamplesPerSec=4.823039931741798, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0042, 'learning_rate': 9.344444444444446e-06, 'epoch': 11.43} | |
| [2022-12-19 15:20:10,508] [INFO] [logging.py:68:log_dist] [Rank 0] step=810, skipped=4, lr=[9.322222222222223e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:20:10,509] [INFO] [timer.py:196:stop] epoch=0/micro_step=810/global_step=810, RunningAvgSamplesPerSec=4.938616706325122, CurrSamplesPerSec=4.615424885903679, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:22:46,952] [INFO] [logging.py:68:log_dist] [Rank 0] step=820, skipped=4, lr=[9.3e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:22:46,954] [INFO] [timer.py:196:stop] epoch=0/micro_step=820/global_step=820, RunningAvgSamplesPerSec=4.936606889293279, CurrSamplesPerSec=4.723105056028439, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0029, 'learning_rate': 9.28888888888889e-06, 'epoch': 11.79} | |
| [2022-12-19 15:25:15,818] [INFO] [logging.py:68:log_dist] [Rank 0] step=830, skipped=4, lr=[9.277777777777778e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:25:15,819] [INFO] [timer.py:196:stop] epoch=0/micro_step=830/global_step=830, RunningAvgSamplesPerSec=4.938464256304168, CurrSamplesPerSec=5.032372515852035, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:27:47,540] [INFO] [logging.py:68:log_dist] [Rank 0] step=840, skipped=4, lr=[9.255555555555556e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:27:47,542] [INFO] [timer.py:196:stop] epoch=0/micro_step=840/global_step=840, RunningAvgSamplesPerSec=4.938820182366459, CurrSamplesPerSec=5.013181412913762, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:30:22,235] [INFO] [logging.py:68:log_dist] [Rank 0] step=850, skipped=4, lr=[9.233333333333334e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:30:22,237] [INFO] [timer.py:196:stop] epoch=0/micro_step=850/global_step=850, RunningAvgSamplesPerSec=4.937928612579426, CurrSamplesPerSec=4.998566565455339, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0029, 'learning_rate': 9.233333333333334e-06, 'epoch': 12.14} | |
| [2022-12-19 15:32:58,690] [INFO] [logging.py:68:log_dist] [Rank 0] step=860, skipped=4, lr=[9.211111111111111e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:32:58,691] [INFO] [timer.py:196:stop] epoch=0/micro_step=860/global_step=860, RunningAvgSamplesPerSec=4.936165433304045, CurrSamplesPerSec=4.865863664726239, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:35:35,572] [INFO] [logging.py:68:log_dist] [Rank 0] step=870, skipped=4, lr=[9.188888888888889e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:35:35,573] [INFO] [timer.py:196:stop] epoch=0/micro_step=870/global_step=870, RunningAvgSamplesPerSec=4.93439287989596, CurrSamplesPerSec=4.796413956971808, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0021, 'learning_rate': 9.17777777777778e-06, 'epoch': 12.5} | |
| [2022-12-19 15:38:05,552] [INFO] [logging.py:68:log_dist] [Rank 0] step=880, skipped=4, lr=[9.166666666666666e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:38:05,553] [INFO] [timer.py:196:stop] epoch=0/micro_step=880/global_step=880, RunningAvgSamplesPerSec=4.935435621213895, CurrSamplesPerSec=5.141677427678901, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:40:40,472] [INFO] [logging.py:68:log_dist] [Rank 0] step=890, skipped=4, lr=[9.144444444444444e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:40:40,474] [INFO] [timer.py:196:stop] epoch=0/micro_step=890/global_step=890, RunningAvgSamplesPerSec=4.9344182185583465, CurrSamplesPerSec=4.7700867918514005, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:43:14,392] [INFO] [logging.py:68:log_dist] [Rank 0] step=900, skipped=4, lr=[9.122222222222223e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:43:14,394] [INFO] [timer.py:196:stop] epoch=0/micro_step=900/global_step=900, RunningAvgSamplesPerSec=4.93406040285683, CurrSamplesPerSec=4.997473128027491, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0018, 'learning_rate': 9.122222222222223e-06, 'epoch': 12.86} | |
| [2022-12-19 15:45:46,440] [INFO] [logging.py:68:log_dist] [Rank 0] step=910, skipped=4, lr=[9.100000000000001e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:45:46,441] [INFO] [timer.py:196:stop] epoch=0/micro_step=910/global_step=910, RunningAvgSamplesPerSec=4.934397846811847, CurrSamplesPerSec=5.335555684355236, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:48:19,718] [INFO] [logging.py:68:log_dist] [Rank 0] step=920, skipped=4, lr=[9.077777777777779e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:48:19,719] [INFO] [timer.py:196:stop] epoch=0/micro_step=920/global_step=920, RunningAvgSamplesPerSec=4.934105943596077, CurrSamplesPerSec=4.728669681717305, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0015, 'learning_rate': 9.066666666666667e-06, 'epoch': 13.21} | |
| [2022-12-19 15:50:55,618] [INFO] [logging.py:68:log_dist] [Rank 0] step=930, skipped=4, lr=[9.055555555555556e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:50:55,620] [INFO] [timer.py:196:stop] epoch=0/micro_step=930/global_step=930, RunningAvgSamplesPerSec=4.9329043169365985, CurrSamplesPerSec=4.770316514010341, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:53:30,553] [INFO] [logging.py:68:log_dist] [Rank 0] step=940, skipped=4, lr=[9.033333333333334e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:53:30,554] [INFO] [timer.py:196:stop] epoch=0/micro_step=940/global_step=940, RunningAvgSamplesPerSec=4.932100753197186, CurrSamplesPerSec=4.881037799953705, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 15:56:06,793] [INFO] [logging.py:68:log_dist] [Rank 0] step=950, skipped=4, lr=[9.011111111111111e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:56:06,794] [INFO] [timer.py:196:stop] epoch=0/micro_step=950/global_step=950, RunningAvgSamplesPerSec=4.930800433891351, CurrSamplesPerSec=4.800189378580432, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0018, 'learning_rate': 9.011111111111111e-06, 'epoch': 13.57} | |
| [2022-12-19 15:58:42,760] [INFO] [logging.py:68:log_dist] [Rank 0] step=960, skipped=4, lr=[8.988888888888889e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 15:58:42,761] [INFO] [timer.py:196:stop] epoch=0/micro_step=960/global_step=960, RunningAvgSamplesPerSec=4.929535632898922, CurrSamplesPerSec=4.854309583752725, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 16:01:17,719] [INFO] [logging.py:68:log_dist] [Rank 0] step=970, skipped=4, lr=[8.966666666666667e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 16:01:17,720] [INFO] [timer.py:196:stop] epoch=0/micro_step=970/global_step=970, RunningAvgSamplesPerSec=4.929074176976312, CurrSamplesPerSec=5.365831210088781, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.0012, 'learning_rate': 8.955555555555555e-06, 'epoch': 13.93} | |
| [2022-12-19 16:03:41,427] [INFO] [logging.py:68:log_dist] [Rank 0] step=980, skipped=4, lr=[8.944444444444446e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 16:03:41,428] [INFO] [timer.py:196:stop] epoch=0/micro_step=980/global_step=980, RunningAvgSamplesPerSec=4.932864222713419, CurrSamplesPerSec=5.449948670863249, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 16:06:06,943] [INFO] [logging.py:68:log_dist] [Rank 0] step=990, skipped=4, lr=[8.922222222222224e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 16:06:06,945] [INFO] [timer.py:196:stop] epoch=0/micro_step=990/global_step=990, RunningAvgSamplesPerSec=4.935715217442322, CurrSamplesPerSec=5.154145743675715, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| [2022-12-19 16:08:33,907] [INFO] [logging.py:68:log_dist] [Rank 0] step=1000, skipped=4, lr=[8.900000000000001e-06], mom=[[0.9, 0.999]] | |
| [2022-12-19 16:08:33,909] [INFO] [timer.py:196:stop] epoch=0/micro_step=1000/global_step=1000, RunningAvgSamplesPerSec=4.9379997074583395, CurrSamplesPerSec=5.124673891667898, MemAllocated=1.52GB, MaxMemAllocated=26.06GB | |
| {'loss': 0.001, 'learning_rate': 8.900000000000001e-06, 'epoch': 14.29} | |
| {'eval_loss': 0.398193359375, 'eval_wer': 23.14374107567825, 'eval_runtime': 830.3901, 'eval_samples_per_second': 2.73, 'eval_steps_per_second': 0.086, 'epoch': 14.29} | |
| [2022-12-19 16:22:27,447] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step1000 is begin to save! | |
| [2022-12-19 16:22:27,482] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-1000/global_step1000/mp_rank_00_model_states.pt | |
| [2022-12-19 16:22:27,482] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-1000/global_step1000/mp_rank_00_model_states.pt... | |
| [2022-12-19 16:22:30,465] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-1000/global_step1000/mp_rank_00_model_states.pt. | |
| [2022-12-19 16:22:30,466] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt... | |
| [2022-12-19 16:22:42,888] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt. | |
| [2022-12-19 16:22:42,888] [INFO] [engine.py:3394:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-1000/global_step1000/zero_pp_rank_0_mp_rank_00_optim_states.pt | |
| [2022-12-19 16:22:42,888] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step1000 is ready now! | |