Training in progress, step 18000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6eb966ad6ac37fa70d72a7fa9519636dafdf4aedbdf2f88a485ea9a74d6b69b0
+oid sha256:1d63b3b7c1076ec597dcdcc6541ece450e677504bd6df3bf6b6d0efdff7161ba
 size 3219908024

wandb/run-20231119_105908-nwx0xdmm/files/output.log

@@ -10796,3 +10796,1060 @@ Reading metadata...: 1704it [00:00, 2782.72it/s]
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▉                                                                 | 17079/60000 [32:19:07<129:01:51, 10.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▉                                                                 | 17100/60000 [32:23:09<119:08:31, 10.00s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|█████████████████████████▉                                                                 | 17120/60000 [32:26:39<126:38:31, 10.63s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|█████████████████████████▉                                                                 | 17139/60000 [32:30:02<126:25:22, 10.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████                                                                 | 17160/60000 [32:33:43<121:42:38, 10.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████                                                                 | 17179/60000 [32:37:14<129:43:00, 10.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████                                                                 | 17200/60000 [32:40:58<126:19:25, 10.63s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████                                                                 | 17220/60000 [32:44:48<121:41:20, 10.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▏                                                                | 17240/60000 [32:48:10<121:17:48, 10.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▏                                                                | 17260/60000 [32:51:46<133:43:41, 11.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▏                                                                | 17280/60000 [32:55:19<124:13:09, 10.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▏                                                                | 17300/60000 [32:58:50<115:22:34,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▎                                                                | 17319/60000 [33:01:58<119:29:26, 10.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▎                                                                | 17340/60000 [33:05:27<116:57:49,  9.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▎                                                                | 17360/60000 [33:08:51<117:19:02,  9.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▎                                                                | 17379/60000 [33:11:59<116:31:25,  9.84s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▍                                                                | 17399/60000 [33:15:15<115:06:32,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▍                                                                | 17419/60000 [33:18:36<116:15:55,  9.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▍                                                                | 17439/60000 [33:22:01<122:21:14, 10.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▍                                                                | 17459/60000 [33:25:20<119:52:56, 10.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▌                                                                | 17479/60000 [33:28:38<117:50:22,  9.98s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▌                                                                | 17499/60000 [33:31:59<117:25:52,  9.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▌                                                                | 17519/60000 [33:35:21<118:30:37, 10.04s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▌                                                                | 17540/60000 [33:38:54<118:32:10, 10.05s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▋                                                                | 17559/60000 [33:42:04<117:47:53,  9.99s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▋                                                                | 17579/60000 [33:45:19<114:13:38,  9.69s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▋                                                                | 17599/60000 [33:48:37<114:49:50,  9.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▋                                                                | 17620/60000 [33:53:09<136:30:35, 11.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▊                                                                | 17639/60000 [33:56:29<115:37:18,  9.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▊                                                                | 17659/60000 [33:59:47<117:19:13,  9.98s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██████████████████████████▊                                                                | 17680/60000 [34:03:21<115:27:02,  9.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██████████████████████████▊                                                                | 17700/60000 [34:06:49<135:01:39, 11.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██████████████████████████▉                                                                | 17720/60000 [34:10:08<114:22:05,  9.74s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██████████████████████████▉                                                                | 17739/60000 [34:13:17<117:01:45,  9.97s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██████████████████████████▉                                                                | 17759/60000 [34:16:37<118:57:18, 10.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██████████████████████████▉                                                                | 17779/60000 [34:19:56<115:35:07,  9.86s/it]
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 9802.57it/s]                                                | 17790/60000 [34:21:51<122:47:12, 10.47s/it]
+
+
+
+
+
+
+
+
+
+ 30%|██████████████████████████▉                                                                | 17799/60000 [34:23:21<114:52:46,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████                                                                | 17820/60000 [34:27:33<130:34:55, 11.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████                                                                | 17840/60000 [34:31:07<123:18:54, 10.53s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████                                                                | 17859/60000 [34:34:29<124:59:35, 10.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████                                                                | 17879/60000 [34:37:57<122:33:15, 10.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▏                                                               | 17900/60000 [34:41:40<125:00:50, 10.69s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|████████████���██████████████▏                                                               | 17920/60000 [34:45:28<132:19:19, 11.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▏                                                               | 17940/60000 [34:49:09<129:39:29, 11.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▏                                                               | 17960/60000 [34:53:03<130:33:00, 11.18s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▎                                                               | 17979/60000 [34:56:34<125:20:14, 10.74s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▎                                                               | 17999/60000 [35:00:17<133:47:48, 11.47s/it]
+ 30%|███████████████████████████▎                                                               | 18000/60000 [35:00:28<133:47:03, 11.47s/it][INFO|trainer.py:3173] 2023-11-20 21:59:43,380 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-20 21:59:43,387 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-20 21:59:43,387 >>   Batch size = 4
+
+
+[INFO|trainer_utils.py:759] 2023-11-20 21:59:47,673 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.1827392578125, 'eval_wer': 8.678854127402941, 'eval_runtime': 734.6885, 'eval_samples_per_second': 2.319, 'eval_steps_per_second': 0.58, 'epoch': 0.3}
+ 30%|███████████████████████████▎                                                               | 18000/60000 [35:12:43<133:47:03, 11.47s/it][INFO|trainer.py:2896] 2023-11-20 22:12:31,234 >> Saving model checkpoint to ./checkpoint-18000
+[INFO|configuration_utils.py:462] 2023-11-20 22:12:31,268 >> Configuration saved in ./checkpoint-18000/config.json
+[INFO|configuration_utils.py:568] 2023-11-20 22:12:31,286 >> Configuration saved in ./checkpoint-18000/generation_config.json
+[2023-11-20 22:13:42,614] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step18000 is about to be saved!
+[2023-11-20 22:13:42,781] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-18000/global_step18000/mp_rank_00_model_states.pt
+[2023-11-20 22:13:42,781] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-18000/global_step18000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-20 22:13:42,402 >> Model weights saved in ./checkpoint-18000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-20 22:13:42,438 >> Feature extractor saved in ./checkpoint-18000/preprocessor_config.json
+[2023-11-20 22:13:58,098] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-18000/global_step18000/mp_rank_00_model_states.pt.
+[2023-11-20 22:13:58,157] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-18000/global_step18000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-20 22:15:54,672] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-18000/global_step18000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-20 22:15:54,682] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-18000/global_step18000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-20 22:15:54,683] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step18000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-20 22:17:07,828 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▎                                                               | 18019/60000 [35:22:13<147:24:35, 12.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███████████████████████████▎                                                               | 18039/60000 [35:26:16<145:51:45, 12.51s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+

wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.0685, "train/learning_rate": 2.1847627118644066e-06, "train/epoch": 0.28, "train/global_step": 17060, "_timestamp": 1700500500.2140026, "_runtime": 116151.17991566658, "_step": 512, "eval/loss": 0.17822265625, "eval/wer": 8.46211835657746, "eval/runtime": 701.0277, "eval/samples_per_second": 2.431, "eval/steps_per_second": 0.608}
+{"train/loss": 0.1355, "train/learning_rate": 2.1349322033898305e-06, "train/epoch": 0.3, "train/global_step": 18040, "_timestamp": 1700511965.7339036, "_runtime": 127616.6998167038, "_step": 562, "eval/loss": 0.1827392578125, "eval/wer": 8.678854127402941, "eval/runtime": 734.6885, "eval/samples_per_second": 2.319, "eval/steps_per_second": 0.58}

wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:36987020cece41883330957dce249337e5d1422bf5461cb4933d340714887b84
-size 7260384
+oid sha256:c661509eff524fa6fc61166ff8fb70b3921709b9feb3fff3ae66325b5480e695
+size 8003044