Training in progress, step 31000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:837fabe2f9887cb975f4c577c865c8c31d3ecc4fcc9afcd2a38e8c97918a8409
+oid sha256:df95fbcfc0eb4bfed629e4280bbb789a37aa3cf5571ceb0ee20543d94a2cc062
 size 3219908024

wandb/run-20231122_234140-blvl5h7j/files/output.log

@@ -2274,3 +2274,1075 @@ Reading metadata...: 1it [00:00,  6.49it/s]
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30079/60000 [12:18:14<80:32:25,  9.69s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30099/60000 [12:21:31<80:38:44,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30120/60000 [12:25:23<83:32:40, 10.07s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30140/60000 [12:28:42<82:54:00,  9.99s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30160/60000 [12:31:52<78:04:06,  9.42s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30180/60000 [12:35:03<79:27:09,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30199/60000 [12:38:06<79:25:53,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30219/60000 [12:41:16<78:47:02,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30240/60000 [12:44:41<77:42:06,  9.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30260/60000 [12:47:50<78:27:31,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30280/60000 [12:51:14<84:49:11, 10.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|██████████          | 30300/60000 [12:54:59<93:55:24, 11.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████          | 30320/60000 [12:58:16<79:14:44,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████          | 30340/60000 [13:01:28<79:01:40,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████          | 30360/60000 [13:04:38<77:57:09,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30379/60000 [13:07:40<78:18:06,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30400/60000 [13:11:02<79:33:20,  9.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30420/60000 [13:14:40<80:27:51,  9.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30439/60000 [13:17:44<80:26:22,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30460/60000 [13:21:07<78:05:36,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30480/60000 [13:24:20<78:26:07,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30500/60000 [13:27:41<79:28:50,  9.70s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30520/60000 [13:30:53<78:24:01,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30540/60000 [13:34:06<78:36:09,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30560/60000 [13:37:18<81:20:11,  9.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30580/60000 [13:40:36<82:45:06, 10.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30600/60000 [13:43:49<78:35:08,  9.62s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30620/60000 [13:47:10<78:22:15,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30640/60000 [13:50:20<70:25:28,  8.64s/it]
+[2023-11-23 13:32:07,459] [INFO] [loss_scaler.py:190:update_scale] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 131072, but hysteresis is 2. Reducing hysteresis to 1
+
+ 51%|██████████▏         | 30641/60000 [13:50:26<64:34:06,  7.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30660/60000 [13:53:29<79:20:08,  9.73s/it]
+
+Reading metadata...: 1650it [00:00, 10271.70it/s]78:55:47,  9.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30680/60000 [13:56:51<78:41:06,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30700/60000 [14:00:04<78:09:47,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30719/60000 [14:03:08<79:19:49,  9.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▏         | 30740/60000 [14:06:32<79:48:04,  9.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▎         | 30760/60000 [14:09:50<80:23:14,  9.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▎         | 30780/60000 [14:13:03<78:14:38,  9.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|█████████▊         | 30800/60000 [14:16:44<100:31:41, 12.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▎         | 30820/60000 [14:20:09<80:03:04,  9.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▎         | 30840/60000 [14:23:52<78:43:13,  9.72s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▎         | 30860/60000 [14:27:41<80:51:40,  9.99s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 51%|██████████▎         | 30880/60000 [14:30:55<78:07:44,  9.66s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 30900/60000 [14:34:12<78:39:08,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 30920/60000 [14:37:26<78:29:36,  9.72s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 30940/60000 [14:40:44<77:20:19,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 30960/60000 [14:44:07<75:55:03,  9.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 30979/60000 [14:47:09<78:34:27,  9.75s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 31000/60000 [14:50:32<77:14:57,  9.59s/it][INFO|trainer.py:3173] 2023-11-23 14:32:19,981 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-23 14:32:19,981 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-23 14:32:19,981 >>   Batch size = 4
+{'loss': 0.0228, 'learning_rate': 1.4771186440677966e-06, 'epoch': 0.52}
+Reading metadata...: 1704it [00:00, 9424.36it/s]
+[INFO|trainer_utils.py:759] 2023-11-23 14:32:21,199 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: down_votes, up_votes, gender, locale, segment, client_id, accent, input_length, path, age. If down_votes, up_votes, gender, locale, segment, client_id, accent, input_length, path, age are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.202880859375, 'eval_wer': 8.386732001507728, 'eval_runtime': 595.6213, 'eval_samples_per_second': 2.861, 'eval_steps_per_second': 0.715, 'epoch': 0.52}
+ 52%|██████████▎         | 31000/60000 [15:00:28<77:14:57,  9.59s/it][INFO|trainer.py:2896] 2023-11-23 14:42:39,683 >> Saving model checkpoint to ./checkpoint-31000
+[INFO|configuration_utils.py:462] 2023-11-23 14:42:39,693 >> Configuration saved in ./checkpoint-31000/config.json
+[INFO|configuration_utils.py:568] 2023-11-23 14:42:39,711 >> Configuration saved in ./checkpoint-31000/generation_config.json
+[2023-11-23 14:43:25,683] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step31000 is about to be saved!
+[2023-11-23 14:43:25,724] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-31000/global_step31000/mp_rank_00_model_states.pt
+[2023-11-23 14:43:25,724] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-31000/global_step31000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-23 14:43:25,656 >> Model weights saved in ./checkpoint-31000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-23 14:43:25,662 >> Feature extractor saved in ./checkpoint-31000/preprocessor_config.json
+[2023-11-23 14:43:39,756] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-31000/global_step31000/mp_rank_00_model_states.pt.
+[2023-11-23 14:43:39,775] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-31000/global_step31000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-23 14:44:06,768] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-31000/global_step31000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-23 14:44:06,798] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-31000/global_step31000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-23 14:44:06,798] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step31000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-23 14:45:03,018 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 31019/60000 [15:06:34<95:41:12, 11.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 31039/60000 [15:09:53<79:19:11,  9.86s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 31059/60000 [15:13:09<78:01:30,  9.71s/it]
+
+
+
+
+
+
+
+
+

wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.0812, "train/learning_rate": 1.5248135593220338e-06, "train/epoch": 0.5, "train/global_step": 30060, "_timestamp": 1700733415.2111008, "_runtime": 44114.32262992859, "_step": 104, "eval/loss": 0.193603515625, "eval/wer": 8.358462118356577, "eval/runtime": 600.6457, "eval/samples_per_second": 2.837, "eval/steps_per_second": 0.709}
+{"train/loss": 0.039, "train/learning_rate": 1.4740677966101696e-06, "train/epoch": 0.52, "train/global_step": 31060, "_timestamp": 1700744106.793268, "_runtime": 54805.90479707718, "_step": 155, "eval/loss": 0.202880859375, "eval/wer": 8.386732001507728, "eval/runtime": 595.6213, "eval/samples_per_second": 2.861, "eval/steps_per_second": 0.715}

wandb/run-20231122_234140-blvl5h7j/run-blvl5h7j.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2907ff4ad50e187d1d77714ad79bcb13875da01d13afbec0001cade84a998108
-size 1994140
+oid sha256:e1d76afc306ea64fa0af8f18e568ce1cd446c18118aa650f13d61464fe0dfd0c
+size 2588702