Training in progress, step 20000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3e038b96faf7382818cc0badadbf52a75b3504be5e84c5740dff060258144b32
+oid sha256:74eacfca3ebe20a52f7d30c59c5d75c0590319ade9029e6353b301d09ac8dbfc
 size 3219908024

wandb/run-20231119_105908-nwx0xdmm/files/output.log

@@ -12941,3 +12941,1078 @@ Reading metadata...: 1704it [00:00, 6722.89it/s]
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▉                                                              | 19079/60000 [38:42:10<107:48:53,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▉                                                              | 19099/60000 [38:45:48<107:55:19,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|████████████████████████████▉                                                              | 19119/60000 [38:49:00<107:04:14,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▎                                                              | 19133/60000 [38:51:09<96:13:34,  8.48s/it]
+
+
+
+
+
+
+
+ 32%|█████████████████████████████                                                              | 19140/60000 [38:52:15<105:34:26,  9.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████                                                              | 19160/60000 [38:55:22<107:01:04,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████                                                              | 19179/60000 [38:58:21<105:55:53,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████                                                              | 19200/60000 [39:01:44<117:40:39, 10.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▏                                                             | 19219/60000 [39:04:44<107:07:13,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▏                                                             | 19240/60000 [39:08:00<105:38:17,  9.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▏                                                             | 19259/60000 [39:11:14<139:16:18, 12.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▏                                                             | 19279/60000 [39:14:25<108:01:05,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▎                                                             | 19300/60000 [39:18:02<109:49:16,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▎                                                             | 19320/60000 [39:21:11<109:05:29,  9.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▎                                                             | 19339/60000 [39:24:11<107:31:50,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▎                                                             | 19360/60000 [39:27:28<106:27:32,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▍                                                             | 19380/60000 [39:30:42<110:31:15,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▍                                                             | 19399/60000 [39:33:44<108:46:20,  9.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▍                                                             | 19420/60000 [39:37:05<108:00:18,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▍                                                             | 19440/60000 [39:40:18<116:33:09, 10.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▌                                                             | 19453/60000 [39:42:24<100:52:54,  8.96s/it]
+
+Reading metadata...: 2165it [00:00, 12527.03it/s]                                               | 19455/60000 [39:42:44<105:26:55,  9.36s/it]
+
+
+
+
+
+ 32%|█████████████████████████████▌                                                             | 19460/60000 [39:43:34<110:43:09,  9.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▌                                                             | 19480/60000 [39:46:50<108:27:51,  9.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|█████████████████████████████▌                                                             | 19499/60000 [39:49:50<105:19:55,  9.36s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▌                                                             | 19519/60000 [39:53:11<109:22:02,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▋                                                             | 19540/60000 [39:56:32<107:12:26,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▋                                                             | 19560/60000 [39:59:45<104:43:33,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▋                                                             | 19580/60000 [40:02:51<103:39:51,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▋                                                             | 19600/60000 [40:06:34<112:45:59, 10.05s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▊                                                             | 19620/60000 [40:09:50<126:51:25, 11.31s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▊                                                             | 19640/60000 [40:12:57<103:51:43,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▊                                                             | 19660/60000 [40:16:08<106:15:40,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▊                                                             | 19680/60000 [40:19:15<104:46:21,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▉                                                             | 19700/60000 [40:22:21<104:21:25,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▉                                                             | 19720/60000 [40:25:28<106:03:53,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▉                                                             | 19740/60000 [40:28:38<102:14:09,  9.14s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▉                                                             | 19759/60000 [40:31:37<105:55:25,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 9427.38it/s]                                                | 19770/60000 [40:33:19<104:04:31,  9.31s/it]
+
+
+
+
+
+
+
+
+
+ 33%|█████████████████████████████▉                                                             | 19779/60000 [40:34:44<104:28:16,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████                                                             | 19799/60000 [40:37:52<107:07:45,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████                                                             | 19819/60000 [40:41:28<149:22:40, 13.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████                                                             | 19839/60000 [40:44:35<103:14:57,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████                                                             | 19859/60000 [40:47:40<103:20:11,  9.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▏                                                            | 19879/60000 [40:50:46<103:22:51,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▏                                                            | 19899/60000 [40:53:51<103:05:17,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▏                                                            | 19919/60000 [40:57:00<102:54:58,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▏                                                            | 19939/60000 [41:00:31<113:58:01, 10.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▎                                                            | 19960/60000 [41:04:19<104:54:09,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▎                                                            | 19979/60000 [41:07:18<103:49:27,  9.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▎                                                            | 19999/60000 [41:10:29<104:45:41,  9.43s/it]
+ 33%|██████████████████████████████▎                                                            | 20000/60000 [41:10:38<103:12:29,  9.29s/it][INFO|trainer.py:3173] 2023-11-21 04:09:53,205 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-21 04:09:53,206 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-21 04:09:53,206 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 8993.55it/s]
+[INFO|trainer_utils.py:759] 2023-11-21 04:09:54,170 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+ 33%|██████████████████████████████▎                                                            | 20000/60000 [41:20:43<103:12:29,  9.29s/it]
+ 33%|██████████████████████████████▎                                                            | 20000/60000 [41:20:43<103:12:29,  9.29s/it][INFO|trainer.py:2896] 2023-11-21 04:20:30,221 >> Saving model checkpoint to ./checkpoint-20000
+[INFO|configuration_utils.py:462] 2023-11-21 04:20:30,236 >> Configuration saved in ./checkpoint-20000/config.json
+[INFO|configuration_utils.py:568] 2023-11-21 04:20:30,248 >> Configuration saved in ./checkpoint-20000/generation_config.json
+[2023-11-21 04:21:16,086] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step20000 is about to be saved!
+[2023-11-21 04:21:16,121] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-20000/global_step20000/mp_rank_00_model_states.pt
+[2023-11-21 04:21:16,121] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-20000/global_step20000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-21 04:21:16,042 >> Model weights saved in ./checkpoint-20000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-21 04:21:16,050 >> Feature extractor saved in ./checkpoint-20000/preprocessor_config.json
+[2023-11-21 04:21:24,359] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-20000/global_step20000/mp_rank_00_model_states.pt.
+[2023-11-21 04:21:24,370] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-20000/global_step20000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-21 04:21:47,972] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-20000/global_step20000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-21 04:21:47,997] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-20000/global_step20000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-21 04:21:47,998] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step20000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-21 04:22:45,221 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▎                                                            | 20019/60000 [41:26:37<113:49:18, 10.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▍                                                            | 20040/60000 [41:29:57<105:38:11,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|██████████████████████████████▍                                                            | 20059/60000 [41:32:57<104:19:24,  9.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+

wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.0749, "train/learning_rate": 2.0831694915254236e-06, "train/epoch": 0.32, "train/global_step": 19060, "_timestamp": 1700523503.6049488, "_runtime": 139154.5708618164, "_step": 614, "eval/loss": 0.17822265625, "eval/wer": 8.518658122879758, "eval/runtime": 591.7997, "eval/samples_per_second": 2.879, "eval/steps_per_second": 0.72}
+{"train/loss": 0.0712, "train/learning_rate": 2.0324237288135594e-06, "train/epoch": 0.33, "train/global_step": 20060, "_timestamp": 1700533941.1437454, "_runtime": 149592.1096584797, "_step": 665, "eval/loss": 0.1776123046875, "eval/wer": 8.509234828496043, "eval/runtime": 604.4094, "eval/samples_per_second": 2.819, "eval/steps_per_second": 0.705}

wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb75bd1ca21ebda7f7845cb9f0501e78a6e6ac1408a014fd785762f2b48b8b87
-size 8749665
+oid sha256:4815856c5203c68d373aea1f58544c4f5864fd2c3cc73cf29d1fde1df0640e7a
+size 9458453