Training in progress, step 17000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d6b908688a7fe9afc5ef9abd519c0d74a01ffa0cc51a675d40e17ea04caa9a53
+oid sha256:6eb966ad6ac37fa70d72a7fa9519636dafdf4aedbdf2f88a485ea9a74d6b69b0
 size 3219908024

wandb/run-20231119_105908-nwx0xdmm/files/output.log

@@ -9720,3 +9720,1079 @@ Reading metadata...: 1704it [00:00, 9310.25it/s]
 
 
  27%|████████████████████████▎                                                                  | 16060/60000 [29:06:43<126:07:15, 10.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▍                                                                  | 16080/60000 [29:10:04<120:15:14,  9.86s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▍                                                                  | 16100/60000 [29:13:24<122:01:30, 10.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▍                                                                  | 16120/60000 [29:16:50<123:15:58, 10.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▍                                                                  | 16140/60000 [29:20:33<154:05:18, 12.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▌                                                                  | 16159/60000 [29:24:08<122:48:59, 10.09s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▌                                                                  | 16180/60000 [29:27:36<122:13:32, 10.04s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▌                                                                  | 16200/60000 [29:31:09<143:48:50, 11.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▌                                                                  | 16219/60000 [29:34:28<127:38:53, 10.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▋                                                                  | 16239/60000 [29:37:53<123:30:41, 10.16s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▋                                                                  | 16259/60000 [29:41:15<123:45:20, 10.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▋                                                                  | 16279/60000 [29:45:02<132:20:33, 10.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▋                                                                  | 16299/60000 [29:48:46<125:29:30, 10.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▊                                                                  | 16319/60000 [29:52:31<128:38:16, 10.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▊                                                                  | 16339/60000 [29:55:54<122:43:55, 10.12s/it]
+
+
+
+ 27%|████████████████████████▊                                                                  | 16342/60000 [29:56:25<124:03:14, 10.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▊                                                                  | 16359/60000 [29:59:15<122:21:04, 10.09s/it]
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▊                                                                  | 16368/60000 [30:00:43<108:23:10,  8.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▊                                                                  | 16380/60000 [30:02:51<125:11:45, 10.33s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▊                                                                  | 16399/60000 [30:06:02<121:19:57, 10.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▉                                                                  | 16419/60000 [30:09:25<123:04:44, 10.17s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▉                                                                  | 16439/60000 [30:12:45<119:06:50,  9.84s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▉                                                                  | 16459/60000 [30:16:07<121:10:38, 10.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|████████████████████████▉                                                                  | 16479/60000 [30:19:34<123:35:09, 10.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|█████████████████████████                                                                  | 16499/60000 [30:22:55<119:45:21,  9.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████                                                                  | 16519/60000 [30:26:37<121:59:39, 10.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████                                                                  | 16539/60000 [30:29:57<119:07:17,  9.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████                                                                  | 16559/60000 [30:33:24<123:19:17, 10.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▏                                                                 | 16579/60000 [30:36:45<122:51:48, 10.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▏                                                                 | 16599/60000 [30:40:05<120:01:23,  9.96s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▏                                                                 | 16619/60000 [30:43:25<120:45:41, 10.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▏                                                                 | 16640/60000 [30:47:04<123:43:58, 10.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▎                                                                 | 16659/60000 [30:50:21<120:27:20, 10.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▎                                                                 | 16679/60000 [30:53:42<119:07:12,  9.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▎                                                                 | 16700/60000 [30:57:11<119:40:19,  9.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▎                                                                 | 16719/60000 [31:00:21<117:32:45,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▍                                                                 | 16739/60000 [31:03:46<119:11:33,  9.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▍                                                                 | 16759/60000 [31:07:07<120:05:49, 10.00s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▍                                                                 | 16780/60000 [31:11:20<130:32:17, 10.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 9681.06it/s]                                                | 16799/60000 [31:14:40<124:14:11, 10.35s/it]
+
+ 28%|█████████████████████████▍                                                                 | 16800/60000 [31:14:52<131:19:10, 10.94s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▌                                                                 | 16820/60000 [31:18:26<131:51:50, 10.99s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▌                                                                 | 16839/60000 [31:21:44<121:46:12, 10.16s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▌                                                                 | 16857/60000 [31:24:44<121:44:56, 10.16s/it]
+
+
+
+ 28%|█████████████████████████▌                                                                 | 16859/60000 [31:25:08<129:12:41, 10.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▌                                                                 | 16880/60000 [31:29:04<148:33:27, 12.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▋                                                                 | 16899/60000 [31:32:18<120:37:45, 10.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▋                                                                 | 16919/60000 [31:35:51<128:47:47, 10.76s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▋                                                                 | 16939/60000 [31:39:27<127:11:31, 10.63s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▋                                                                 | 16959/60000 [31:43:12<138:13:14, 11.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▊                                                                 | 16980/60000 [31:46:55<127:12:29, 10.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▊                                                                 | 16999/60000 [31:50:26<133:32:15, 11.18s/it]
+ 28%|█████████████████████████▊                                                                 | 17000/60000 [31:50:37<133:22:18, 11.17s/it][INFO|trainer.py:3173] 2023-11-20 18:49:52,335 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-20 18:49:52,336 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-20 18:49:52,336 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 2782.72it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-20 18:49:55,080 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale. If client_id, up_votes, input_length, segment, age, down_votes, gender, accent, path, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.17822265625, 'eval_wer': 8.46211835657746, 'eval_runtime': 701.0277, 'eval_samples_per_second': 2.431, 'eval_steps_per_second': 0.608, 'epoch': 0.28}
+ 28%|█████████████████████████▊                                                                 | 17000/60000 [32:02:18<133:22:18, 11.17s/it][INFO|trainer.py:2896] 2023-11-20 19:02:01,287 >> Saving model checkpoint to ./checkpoint-17000
+[INFO|configuration_utils.py:462] 2023-11-20 19:02:01,296 >> Configuration saved in ./checkpoint-17000/config.json
+[INFO|configuration_utils.py:568] 2023-11-20 19:02:01,301 >> Configuration saved in ./checkpoint-17000/generation_config.json
+[INFO|modeling_utils.py:2194] 2023-11-20 19:02:42,296 >> Model weights saved in ./checkpoint-17000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-20 19:02:42,302 >> Feature extractor saved in ./checkpoint-17000/preprocessor_config.json
+[2023-11-20 19:02:42,327] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step17000 is about to be saved!
+[2023-11-20 19:02:42,356] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-17000/global_step17000/mp_rank_00_model_states.pt
+[2023-11-20 19:02:42,356] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-17000/global_step17000/mp_rank_00_model_states.pt...
+[2023-11-20 19:02:48,218] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-17000/global_step17000/mp_rank_00_model_states.pt.
+[2023-11-20 19:02:48,224] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-17000/global_step17000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-20 19:03:09,226] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-17000/global_step17000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-20 19:03:09,242] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-17000/global_step17000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-20 19:03:09,242] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step17000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-20 19:04:16,047 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▊                                                                 | 17019/60000 [32:08:20<130:17:55, 10.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▊                                                                 | 17040/60000 [32:12:10<131:16:58, 11.00s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|█████████████████████████▊                                                                 | 17059/60000 [32:15:35<126:48:28, 10.63s/it]
+
+
+
+

wandb/run-20231119_105908-nwx0xdmm/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.0506, "train/learning_rate": 2.2355084745762713e-06, "train/epoch": 0.27, "train/global_step": 16060, "_timestamp": 1700489158.0882459, "_runtime": 104809.05415892601, "_step": 461, "eval/loss": 0.177978515625, "eval/wer": 8.386732001507728, "eval/runtime": 656.1765, "eval/samples_per_second": 2.597, "eval/steps_per_second": 0.649}
+{"train/loss": 0.0685, "train/learning_rate": 2.1847627118644066e-06, "train/epoch": 0.28, "train/global_step": 17060, "_timestamp": 1700500500.2140026, "_runtime": 116151.17991566658, "_step": 512, "eval/loss": 0.17822265625, "eval/wer": 8.46211835657746, "eval/runtime": 701.0277, "eval/samples_per_second": 2.431, "eval/steps_per_second": 0.608}

wandb/run-20231119_105908-nwx0xdmm/run-nwx0xdmm.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f086bde30c0954175495748b54e3cb90faf08e5fca8c56b8d38fc84fbb161504
-size 6524971
+oid sha256:36987020cece41883330957dce249337e5d1422bf5461cb4933d340714887b84
+size 7260384