Training in progress, step 18000

errs

@@ -2019,3 +2019,49 @@ Configuration saved in ./checkpoint-17500/config.json
 Model weights saved in ./checkpoint-17500/pytorch_model.bin
 Configuration saved in ./checkpoint-17500/preprocessor_config.json
 Configuration saved in ./preprocessor_config.json
+Deleting older checkpoint [checkpoint-17000] due to args.save_total_limit
+
 97% 17501/18000 [47:33:05<9:39:58, 69.74s/it]
 97% 17502/18000 [47:33:12<7:02:45, 50.94s/it]
 97% 17503/18000 [47:33:18<5:10:19, 37.46s/it]
 97% 17504/18000 [47:33:22<3:48:09, 27.60s/it]
 97% 17505/18000 [47:33:39<3:19:25, 24.17s/it]
 97% 17506/18000 [47:33:52<2:52:32, 20.96s/it]
 97% 17507/18000 [47:34:03<2:27:47, 17.99s/it]
 97% 17508/18000 [47:34:13<2:06:58, 15.48s/it]
 97% 17509/18000 [47:34:21<1:49:25, 13.37s/it]
 97% 17510/18000 [47:34:28<1:34:00, 11.51s/it]
 97% 17511/18000 [47:34:34<1:19:52,  9.80s/it]
 97% 17512/18000 [47:34:39<1:06:52,  8.22s/it]
 97% 17513/18000 [47:34:55<1:26:56, 10.71s/it]
 97% 17514/18000 [47:35:08<1:32:51, 11.46s/it]
 97% 17515/18000 [47:35:19<1:30:53, 11.24s/it]
 97% 17516/18000 [47:35:29<1:26:14, 10.69s/it]
 97% 17517/18000 [47:35:37<1:20:07,  9.95s/it]
 97% 17518/18000 [47:35:44<1:12:32,  9.03s/it]
 97% 17519/18000 [47:35:49<1:04:03,  7.99s/it]
 97% 17520/18000 [47:35:54<55:03,  6.88s/it]  
 97% 17521/18000 [47:36:10<1:18:08,  9.79s/it]
 97% 17522/18000 [47:36:23<1:26:18, 10.83s/it]
 97% 17523/18000 [47:36:34<1:26:16, 10.85s/it]
 97% 17524/18000 [47:36:44<1:22:52, 10.45s/it]
 97% 17525/18000 [47:36:52<1:17:41,  9.81s/it]
 97% 17526/18000 [47:36:59<1:10:40,  8.95s/it]
 97% 17527/18000 [47:37:05<1:02:31,  7.93s/it]
 97% 17528/18000 [47:37:09<53:52,  6.85s/it]  
 97% 17529/18000 [47:37:24<1:14:02,  9.43s/it]
 97% 17530/18000 [47:37:34<1:15:16,  9.61s/it]
 97% 17531/18000 [47:37:42<1:09:27,  8.89s/it]
 97% 17532/18000 [47:37:45<56:25,  7.23s/it]  
 97% 17533/18000 [47:38:02<1:18:19, 10.06s/it]
 97% 17534/18000 [47:38:15<1:26:02, 11.08s/it]
 97% 17535/18000 [47:38:26<1:25:33, 11.04s/it]
 97% 17536/18000 [47:38:36<1:21:56, 10.60s/it]
 97% 17537/18000 [47:38:44<1:16:37,  9.93s/it]
 97% 17538/18000 [47:38:51<1:10:02,  9.10s/it]
 97% 17539/18000 [47:38:57<1:02:59,  8.20s/it]
 97% 17540/18000 [47:39:02<54:29,  7.11s/it]  
 97% 17541/18000 [47:39:18<1:15:30,  9.87s/it]
 97% 17542/18000 [47:39:31<1:23:06, 10.89s/it]
 97% 17543/18000 [47:39:42<1:22:42, 10.86s/it]
 97% 17544/18000 [47:39:52<1:19:09, 10.42s/it]
 97% 17545/18000 [47:40:00<1:13:44,  9.72s/it]
 97% 17546/18000 [47:40:06<1:06:40,  8.81s/it]
 97% 17547/18000 [47:40:12<59:32,  7.89s/it]  
 97% 17548/18000 [47:40:16<51:35,  6.85s/it]
 97% 17549/18000 [47:40:33<1:13:29,  9.78s/it]
 98% 17550/18000 [47:40:46<1:20:54, 10.79s/it]
 98% 17551/18000 [47:40:57<1:21:22, 10.87s/it]
 98% 17552/18000 [47:41:07<1:17:45, 10.41s/it]
 98% 17553/18000 [47:41:15<1:13:14,  9.83s/it]
 98% 17554/18000 [47:41:22<1:07:08,  9.03s/it]
 98% 17555/18000 [47:41:28<1:00:00,  8.09s/it]
 98% 17556/18000 [47:41:33<51:53,  7.01s/it]  
 98% 17557/18000 [47:41:49<1:13:18,  9.93s/it]
 98% 17558/18000 [47:42:03<1:20:45, 10.96s/it]
 98% 17559/18000 [47:42:13<1:19:57, 10.88s/it]
 98% 17560/18000 [47:42:23<1:16:23, 10.42s/it]
 98% 17561/18000 [47:42:31<1:11:13,  9.74s/it]
 98% 17562/18000 [47:42:38<1:04:12,  8.80s/it]
 98% 17563/18000 [47:42:43<57:07,  7.84s/it]  
 98% 17564/18000 [47:42:47<49:12,  6.77s/it]
 98% 17565/18000 [47:43:03<1:07:34,  9.32s/it]
 98% 17566/18000 [47:43:13<1:08:52,  9.52s/it]
 98% 17567/18000 [47:43:20<1:03:18,  8.77s/it]
 98% 17568/18000 [47:43:23<51:07,  7.10s/it]  
 98% 17569/18000 [47:43:39<1:11:13,  9.92s/it]
 98% 17570/18000 [47:43:53<1:18:01, 10.89s/it]
 98% 17571/18000 [47:44:03<1:17:08, 10.79s/it]
 98% 17572/18000 [47:44:12<1:13:34, 10.31s/it]
 98% 17573/18000 [47:44:20<1:08:41,  9.65s/it]
 98% 17574/18000 [47:44:27<1:02:06,  8.75s/it]
 98% 17575/18000 [47:44:33<55:17,  7.81s/it]  
 98% 17576/18000 [47:44:37<47:51,  6.77s/it]
 98% 17577/18000 [47:44:54<1:08:46,  9.75s/it]
 98% 17578/18000 [47:45:07<1:16:13, 10.84s/it]
 98% 17579/18000 [47:45:18<1:16:05, 10.84s/it]
 98% 17580/18000 [47:45:27<1:12:45, 10.39s/it]
 98% 17581/18000 [47:45:35<1:07:30,  9.67s/it]
 98% 17582/18000 [47:45:42<1:01:28,  8.82s/it]
 98% 17583/18000 [47:45:48<55:03,  7.92s/it]  
 98% 17584/18000 [47:45:52<47:37,  6.87s/it]
 98% 17585/18000 [47:46:09<1:07:28,  9.76s/it]
 98% 17586/18000 [47:46:22<1:14:57, 10.86s/it]
 98% 17587/18000 [47:46:33<1:14:42, 10.85s/it]
 98% 17588/18000 [47:46:43<1:11:46, 10.45s/it]
 98% 17589/18000 [47:46:51<1:07:18,  9.83s/it]
 98% 17590/18000 [47:46:58<1:01:38,  9.02s/it]
 98% 17591/18000 [47:47:04<54:55,  8.06s/it]  
 98% 17592/18000 [47:47:08<47:19,  6.96s/it]
 98% 17593/18000 [47:47:25<1:06:24,  9.79s/it]
 98% 17594/18000 [47:47:38<1:13:07, 10.81s/it]
 98% 17595/18000 [47:47:49<1:12:54, 10.80s/it]
 98% 17596/18000 [47:47:58<1:10:20, 10.45s/it]
 98% 17597/18000 [47:48:07<1:06:08,  9.85s/it]
 98% 17598/18000 [47:48:14<1:00:17,  9.00s/it]
 98% 17599/18000 [47:48:19<53:24,  7.99s/it]  
 98% 17600/18000 [47:48:24<46:26,  6.97s/it]
                                            
 98% 17600/18000 [47:48:24<46:26,  6.97s/it]
 98% 17601/18000 [47:48:40<1:03:29,  9.55s/it]
 98% 17602/18000 [47:48:50<1:04:40,  9.75s/it]
 98% 17603/18000 [47:48:57<1:00:03,  9.08s/it]
 98% 17604/18000 [47:49:01<48:28,  7.35s/it]  
 98% 17605/18000 [47:49:17<1:06:17, 10.07s/it]
 98% 17606/18000 [47:49:30<1:12:06, 10.98s/it]
 98% 17607/18000 [47:49:41<1:11:14, 10.88s/it]
 98% 17608/18000 [47:49:50<1:07:56, 10.40s/it]
 98% 17609/18000 [47:49:58<1:02:56,  9.66s/it]
 98% 17610/18000 [47:50:05<57:32,  8.85s/it]  
 98% 17611/18000 [47:50:11<51:04,  7.88s/it]
 98% 17612/18000 [47:50:15<44:21,  6.86s/it]
 98% 17613/18000 [47:50:32<1:02:49,  9.74s/it]
 98% 17614/18000 [47:50:44<1:08:27, 10.64s/it]
 98% 17615/18000 [47:50:55<1:08:23, 10.66s/it]
 98% 17616/18000 [47:51:04<1:05:49, 10.29s/it]
 98% 17617/18000 [47:51:13<1:02:07,  9.73s/it]
 98% 17618/18000 [47:51:20<56:58,  8.95s/it]  
 98% 17619/18000 [47:51:26<51:01,  8.03s/it]
 98% 17620/18000 [47:51:30<44:07,  6.97s/it]
 98% 17621/18000 [47:51:47<1:02:19,  9.87s/it]
 98% 17622/18000 [47:52:00<1:08:53, 10.94s/it]
 98% 17623/18000 [47:52:11<1:08:26, 10.89s/it]
 98% 17624/18000 [47:52:20<1:05:01, 10.38s/it]
 98% 17625/18000 [47:52:28<1:00:14,  9.64s/it]
 98% 17626/18000 [47:52:35<54:45,  8.78s/it]  
 98% 17627/18000 [47:52:41<48:42,  7.83s/it]
 98% 17628/18000 [47:52:45<41:56,  6.76s/it]
 98% 17629/18000 [47:53:02<1:00:16,  9.75s/it]
 98% 17630/18000 [47:53:15<1:07:18, 10.92s/it]
 98% 17631/18000 [47:53:26<1:07:15, 10.94s/it]
 98% 17632/18000 [47:53:36<1:04:34, 10.53s/it]
 98% 17633/18000 [47:53:44<1:00:30,  9.89s/it]
 98% 17634/18000 [47:53:51<55:13,  9.05s/it]  
 98% 17635/18000 [47:53:57<49:26,  8.13s/it]
 98% 17636/18000 [47:54:02<42:51,  7.06s/it]
 98% 17637/18000 [47:54:17<58:00,  9.59s/it]
 98% 17638/18000 [47:54:28<59:01,  9.78s/it]
 98% 17639/18000 [47:54:35<54:46,  9.10s/it]
 98% 17640/18000 [47:54:39<44:20,  7.39s/it]
 98% 17641/18000 [47:54:55<1:00:50, 10.17s/it]
 98% 17642/18000 [47:55:09<1:06:23, 11.13s/it]
 98% 17643/18000 [47:55:19<1:05:21, 10.98s/it]
 98% 17644/18000 [47:55:29<1:02:09, 10.48s/it]
 98% 17645/18000 [47:55:37<57:50,  9.78s/it]  
 98% 17646/18000 [47:55:44<52:31,  8.90s/it]
 98% 17647/18000 [47:55:49<46:14,  7.86s/it]
 98% 17648/18000 [47:55:53<39:49,  6.79s/it]
 98% 17649/18000 [47:56:10<56:36,  9.68s/it]
 98% 17650/18000 [47:56:23<1:02:50, 10.77s/it]
 98% 17651/18000 [47:56:34<1:02:42, 10.78s/it]
 98% 17652/18000 [47:56:43<1:00:11, 10.38s/it]
 98% 17653/18000 [47:56:52<56:31,  9.77s/it]  
 98% 17654/18000 [47:56:59<51:34,  8.94s/it]
 98% 17655/18000 [47:57:04<46:00,  8.00s/it]
 98% 17656/18000 [47:57:09<39:41,  6.92s/it]
 98% 17657/18000 [47:57:26<56:21,  9.86s/it]
 98% 17658/18000 [47:57:39<1:02:35, 10.98s/it]
 98% 17659/18000 [47:57:50<1:02:20, 10.97s/it]
 98% 17660/18000 [47:58:00<59:57, 10.58s/it]  
 98% 17661/18000 [47:58:08<56:01,  9.92s/it]
 98% 17662/18000 [47:58:15<50:48,  9.02s/it]
 98% 17663/18000 [47:58:21<45:10,  8.04s/it]
 98% 17664/18000 [47:58:25<38:48,  6.93s/it]
 98% 17665/18000 [47:58:42<54:38,  9.79s/it]
 98% 17666/18000 [47:58:55<1:00:12, 10.81s/it]
 98% 17667/18000 [47:59:06<59:54, 10.79s/it]  
 98% 17668/18000 [47:59:15<57:23, 10.37s/it]
 98% 17669/18000 [47:59:23<53:12,  9.64s/it]
 98% 17670/18000 [47:59:30<48:34,  8.83s/it]
 98% 17671/18000 [47:59:36<43:33,  7.94s/it]
 98% 17672/18000 [47:59:40<37:47,  6.91s/it]
 98% 17673/18000 [47:59:56<51:32,  9.46s/it]
 98% 17674/18000 [48:00:06<52:15,  9.62s/it]
 98% 17675/18000 [48:00:13<48:08,  8.89s/it]
 98% 17676/18000 [48:00:16<38:50,  7.19s/it]
 98% 17677/18000 [48:00:33<54:00, 10.03s/it]
 98% 17678/18000 [48:00:46<58:55, 10.98s/it]
 98% 17679/18000 [48:00:57<58:20, 10.90s/it]
 98% 17680/18000 [48:01:06<55:43, 10.45s/it]
 98% 17681/18000 [48:01:14<51:30,  9.69s/it]
 98% 17682/18000 [48:01:21<47:04,  8.88s/it]
 98% 17683/18000 [48:01:26<41:42,  7.89s/it]
 98% 17684/18000 [48:01:31<35:50,  6.81s/it]
 98% 17685/18000 [48:01:47<51:05,  9.73s/it]
 98% 17686/18000 [48:02:01<56:43, 10.84s/it]
 98% 17687/18000 [48:02:12<56:49, 10.89s/it]
 98% 17688/18000 [48:02:21<54:12, 10.42s/it]
 98% 17689/18000 [48:02:30<50:58,  9.83s/it]
 98% 17690/18000 [48:02:36<46:18,  8.96s/it]
 98% 17691/18000 [48:02:42<41:34,  8.07s/it]
 98% 17692/18000 [48:02:47<35:54,  7.00s/it]
 98% 17693/18000 [48:03:04<50:31,  9.87s/it]
 98% 17694/18000 [48:03:17<55:32, 10.89s/it]
 98% 17695/18000 [48:03:28<55:09, 10.85s/it]
 98% 17696/18000 [48:03:37<52:48, 10.42s/it]
 98% 17697/18000 [48:03:45<49:17,  9.76s/it]
 98% 17698/18000 [48:03:52<44:34,  8.86s/it]
 98% 17699/18000 [48:03:57<39:29,  7.87s/it]
 98% 17700/18000 [48:04:02<34:14,  6.85s/it]
                                            
 98% 17700/18000 [48:04:02<34:14,  6.85s/it]
 98% 17701/18000 [48:04:18<48:34,  9.75s/it]
 98% 17702/18000 [48:04:32<53:31, 10.78s/it]
 98% 17703/18000 [48:04:42<53:10, 10.74s/it]
 98% 17704/18000 [48:04:52<50:57, 10.33s/it]
 98% 17705/18000 [48:05:00<47:35,  9.68s/it]
 98% 17706/18000 [48:05:07<43:24,  8.86s/it]
 98% 17707/18000 [48:05:13<38:43,  7.93s/it]
 98% 17708/18000 [48:05:17<33:38,  6.91s/it]
 98% 17709/18000 [48:05:32<45:37,  9.41s/it]
 98% 17710/18000 [48:05:42<46:21,  9.59s/it]
 98% 17711/18000 [48:05:50<42:58,  8.92s/it]
 98% 17712/18000 [48:05:53<34:32,  7.19s/it]
 98% 17713/18000 [48:06:09<47:46,  9.99s/it]
 98% 17714/18000 [48:06:22<52:01, 10.91s/it]
 98% 17715/18000 [48:06:33<51:30, 10.84s/it]
 98% 17716/18000 [48:06:42<49:11, 10.39s/it]
 98% 17717/18000 [48:06:50<45:26,  9.64s/it]
 98% 17718/18000 [48:06:57<41:16,  8.78s/it]
 98% 17719/18000 [48:07:03<36:36,  7.82s/it]
 98% 17720/18000 [48:07:07<31:40,  6.79s/it]
 98% 17721/18000 [48:07:24<45:20,  9.75s/it]
 98% 17722/18000 [48:07:37<50:20, 10.86s/it]
 98% 17723/18000 [48:07:48<50:13, 10.88s/it]
 98% 17724/18000 [48:07:58<48:17, 10.50s/it]
 98% 17725/18000 [48:08:06<45:12,  9.86s/it]
 98% 17726/18000 [48:08:13<41:09,  9.01s/it]
 98% 17727/18000 [48:08:19<36:36,  8.04s/it]
 98% 17728/18000 [48:08:23<31:26,  6.94s/it]
 98% 17729/18000 [48:08:40<44:17,  9.80s/it]
 98% 17730/18000 [48:08:53<48:34, 10.80s/it]
 99% 17731/18000 [48:09:04<48:51, 10.90s/it]
 99% 17732/18000 [48:09:14<46:50, 10.49s/it]
 99% 17733/18000 [48:09:22<43:34,  9.79s/it]
 99% 17734/18000 [48:09:29<39:37,  8.94s/it]
 99% 17735/18000 [48:09:34<35:01,  7.93s/it]
 99% 17736/18000 [48:09:39<30:09,  6.86s/it]
 99% 17737/18000 [48:09:55<42:38,  9.73s/it]
 99% 17738/18000 [48:10:08<46:41, 10.69s/it]
 99% 17739/18000 [48:10:19<46:23, 10.66s/it]
 99% 17740/18000 [48:10:28<44:29, 10.27s/it]
 99% 17741/18000 [48:10:36<41:42,  9.66s/it]
 99% 17742/18000 [48:10:43<38:11,  8.88s/it]
 99% 17743/18000 [48:10:49<34:16,  8.00s/it]
 99% 17744/18000 [48:10:54<29:36,  6.94s/it]
 99% 17745/18000 [48:11:09<40:00,  9.41s/it]
 99% 17746/18000 [48:11:19<40:21,  9.53s/it]
 99% 17747/18000 [48:11:26<37:20,  8.86s/it]
 99% 17748/18000 [48:11:29<30:16,  7.21s/it]
 99% 17749/18000 [48:11:46<41:52, 10.01s/it]
 99% 17750/18000 [48:11:59<46:01, 11.05s/it]
 99% 17751/18000 [48:12:10<45:27, 10.95s/it]
 99% 17752/18000 [48:12:19<43:07, 10.43s/it]
 99% 17753/18000 [48:12:27<40:02,  9.73s/it]
 99% 17754/18000 [48:12:34<36:12,  8.83s/it]
 99% 17755/18000 [48:12:40<32:06,  7.86s/it]
 99% 17756/18000 [48:12:44<27:40,  6.81s/it]
 99% 17757/18000 [48:13:01<39:27,  9.74s/it]
 99% 17758/18000 [48:13:14<43:41, 10.83s/it]
 99% 17759/18000 [48:13:25<43:30, 10.83s/it]
 99% 17760/18000 [48:13:34<41:45, 10.44s/it]
 99% 17761/18000 [48:13:43<38:55,  9.77s/it]
 99% 17762/18000 [48:13:49<35:23,  8.92s/it]
 99% 17763/18000 [48:13:55<31:21,  7.94s/it]
 99% 17764/18000 [48:13:59<26:59,  6.86s/it]
 99% 17765/18000 [48:14:16<38:19,  9.79s/it]
 99% 17766/18000 [48:14:30<42:29, 10.90s/it]
 99% 17767/18000 [48:14:41<42:27, 10.93s/it]
 99% 17768/18000 [48:14:50<40:48, 10.55s/it]
 99% 17769/18000 [48:14:59<38:09,  9.91s/it]
 99% 17770/18000 [48:15:06<34:51,  9.09s/it]
 99% 17771/18000 [48:15:12<31:09,  8.16s/it]
 99% 17772/18000 [48:15:16<26:54,  7.08s/it]
 99% 17773/18000 [48:15:33<37:29,  9.91s/it]
 99% 17774/18000 [48:15:46<40:39, 10.79s/it]
 99% 17775/18000 [48:15:56<40:21, 10.76s/it]
 99% 17776/18000 [48:16:06<38:39, 10.36s/it]
 99% 17777/18000 [48:16:14<36:11,  9.74s/it]
 99% 17778/18000 [48:16:21<32:50,  8.88s/it]
 99% 17779/18000 [48:16:27<29:18,  7.96s/it]
 99% 17780/18000 [48:16:31<25:12,  6.88s/it]
 99% 17781/18000 [48:16:46<34:04,  9.34s/it]
 99% 17782/18000 [48:16:56<34:06,  9.39s/it]
 99% 17783/18000 [48:17:03<31:35,  8.73s/it]
 99% 17784/18000 [48:17:06<25:28,  7.07s/it]
 99% 17785/18000 [48:17:23<35:34,  9.93s/it]
 99% 17786/18000 [48:17:36<38:51, 10.90s/it]
 99% 17787/18000 [48:17:46<38:07, 10.74s/it]
 99% 17788/18000 [48:17:56<36:37, 10.36s/it]
 99% 17789/18000 [48:18:04<34:17,  9.75s/it]
 99% 17790/18000 [48:18:11<31:12,  8.92s/it]
 99% 17791/18000 [48:18:17<27:39,  7.94s/it]
 99% 17792/18000 [48:18:21<23:51,  6.88s/it]
 99% 17793/18000 [48:18:38<33:37,  9.75s/it]
 99% 17794/18000 [48:18:51<37:12, 10.84s/it]
 99% 17795/18000 [48:19:02<36:51, 10.79s/it]
 99% 17796/18000 [48:19:11<35:29, 10.44s/it]
 99% 17797/18000 [48:19:20<33:09,  9.80s/it]
 99% 17798/18000 [48:19:27<30:17,  9.00s/it]
 99% 17799/18000 [48:19:33<27:04,  8.08s/it]
 99% 17800/18000 [48:19:37<23:33,  7.07s/it]
                                            
 99% 17800/18000 [48:19:37<23:33,  7.07s/it]
 99% 17801/18000 [48:19:54<32:52,  9.91s/it]
 99% 17802/18000 [48:20:07<35:52, 10.87s/it]
 99% 17803/18000 [48:20:17<35:05, 10.69s/it]
 99% 17804/18000 [48:20:26<33:27, 10.24s/it]
 99% 17805/18000 [48:20:35<31:12,  9.60s/it]
 99% 17806/18000 [48:20:41<28:19,  8.76s/it]
 99% 17807/18000 [48:20:47<25:11,  7.83s/it]
 99% 17808/18000 [48:20:51<21:40,  6.77s/it]
 99% 17809/18000 [48:21:08<30:53,  9.71s/it]
 99% 17810/18000 [48:21:21<34:26, 10.88s/it]
 99% 17811/18000 [48:21:32<34:03, 10.81s/it]
 99% 17812/18000 [48:21:42<32:48, 10.47s/it]
 99% 17813/18000 [48:21:50<30:46,  9.87s/it]
 99% 17814/18000 [48:21:57<27:45,  8.95s/it]
 99% 17815/18000 [48:22:03<24:41,  8.01s/it]
 99% 17816/18000 [48:22:07<21:12,  6.92s/it]
 99% 17817/18000 [48:22:23<28:48,  9.44s/it]
 99% 17818/18000 [48:22:32<28:41,  9.46s/it]
 99% 17819/18000 [48:22:39<26:26,  8.76s/it]
 99% 17820/18000 [48:22:43<21:25,  7.14s/it]
 99% 17821/18000 [48:22:59<29:43,  9.96s/it]
 99% 17822/18000 [48:23:12<32:26, 10.94s/it]
 99% 17823/18000 [48:23:23<31:48, 10.78s/it]
 99% 17824/18000 [48:23:32<30:13, 10.31s/it]
 99% 17825/18000 [48:23:40<28:22,  9.73s/it]
 99% 17826/18000 [48:23:47<25:44,  8.88s/it]
 99% 17827/18000 [48:23:53<22:53,  7.94s/it]
 99% 17828/18000 [48:23:57<19:39,  6.86s/it]
 99% 17829/18000 [48:24:14<27:54,  9.79s/it]
 99% 17830/18000 [48:24:27<30:52, 10.90s/it]
 99% 17831/18000 [48:24:38<30:41, 10.89s/it]
 99% 17832/18000 [48:24:48<29:21, 10.48s/it]
 99% 17833/18000 [48:24:56<27:23,  9.84s/it]
 99% 17834/18000 [48:25:03<24:52,  8.99s/it]
 99% 17835/18000 [48:25:09<22:12,  8.07s/it]
 99% 17836/18000 [48:25:14<19:07,  7.00s/it]
 99% 17837/18000 [48:25:30<26:41,  9.83s/it]
 99% 17838/18000 [48:25:43<29:07, 10.79s/it]
 99% 17839/18000 [48:25:54<28:51, 10.76s/it]
 99% 17840/18000 [48:26:03<27:31, 10.32s/it]
 99% 17841/18000 [48:26:11<25:39,  9.68s/it]
 99% 17842/18000 [48:26:18<23:17,  8.84s/it]
 99% 17843/18000 [48:26:24<20:37,  7.88s/it]
 99% 17844/18000 [48:26:28<17:46,  6.84s/it]
 99% 17845/18000 [48:26:45<25:12,  9.76s/it]
 99% 17846/18000 [48:26:58<27:53, 10.87s/it]
 99% 17847/18000 [48:27:09<27:44, 10.88s/it]
 99% 17848/18000 [48:27:19<26:29, 10.46s/it]
 99% 17849/18000 [48:27:27<24:40,  9.80s/it]
 99% 17850/18000 [48:27:34<22:29,  9.00s/it]
 99% 17851/18000 [48:27:40<19:55,  8.02s/it]
 99% 17852/18000 [48:27:44<17:03,  6.91s/it]
 99% 17853/18000 [48:27:59<23:00,  9.39s/it]
 99% 17854/18000 [48:28:09<23:01,  9.46s/it]
 99% 17855/18000 [48:28:16<21:13,  8.78s/it]
 99% 17856/18000 [48:28:19<17:10,  7.16s/it]
 99% 17857/18000 [48:28:36<24:00, 10.07s/it]
 99% 17858/18000 [48:28:50<26:26, 11.17s/it]
 99% 17859/18000 [48:29:01<26:15, 11.18s/it]
 99% 17860/18000 [48:29:11<25:08, 10.78s/it]
 99% 17861/18000 [48:29:20<23:28, 10.13s/it]
 99% 17862/18000 [48:29:27<21:24,  9.31s/it]
 99% 17863/18000 [48:29:33<18:59,  8.32s/it]
 99% 17864/18000 [48:29:38<16:15,  7.17s/it]
 99% 17865/18000 [48:29:54<22:24,  9.96s/it]
 99% 17866/18000 [48:30:07<24:21, 10.91s/it]
 99% 17867/18000 [48:30:18<24:02, 10.85s/it]
 99% 17868/18000 [48:30:27<22:50, 10.38s/it]
 99% 17869/18000 [48:30:35<21:14,  9.73s/it]
 99% 17870/18000 [48:30:42<19:22,  8.94s/it]
 99% 17871/18000 [48:30:48<17:13,  8.02s/it]
 99% 17872/18000 [48:30:53<14:50,  6.95s/it]
 99% 17873/18000 [48:31:09<20:43,  9.79s/it]
 99% 17874/18000 [48:31:22<22:44, 10.83s/it]
 99% 17875/18000 [48:31:33<22:31, 10.81s/it]
 99% 17876/18000 [48:31:43<21:31, 10.41s/it]
 99% 17877/18000 [48:31:51<20:01,  9.77s/it]
 99% 17878/18000 [48:31:58<18:06,  8.91s/it]
 99% 17879/18000 [48:32:03<15:57,  7.91s/it]
 99% 17880/18000 [48:32:08<13:43,  6.86s/it]
 99% 17881/18000 [48:32:24<19:23,  9.77s/it]
 99% 17882/18000 [48:32:38<21:18, 10.83s/it]
 99% 17883/18000 [48:32:49<21:06, 10.82s/it]
 99% 17884/18000 [48:32:58<19:57, 10.32s/it]
 99% 17885/18000 [48:33:06<18:31,  9.67s/it]
 99% 17886/18000 [48:33:12<16:38,  8.76s/it]
 99% 17887/18000 [48:33:18<14:40,  7.79s/it]
 99% 17888/18000 [48:33:22<12:34,  6.73s/it]
 99% 17889/18000 [48:33:37<16:47,  9.08s/it]
 99% 17890/18000 [48:33:46<16:48,  9.17s/it]
 99% 17891/18000 [48:33:53<15:18,  8.42s/it]
 99% 17892/18000 [48:33:56<12:24,  6.89s/it]
 99% 17893/18000 [48:34:13<17:30,  9.82s/it]
 99% 17894/18000 [48:34:26<19:14, 10.89s/it]
 99% 17895/18000 [48:34:37<18:58, 10.85s/it]
 99% 17896/18000 [48:34:46<18:04, 10.43s/it]
 99% 17897/18000 [48:34:54<16:39,  9.70s/it]
 99% 17898/18000 [48:35:01<15:06,  8.89s/it]
 99% 17899/18000 [48:35:07<13:20,  7.93s/it]
 99% 17900/18000 [48:35:12<11:29,  6.89s/it]
                                            
 99% 17900/18000 [48:35:12<11:29,  6.89s/it]
 99% 17901/18000 [48:35:28<16:13,  9.84s/it]
 99% 17902/18000 [48:35:42<17:53, 10.95s/it]
 99% 17903/18000 [48:35:53<17:38, 10.91s/it]
 99% 17904/18000 [48:36:02<16:41, 10.43s/it]
 99% 17905/18000 [48:36:10<15:29,  9.78s/it]
 99% 17906/18000 [48:36:17<14:03,  8.97s/it]
 99% 17907/18000 [48:36:23<12:29,  8.05s/it]
 99% 17908/18000 [48:36:28<10:43,  6.99s/it]
 99% 17909/18000 [48:36:44<14:52,  9.81s/it]
                                            
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+
+
  0% 0/33 [00:00<?, ?it/s]
+
  6% 2/33 [00:04<01:07,  2.18s/it]
+
  9% 3/33 [00:08<01:32,  3.08s/it]
+
 12% 4/33 [00:12<01:38,  3.41s/it]
+
 15% 5/33 [00:16<01:41,  3.64s/it]
+
 18% 6/33 [00:20<01:42,  3.81s/it]
+
 21% 7/33 [00:24<01:38,  3.80s/it]
+
 24% 8/33 [00:28<01:36,  3.85s/it]
+
 27% 9/33 [00:32<01:33,  3.88s/it]
+
 30% 10/33 [00:37<01:33,  4.07s/it]
+
 33% 11/33 [00:41<01:29,  4.06s/it]
+
 36% 12/33 [00:45<01:24,  4.03s/it]
+
 39% 13/33 [00:48<01:18,  3.95s/it]
+
 42% 14/33 [00:52<01:15,  4.00s/it]
+
 45% 15/33 [00:57<01:12,  4.02s/it]
+
 48% 16/33 [01:01<01:10,  4.15s/it]
+
 52% 17/33 [01:06<01:08,  4.28s/it]
+
 55% 18/33 [01:10<01:03,  4.20s/it]
+
 58% 19/33 [01:14<00:58,  4.20s/it]
+
 61% 20/33 [01:18<00:55,  4.27s/it]
+
 64% 21/33 [01:23<00:51,  4.31s/it]
+
 67% 22/33 [01:27<00:47,  4.31s/it]
+
 70% 23/33 [01:31<00:43,  4.35s/it]
+
 73% 24/33 [01:36<00:39,  4.36s/it]
+
 76% 25/33 [01:40<00:35,  4.40s/it]
+
 79% 26/33 [01:44<00:30,  4.34s/it]
+
 82% 27/33 [01:49<00:25,  4.30s/it]
+
 85% 28/33 [01:53<00:21,  4.33s/it]
+
 88% 29/33 [01:57<00:16,  4.08s/it]
+
 91% 30/33 [01:59<00:11,  3.68s/it]
+
 94% 31/33 [02:02<00:06,  3.39s/it]
+
 97% 32/33 [02:05<00:03,  3.16s/it]
+
                                            
+
                                   
+
+
                                   Saving model checkpoint to ./checkpoint-18000
+Configuration saved in ./checkpoint-18000/config.json
+Model weights saved in ./checkpoint-18000/pytorch_model.bin
+Configuration saved in ./checkpoint-18000/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json

logs

@@ -302,3 +302,9 @@ xpu_backend=None,
 {'loss': 0.0112, 'learning_rate': 3.976470934946075e-06, 'epoch': 480.56}
 {'loss': 0.0117, 'learning_rate': 2.92667263149321e-06, 'epoch': 483.33}
 {'loss': 0.0111, 'learning_rate': 2.037063237338044e-06, 'epoch': 486.11}
+{'eval_loss': 0.6513079404830933, 'eval_wer': 0.4059497295577474, 'eval_runtime': 131.4726, 'eval_samples_per_second': 19.844, 'eval_steps_per_second': 0.251, 'epoch': 486.11}
+{'loss': 0.0109, 'learning_rate': 1.3144256109794418e-06, 'epoch': 488.89}
+{'loss': 0.0112, 'learning_rate': 7.44394288673067e-07, 'epoch': 491.67}
+{'loss': 0.0113, 'learning_rate': 3.352551457218578e-07, 'epoch': 494.44}
+{'loss': 0.011, 'learning_rate': 8.714003300058781e-08, 'epoch': 497.22}
+{'loss': 0.0113, 'learning_rate': 1.2890911316176812e-10, 'epoch': 500.0}

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0df405d0e1ea0aa1d9a1b4a093c3d7582202187d5bac0e54b3b74e263b89570b
+oid sha256:a841c1d52b5543205b5280dafe76d3b79f4a8e662faf5145a8f24b4ff311b969
 size 1262083569

wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8e2f2f8c8a99a57b77cfe4a9a380d00f976d1c0176f17bde23883e81badc72b7
-size 121132803
+oid sha256:c8e37bb7373fe9b7546baac0b5fe983a56c0541b404dc3322d7c6f15d0a0b5f5
+size 124693876