Training in progress, step 1500

.gitattributes

@@ -39,3 +39,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.9/site-packages/torch/lib/libtorch_cpu.so filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.9/site-packages/torch/lib/libtorch_cuda.so filter=lfs diff=lfs merge=lfs -text
 .venv/lib/python3.9/site-packages/torch/lib/libtorch_python.so filter=lfs diff=lfs merge=lfs -text
+wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb filter=lfs diff=lfs merge=lfs -text

errs

@@ -499,3 +499,50 @@ Configuration saved in ./checkpoint-1000/config.json
 Model weights saved in ./checkpoint-1000/pytorch_model.bin
 Configuration saved in ./checkpoint-1000/preprocessor_config.json
 Configuration saved in ./preprocessor_config.json
+Deleting older checkpoint [checkpoint-500] due to args.save_total_limit
+
  6% 1001/18000 [2:42:47<332:31:20, 70.42s/it]
  6% 1002/18000 [2:42:53<242:23:30, 51.34s/it]
  6% 1003/18000 [2:42:59<177:33:21, 37.61s/it]
  6% 1004/18000 [2:43:03<130:36:16, 27.66s/it]
  6% 1005/18000 [2:43:18<112:33:51, 23.84s/it]
  6% 1006/18000 [2:43:28<93:00:29, 19.70s/it] 
  6% 1007/18000 [2:43:36<75:32:04, 16.00s/it]
  6% 1008/18000 [2:43:39<57:30:32, 12.18s/it]
  6% 1009/18000 [2:43:56<64:02:38, 13.57s/it]
  6% 1010/18000 [2:44:09<63:42:41, 13.50s/it]
  6% 1011/18000 [2:44:20<59:48:33, 12.67s/it]
  6% 1012/18000 [2:44:29<55:18:05, 11.72s/it]
  6% 1013/18000 [2:44:38<50:42:29, 10.75s/it]
  6% 1014/18000 [2:44:45<45:27:44,  9.64s/it]
  6% 1015/18000 [2:44:51<40:14:24,  8.53s/it]
  6% 1016/18000 [2:44:55<34:30:13,  7.31s/it]
  6% 1017/18000 [2:45:12<47:45:43, 10.12s/it]
  6% 1018/18000 [2:45:25<52:27:50, 11.12s/it]
  6% 1019/18000 [2:45:37<52:22:20, 11.10s/it]
  6% 1020/18000 [2:45:46<50:20:08, 10.67s/it]
  6% 1021/18000 [2:45:55<47:03:04,  9.98s/it]
  6% 1022/18000 [2:46:02<43:00:46,  9.12s/it]
  6% 1023/18000 [2:46:08<38:29:24,  8.16s/it]
  6% 1024/18000 [2:46:12<33:18:31,  7.06s/it]
  6% 1025/18000 [2:46:29<46:34:43,  9.88s/it]
  6% 1026/18000 [2:46:42<51:16:01, 10.87s/it]
  6% 1027/18000 [2:46:52<51:01:15, 10.82s/it]
  6% 1028/18000 [2:47:02<49:10:16, 10.43s/it]
  6% 1029/18000 [2:47:10<46:21:17,  9.83s/it]
  6% 1030/18000 [2:47:18<42:34:24,  9.03s/it]
  6% 1031/18000 [2:47:23<38:10:58,  8.10s/it]
  6% 1032/18000 [2:47:28<33:05:04,  7.02s/it]
  6% 1033/18000 [2:47:45<46:32:54,  9.88s/it]
  6% 1034/18000 [2:47:58<51:29:01, 10.92s/it]
  6% 1035/18000 [2:48:09<51:20:21, 10.89s/it]
  6% 1036/18000 [2:48:18<49:10:51, 10.44s/it]
  6% 1037/18000 [2:48:26<45:53:48,  9.74s/it]
  6% 1038/18000 [2:48:33<41:38:43,  8.84s/it]
  6% 1039/18000 [2:48:39<37:04:05,  7.87s/it]
  6% 1040/18000 [2:48:43<32:12:59,  6.84s/it]
  6% 1041/18000 [2:48:59<44:35:33,  9.47s/it]
  6% 1042/18000 [2:49:09<45:42:49,  9.70s/it]
  6% 1043/18000 [2:49:16<42:03:26,  8.93s/it]
  6% 1044/18000 [2:49:19<34:01:27,  7.22s/it]
  6% 1045/18000 [2:49:36<47:14:27, 10.03s/it]
  6% 1046/18000 [2:49:49<51:45:22, 10.99s/it]
  6% 1047/18000 [2:50:00<51:17:37, 10.89s/it]
  6% 1048/18000 [2:50:09<49:12:47, 10.45s/it]
  6% 1049/18000 [2:50:17<46:08:51,  9.80s/it]
  6% 1050/18000 [2:50:24<42:19:36,  8.99s/it]
  6% 1051/18000 [2:50:30<37:56:04,  8.06s/it]
  6% 1052/18000 [2:50:35<32:49:48,  6.97s/it]
  6% 1053/18000 [2:50:51<46:30:36,  9.88s/it]
  6% 1054/18000 [2:51:05<51:42:46, 10.99s/it]
  6% 1055/18000 [2:51:16<51:47:42, 11.00s/it]
  6% 1056/18000 [2:51:26<49:50:47, 10.59s/it]
  6% 1057/18000 [2:51:34<46:44:06,  9.93s/it]
  6% 1058/18000 [2:51:41<42:42:34,  9.08s/it]
  6% 1059/18000 [2:51:47<37:47:46,  8.03s/it]
  6% 1060/18000 [2:51:51<32:38:48,  6.94s/it]
  6% 1061/18000 [2:52:08<46:24:39,  9.86s/it]
  6% 1062/18000 [2:52:21<50:34:40, 10.75s/it]
  6% 1063/18000 [2:52:31<50:27:38, 10.73s/it]
  6% 1064/18000 [2:52:41<48:36:19, 10.33s/it]
  6% 1065/18000 [2:52:49<45:49:15,  9.74s/it]
  6% 1066/18000 [2:52:56<41:45:48,  8.88s/it]
  6% 1067/18000 [2:53:02<37:23:45,  7.95s/it]
  6% 1068/18000 [2:53:06<32:16:55,  6.86s/it]
  6% 1069/18000 [2:53:23<45:58:10,  9.77s/it]
  6% 1070/18000 [2:53:36<51:18:29, 10.91s/it]
  6% 1071/18000 [2:53:47<51:13:27, 10.89s/it]
  6% 1072/18000 [2:53:56<49:01:16, 10.43s/it]
  6% 1073/18000 [2:54:05<45:58:06,  9.78s/it]
  6% 1074/18000 [2:54:12<42:03:59,  8.95s/it]
  6% 1075/18000 [2:54:18<37:56:19,  8.07s/it]
  6% 1076/18000 [2:54:22<32:53:17,  7.00s/it]
  6% 1077/18000 [2:54:37<44:22:46,  9.44s/it]
  6% 1078/18000 [2:54:47<44:34:39,  9.48s/it]
  6% 1079/18000 [2:54:54<41:00:17,  8.72s/it]
  6% 1080/18000 [2:54:57<33:18:16,  7.09s/it]
  6% 1081/18000 [2:55:14<46:33:47,  9.91s/it]
  6% 1082/18000 [2:55:26<50:41:32, 10.79s/it]
  6% 1083/18000 [2:55:37<50:38:27, 10.78s/it]
  6% 1084/18000 [2:55:46<48:21:00, 10.29s/it]
  6% 1085/18000 [2:55:55<45:37:19,  9.71s/it]
  6% 1086/18000 [2:56:02<41:48:24,  8.90s/it]
  6% 1087/18000 [2:56:08<37:25:47,  7.97s/it]
  6% 1088/18000 [2:56:12<32:25:26,  6.90s/it]
  6% 1089/18000 [2:56:28<45:55:32,  9.78s/it]
  6% 1090/18000 [2:56:42<50:39:23, 10.78s/it]
  6% 1091/18000 [2:56:52<50:21:19, 10.72s/it]
  6% 1092/18000 [2:57:01<47:31:50, 10.12s/it]
  6% 1093/18000 [2:57:09<44:45:32,  9.53s/it]
  6% 1094/18000 [2:57:16<41:12:21,  8.77s/it]
  6% 1095/18000 [2:57:22<37:01:50,  7.89s/it]
  6% 1096/18000 [2:57:26<32:07:13,  6.84s/it]
  6% 1097/18000 [2:57:43<45:45:06,  9.74s/it]
  6% 1098/18000 [2:57:56<50:37:30, 10.78s/it]
  6% 1099/18000 [2:58:07<50:47:44, 10.82s/it]
  6% 1100/18000 [2:58:16<48:43:14, 10.38s/it]
                                             
  6% 1100/18000 [2:58:16<48:43:14, 10.38s/it]
  6% 1101/18000 [2:58:24<45:43:56,  9.74s/it]
  6% 1102/18000 [2:58:31<41:28:54,  8.84s/it]
  6% 1103/18000 [2:58:37<37:00:22,  7.88s/it]
  6% 1104/18000 [2:58:41<32:01:26,  6.82s/it]
  6% 1105/18000 [2:58:58<45:56:34,  9.79s/it]
  6% 1106/18000 [2:59:11<51:06:15, 10.89s/it]
  6% 1107/18000 [2:59:22<51:03:28, 10.88s/it]
  6% 1108/18000 [2:59:31<48:39:05, 10.37s/it]
  6% 1109/18000 [2:59:40<45:49:26,  9.77s/it]
  6% 1110/18000 [2:59:47<42:05:05,  8.97s/it]
  6% 1111/18000 [2:59:53<37:43:29,  8.04s/it]
  6% 1112/18000 [2:59:57<32:47:40,  6.99s/it]
  6% 1113/18000 [3:00:13<44:52:16,  9.57s/it]
  6% 1114/18000 [3:00:23<45:51:50,  9.78s/it]
  6% 1115/18000 [3:00:31<42:40:47,  9.10s/it]
  6% 1116/18000 [3:00:34<34:44:04,  7.41s/it]
  6% 1117/18000 [3:00:51<47:53:34, 10.21s/it]
  6% 1118/18000 [3:01:04<51:23:35, 10.96s/it]
  6% 1119/18000 [3:01:14<50:49:20, 10.84s/it]
  6% 1120/18000 [3:01:23<48:41:39, 10.39s/it]
  6% 1121/18000 [3:01:32<45:38:04,  9.73s/it]
  6% 1122/18000 [3:01:39<41:43:28,  8.90s/it]
  6% 1123/18000 [3:01:44<37:18:42,  7.96s/it]
  6% 1124/18000 [3:01:49<32:28:23,  6.93s/it]
  6% 1125/18000 [3:02:05<45:45:25,  9.76s/it]
  6% 1126/18000 [3:02:18<50:34:05, 10.79s/it]
  6% 1127/18000 [3:02:29<50:34:29, 10.79s/it]
  6% 1128/18000 [3:02:39<48:45:26, 10.40s/it]
  6% 1129/18000 [3:02:47<45:52:00,  9.79s/it]
  6% 1130/18000 [3:02:54<41:57:38,  8.95s/it]
  6% 1131/18000 [3:03:00<37:31:14,  8.01s/it]
  6% 1132/18000 [3:03:04<32:27:51,  6.93s/it]
  6% 1133/18000 [3:03:21<45:53:51,  9.80s/it]
  6% 1134/18000 [3:03:34<51:05:57, 10.91s/it]
  6% 1135/18000 [3:03:45<51:05:28, 10.91s/it]
  6% 1136/18000 [3:03:55<48:48:54, 10.42s/it]
  6% 1137/18000 [3:04:03<45:53:05,  9.80s/it]
  6% 1138/18000 [3:04:10<41:58:27,  8.96s/it]
  6% 1139/18000 [3:04:16<37:39:53,  8.04s/it]
  6% 1140/18000 [3:04:20<32:29:38,  6.94s/it]
  6% 1141/18000 [3:04:36<45:09:50,  9.64s/it]
  6% 1142/18000 [3:04:49<50:25:12, 10.77s/it]
  6% 1143/18000 [3:05:00<50:26:13, 10.77s/it]
  6% 1144/18000 [3:05:10<48:52:39, 10.44s/it]
  6% 1145/18000 [3:05:18<45:51:05,  9.79s/it]
  6% 1146/18000 [3:05:25<41:53:37,  8.95s/it]
  6% 1147/18000 [3:05:31<37:33:01,  8.02s/it]
  6% 1148/18000 [3:05:36<32:35:58,  6.96s/it]
  6% 1149/18000 [3:05:51<44:17:06,  9.46s/it]
  6% 1150/18000 [3:06:01<45:13:07,  9.66s/it]
  6% 1151/18000 [3:06:08<41:55:31,  8.96s/it]
  6% 1152/18000 [3:06:12<33:54:29,  7.25s/it]
  6% 1153/18000 [3:06:28<47:20:41, 10.12s/it]
  6% 1154/18000 [3:06:42<52:13:10, 11.16s/it]
  6% 1155/18000 [3:06:53<52:00:14, 11.11s/it]
  6% 1156/18000 [3:07:02<49:46:57, 10.64s/it]
  6% 1157/18000 [3:07:11<46:20:24,  9.90s/it]
  6% 1158/18000 [3:07:18<42:20:22,  9.05s/it]
  6% 1159/18000 [3:07:23<37:35:16,  8.03s/it]
  6% 1160/18000 [3:07:28<32:25:12,  6.93s/it]
  6% 1161/18000 [3:07:44<45:59:40,  9.83s/it]
  6% 1162/18000 [3:07:58<50:58:53, 10.90s/it]
  6% 1163/18000 [3:08:08<50:45:36, 10.85s/it]
  6% 1164/18000 [3:08:18<48:44:37, 10.42s/it]
  6% 1165/18000 [3:08:26<45:33:56,  9.74s/it]
  6% 1166/18000 [3:08:33<41:39:58,  8.91s/it]
  6% 1167/18000 [3:08:39<37:01:39,  7.92s/it]
  6% 1168/18000 [3:08:43<32:05:28,  6.86s/it]
  6% 1169/18000 [3:09:00<45:43:15,  9.78s/it]
  6% 1170/18000 [3:09:13<50:49:47, 10.87s/it]
  7% 1171/18000 [3:09:24<50:41:54, 10.85s/it]
  7% 1172/18000 [3:09:33<48:55:16, 10.47s/it]
  7% 1173/18000 [3:09:42<45:51:15,  9.81s/it]
  7% 1174/18000 [3:09:49<41:50:47,  8.95s/it]
  7% 1175/18000 [3:09:54<37:28:36,  8.02s/it]
  7% 1176/18000 [3:09:59<32:36:36,  6.98s/it]
  7% 1177/18000 [3:10:15<45:43:35,  9.79s/it]
  7% 1178/18000 [3:10:28<50:23:31, 10.78s/it]
  7% 1179/18000 [3:10:39<50:25:20, 10.79s/it]
  7% 1180/18000 [3:10:49<48:28:08, 10.37s/it]
  7% 1181/18000 [3:10:57<45:41:59,  9.78s/it]
  7% 1182/18000 [3:11:04<41:46:12,  8.94s/it]
  7% 1183/18000 [3:11:10<37:41:14,  8.07s/it]
  7% 1184/18000 [3:11:15<32:43:14,  7.00s/it]
  7% 1185/18000 [3:11:30<43:57:12,  9.41s/it]
  7% 1186/18000 [3:11:40<44:42:58,  9.57s/it]
  7% 1187/18000 [3:11:47<41:14:53,  8.83s/it]
  7% 1188/18000 [3:11:50<33:28:14,  7.17s/it]
  7% 1189/18000 [3:12:07<46:59:48, 10.06s/it]
  7% 1190/18000 [3:12:20<51:18:40, 10.99s/it]
  7% 1191/18000 [3:12:30<50:40:15, 10.85s/it]
  7% 1192/18000 [3:12:40<48:25:52, 10.37s/it]
  7% 1193/18000 [3:12:48<45:20:48,  9.71s/it]
  7% 1194/18000 [3:12:55<41:33:26,  8.90s/it]
  7% 1195/18000 [3:13:01<37:17:00,  7.99s/it]
  7% 1196/18000 [3:13:05<32:20:35,  6.93s/it]
  7% 1197/18000 [3:13:22<45:35:54,  9.77s/it]
  7% 1198/18000 [3:13:35<50:40:20, 10.86s/it]
  7% 1199/18000 [3:13:46<50:44:51, 10.87s/it]
  7% 1200/18000 [3:13:55<48:31:00, 10.40s/it]
                                             
  7% 1200/18000 [3:13:55<48:31:00, 10.40s/it]
  7% 1201/18000 [3:14:04<45:36:29,  9.77s/it]
  7% 1202/18000 [3:14:11<41:50:40,  8.97s/it]
  7% 1203/18000 [3:14:16<37:30:53,  8.04s/it]
  7% 1204/18000 [3:14:21<32:23:13,  6.94s/it]
  7% 1205/18000 [3:14:37<45:55:13,  9.84s/it]
  7% 1206/18000 [3:14:51<50:50:40, 10.90s/it]
  7% 1207/18000 [3:15:02<50:43:05, 10.87s/it]
  7% 1208/18000 [3:15:11<48:32:04, 10.41s/it]
  7% 1209/18000 [3:15:19<45:45:04,  9.81s/it]
  7% 1210/18000 [3:15:26<41:52:40,  8.98s/it]
  7% 1211/18000 [3:15:32<37:25:32,  8.03s/it]
  7% 1212/18000 [3:15:37<32:15:59,  6.92s/it]
  7% 1213/18000 [3:15:53<45:48:24,  9.82s/it]
  7% 1214/18000 [3:16:06<50:05:51, 10.74s/it]
  7% 1215/18000 [3:16:17<50:20:49, 10.80s/it]
  7% 1216/18000 [3:16:27<48:40:31, 10.44s/it]
  7% 1217/18000 [3:16:35<45:40:17,  9.80s/it]
  7% 1218/18000 [3:16:42<41:47:34,  8.97s/it]
  7% 1219/18000 [3:16:48<37:23:11,  8.02s/it]
  7% 1220/18000 [3:16:52<32:35:07,  6.99s/it]
  7% 1221/18000 [3:17:08<44:29:45,  9.55s/it]
  7% 1222/18000 [3:17:18<45:08:29,  9.69s/it]
  7% 1223/18000 [3:17:25<41:19:25,  8.87s/it]
  7% 1224/18000 [3:17:28<33:18:08,  7.15s/it]
  7% 1225/18000 [3:17:45<46:35:52, 10.00s/it]
  7% 1226/18000 [3:17:58<50:58:00, 10.94s/it]
  7% 1227/18000 [3:18:08<50:40:35, 10.88s/it]
  7% 1228/18000 [3:18:18<48:09:39, 10.34s/it]
  7% 1229/18000 [3:18:26<45:15:03,  9.71s/it]
  7% 1230/18000 [3:18:33<41:28:45,  8.90s/it]
  7% 1231/18000 [3:18:39<37:13:34,  7.99s/it]
  7% 1232/18000 [3:18:43<32:13:52,  6.92s/it]
  7% 1233/18000 [3:18:59<45:24:08,  9.75s/it]
  7% 1234/18000 [3:19:13<50:34:54, 10.86s/it]
  7% 1235/18000 [3:19:23<50:06:46, 10.76s/it]
  7% 1236/18000 [3:19:33<48:18:12, 10.37s/it]
  7% 1237/18000 [3:19:41<45:25:15,  9.75s/it]
  7% 1238/18000 [3:19:48<41:39:02,  8.95s/it]
  7% 1239/18000 [3:19:54<37:35:54,  8.08s/it]
  7% 1240/18000 [3:19:59<32:45:04,  7.03s/it]
  7% 1241/18000 [3:20:15<46:02:36,  9.89s/it]
  7% 1242/18000 [3:20:29<50:42:03, 10.89s/it]
  7% 1243/18000 [3:20:40<50:36:19, 10.87s/it]
  7% 1244/18000 [3:20:49<48:49:03, 10.49s/it]
  7% 1245/18000 [3:20:57<45:45:11,  9.83s/it]
  7% 1246/18000 [3:21:04<41:53:36,  9.00s/it]
  7% 1247/18000 [3:21:10<37:30:25,  8.06s/it]
  7% 1248/18000 [3:21:15<32:22:03,  6.96s/it]
  7% 1249/18000 [3:21:31<45:48:52,  9.85s/it]
  7% 1250/18000 [3:21:45<51:15:56, 11.02s/it]
  7% 1251/18000 [3:21:56<51:12:45, 11.01s/it]
  7% 1252/18000 [3:22:05<48:42:56, 10.47s/it]
  7% 1253/18000 [3:22:14<45:39:15,  9.81s/it]
  7% 1254/18000 [3:22:20<41:37:02,  8.95s/it]
  7% 1255/18000 [3:22:26<37:01:47,  7.96s/it]
  7% 1256/18000 [3:22:31<32:02:46,  6.89s/it]
  7% 1257/18000 [3:22:46<43:44:48,  9.41s/it]
  7% 1258/18000 [3:22:56<44:48:00,  9.63s/it]
  7% 1259/18000 [3:23:03<41:31:49,  8.93s/it]
  7% 1260/18000 [3:23:07<33:46:19,  7.26s/it]
  7% 1261/18000 [3:23:23<46:48:24, 10.07s/it]
  7% 1262/18000 [3:23:36<50:53:39, 10.95s/it]
  7% 1263/18000 [3:23:47<50:22:24, 10.83s/it]
  7% 1264/18000 [3:23:56<48:16:32, 10.38s/it]
  7% 1265/18000 [3:24:04<45:15:04,  9.73s/it]
  7% 1266/18000 [3:24:11<41:25:24,  8.91s/it]
  7% 1267/18000 [3:24:17<36:55:00,  7.94s/it]
  7% 1268/18000 [3:24:21<32:01:07,  6.89s/it]
  7% 1269/18000 [3:24:38<45:24:08,  9.77s/it]
  7% 1270/18000 [3:24:51<50:32:46, 10.88s/it]
  7% 1271/18000 [3:25:02<50:14:04, 10.81s/it]
  7% 1272/18000 [3:25:11<48:18:18, 10.40s/it]
  7% 1273/18000 [3:25:20<45:13:00,  9.73s/it]
  7% 1274/18000 [3:25:27<41:13:43,  8.87s/it]
  7% 1275/18000 [3:25:32<36:55:36,  7.95s/it]
  7% 1276/18000 [3:25:37<31:55:28,  6.87s/it]
  7% 1277/18000 [3:25:53<45:34:40,  9.81s/it]
  7% 1278/18000 [3:26:07<51:00:53, 10.98s/it]
  7% 1279/18000 [3:26:18<50:40:53, 10.91s/it]
  7% 1280/18000 [3:26:28<48:58:53, 10.55s/it]
  7% 1281/18000 [3:26:36<46:07:32,  9.93s/it]
  7% 1282/18000 [3:26:43<42:17:35,  9.11s/it]
  7% 1283/18000 [3:26:49<38:08:34,  8.21s/it]
  7% 1284/18000 [3:26:54<33:10:41,  7.15s/it]
  7% 1285/18000 [3:27:11<46:37:04, 10.04s/it]
  7% 1286/18000 [3:27:24<51:24:11, 11.07s/it]
  7% 1287/18000 [3:27:35<51:10:50, 11.02s/it]
  7% 1288/18000 [3:27:44<48:38:05, 10.48s/it]
  7% 1289/18000 [3:27:53<45:33:23,  9.81s/it]
  7% 1290/18000 [3:28:00<41:39:53,  8.98s/it]
  7% 1291/18000 [3:28:05<37:15:15,  8.03s/it]
  7% 1292/18000 [3:28:10<32:07:44,  6.92s/it]
  7% 1293/18000 [3:28:25<43:42:08,  9.42s/it]
  7% 1294/18000 [3:28:35<44:31:49,  9.60s/it]
  7% 1295/18000 [3:28:42<41:18:37,  8.90s/it]
  7% 1296/18000 [3:28:46<33:32:35,  7.23s/it]
  7% 1297/18000 [3:29:02<46:47:00, 10.08s/it]
  7% 1298/18000 [3:29:16<51:36:16, 11.12s/it]
  7% 1299/18000 [3:29:27<51:34:01, 11.12s/it]
  7% 1300/18000 [3:29:37<49:46:53, 10.73s/it]
                                             
  7% 1300/18000 [3:29:37<49:46:53, 10.73s/it]
  7% 1301/18000 [3:29:45<46:38:07, 10.05s/it]
  7% 1302/18000 [3:29:53<42:42:32,  9.21s/it]
  7% 1303/18000 [3:29:59<38:24:35,  8.28s/it]
  7% 1304/18000 [3:30:03<33:26:22,  7.21s/it]
  7% 1305/18000 [3:30:20<46:15:50,  9.98s/it]
  7% 1306/18000 [3:30:33<50:30:47, 10.89s/it]
  7% 1307/18000 [3:30:43<50:05:16, 10.80s/it]
  7% 1308/18000 [3:30:53<47:37:10, 10.27s/it]
  7% 1309/18000 [3:31:00<44:22:23,  9.57s/it]
  7% 1310/18000 [3:31:07<40:28:34,  8.73s/it]
  7% 1311/18000 [3:31:13<36:04:12,  7.78s/it]
  7% 1312/18000 [3:31:17<31:17:56,  6.75s/it]
  7% 1313/18000 [3:31:34<44:58:36,  9.70s/it]
  7% 1314/18000 [3:31:47<49:18:38, 10.64s/it]
  7% 1315/18000 [3:31:57<49:43:54, 10.73s/it]
  7% 1316/18000 [3:32:07<48:08:55, 10.39s/it]
  7% 1317/18000 [3:32:15<45:14:56,  9.76s/it]
  7% 1318/18000 [3:32:22<41:25:50,  8.94s/it]
  7% 1319/18000 [3:32:28<37:01:08,  7.99s/it]
  7% 1320/18000 [3:32:33<32:08:20,  6.94s/it]
  7% 1321/18000 [3:32:49<45:21:54,  9.79s/it]
  7% 1322/18000 [3:33:02<50:16:46, 10.85s/it]
  7% 1323/18000 [3:33:13<50:13:05, 10.84s/it]
  7% 1324/18000 [3:33:23<48:20:29, 10.44s/it]
  7% 1325/18000 [3:33:31<45:20:02,  9.79s/it]
  7% 1326/18000 [3:33:38<41:26:43,  8.95s/it]
  7% 1327/18000 [3:33:44<36:46:19,  7.94s/it]
  7% 1328/18000 [3:33:48<31:36:35,  6.83s/it]
  7% 1329/18000 [3:34:03<43:10:05,  9.32s/it]
  7% 1330/18000 [3:34:13<43:57:19,  9.49s/it]
  7% 1331/18000 [3:34:20<40:54:39,  8.84s/it]
  7% 1332/18000 [3:34:24<33:26:36,  7.22s/it]
  7% 1333/18000 [3:34:40<46:20:44, 10.01s/it]
  7% 1334/18000 [3:34:53<49:54:40, 10.78s/it]
  7% 1335/18000 [3:35:03<49:29:47, 10.69s/it]
  7% 1336/18000 [3:35:12<47:18:39, 10.22s/it]
  7% 1337/18000 [3:35:20<44:22:39,  9.59s/it]
  7% 1338/18000 [3:35:27<40:38:14,  8.78s/it]
  7% 1339/18000 [3:35:33<36:17:47,  7.84s/it]
  7% 1340/18000 [3:35:37<31:28:59,  6.80s/it]
  7% 1341/18000 [3:35:54<44:51:50,  9.70s/it]
  7% 1342/18000 [3:36:06<48:36:46, 10.51s/it]
  7% 1343/18000 [3:36:17<48:40:34, 10.52s/it]
  7% 1344/18000 [3:36:26<47:04:04, 10.17s/it]
  7% 1345/18000 [3:36:34<44:26:24,  9.61s/it]
  7% 1346/18000 [3:36:41<40:44:20,  8.81s/it]
  7% 1347/18000 [3:36:47<36:40:31,  7.93s/it]
  7% 1348/18000 [3:36:52<31:52:06,  6.89s/it]
  7% 1349/18000 [3:37:08<45:20:09,  9.80s/it]
  8% 1350/18000 [3:37:22<51:22:47, 11.11s/it]
  8% 1351/18000 [3:37:34<51:43:02, 11.18s/it]
  8% 1352/18000 [3:37:44<49:57:27, 10.80s/it]
  8% 1353/18000 [3:37:52<46:50:45, 10.13s/it]
  8% 1354/18000 [3:37:59<42:43:40,  9.24s/it]
  8% 1355/18000 [3:38:05<38:04:52,  8.24s/it]
  8% 1356/18000 [3:38:10<32:40:26,  7.07s/it]
  8% 1357/18000 [3:38:26<45:59:30,  9.95s/it]
  8% 1358/18000 [3:38:40<51:01:44, 11.04s/it]
  8% 1359/18000 [3:38:51<50:54:09, 11.01s/it]
  8% 1360/18000 [3:39:00<48:36:24, 10.52s/it]
  8% 1361/18000 [3:39:09<45:41:50,  9.89s/it]
  8% 1362/18000 [3:39:16<41:32:45,  8.99s/it]
  8% 1363/18000 [3:39:21<37:11:14,  8.05s/it]
  8% 1364/18000 [3:39:26<32:09:00,  6.96s/it]
  8% 1365/18000 [3:39:41<43:31:00,  9.42s/it]
  8% 1366/18000 [3:39:51<44:08:36,  9.55s/it]
  8% 1367/18000 [3:39:58<40:45:56,  8.82s/it]
  8% 1368/18000 [3:40:01<32:46:34,  7.09s/it]
  8% 1369/18000 [3:40:18<46:09:28,  9.99s/it]
  8% 1370/18000 [3:40:31<51:09:38, 11.08s/it]
  8% 1371/18000 [3:40:42<50:57:19, 11.03s/it]
  8% 1372/18000 [3:40:52<49:05:27, 10.63s/it]
  8% 1373/18000 [3:41:00<46:04:28,  9.98s/it]
  8% 1374/18000 [3:41:07<41:55:53,  9.08s/it]
  8% 1375/18000 [3:41:13<37:33:22,  8.13s/it]
  8% 1376/18000 [3:41:18<32:31:30,  7.04s/it]
  8% 1377/18000 [3:41:34<45:40:15,  9.89s/it]
  8% 1378/18000 [3:41:48<50:43:21, 10.99s/it]
  8% 1379/18000 [3:41:59<50:37:04, 10.96s/it]
  8% 1380/18000 [3:42:08<48:26:44, 10.49s/it]
  8% 1381/18000 [3:42:17<45:35:38,  9.88s/it]
  8% 1382/18000 [3:42:24<41:46:48,  9.05s/it]
  8% 1383/18000 [3:42:30<37:22:30,  8.10s/it]
  8% 1384/18000 [3:42:34<32:26:46,  7.03s/it]
  8% 1385/18000 [3:42:51<45:30:34,  9.86s/it]
  8% 1386/18000 [3:43:04<50:05:12, 10.85s/it]
  8% 1387/18000 [3:43:15<49:55:19, 10.82s/it]
  8% 1388/18000 [3:43:24<47:54:12, 10.38s/it]
  8% 1389/18000 [3:43:32<45:01:34,  9.76s/it]
  8% 1390/18000 [3:43:39<41:07:33,  8.91s/it]
  8% 1391/18000 [3:43:45<36:43:12,  7.96s/it]
  8% 1392/18000 [3:43:49<31:42:30,  6.87s/it]
  8% 1393/18000 [3:44:06<44:58:20,  9.75s/it]
  8% 1394/18000 [3:44:19<49:50:12, 10.80s/it]
  8% 1395/18000 [3:44:30<49:33:26, 10.74s/it]
  8% 1396/18000 [3:44:39<47:11:04, 10.23s/it]
  8% 1397/18000 [3:44:47<44:17:29,  9.60s/it]
  8% 1398/18000 [3:44:54<40:35:03,  8.80s/it]
  8% 1399/18000 [3:44:59<36:24:38,  7.90s/it]
  8% 1400/18000 [3:45:04<31:46:40,  6.89s/it]
                                             
  8% 1400/18000 [3:45:04<31:46:40,  6.89s/it]
  8% 1401/18000 [3:45:19<43:12:04,  9.37s/it]
  8% 1402/18000 [3:45:29<44:03:06,  9.55s/it]
  8% 1403/18000 [3:45:36<40:55:08,  8.88s/it]
  8% 1404/18000 [3:45:40<33:07:39,  7.19s/it]
  8% 1405/18000 [3:45:56<46:14:46, 10.03s/it]
  8% 1406/18000 [3:46:10<50:46:40, 11.02s/it]
  8% 1407/18000 [3:46:21<50:40:51, 11.00s/it]
  8% 1408/18000 [3:46:30<48:25:39, 10.51s/it]
  8% 1409/18000 [3:46:38<45:23:36,  9.85s/it]
  8% 1410/18000 [3:46:45<41:31:45,  9.01s/it]
  8% 1411/18000 [3:46:51<36:53:29,  8.01s/it]
  8% 1412/18000 [3:46:55<31:59:37,  6.94s/it]
  8% 1413/18000 [3:47:12<45:19:38,  9.84s/it]
  8% 1414/18000 [3:47:26<50:26:34, 10.95s/it]
  8% 1415/18000 [3:47:36<49:38:58, 10.78s/it]
  8% 1416/18000 [3:47:45<47:47:17, 10.37s/it]
  8% 1417/18000 [3:47:53<44:32:51,  9.67s/it]
  8% 1418/18000 [3:48:00<40:50:45,  8.87s/it]
  8% 1419/18000 [3:48:06<36:40:01,  7.96s/it]
  8% 1420/18000 [3:48:11<31:55:59,  6.93s/it]
  8% 1421/18000 [3:48:27<45:10:48,  9.81s/it]
  8% 1422/18000 [3:48:41<50:05:36, 10.88s/it]
  8% 1423/18000 [3:48:51<49:50:50, 10.83s/it]
  8% 1424/18000 [3:49:01<47:52:14, 10.40s/it]
  8% 1425/18000 [3:49:09<44:56:10,  9.76s/it]
  8% 1426/18000 [3:49:16<41:04:25,  8.92s/it]
  8% 1427/18000 [3:49:22<36:42:37,  7.97s/it]
  8% 1428/18000 [3:49:26<31:44:22,  6.89s/it]
  8% 1429/18000 [3:49:43<44:53:01,  9.75s/it]
  8% 1430/18000 [3:49:56<49:38:34, 10.79s/it]
  8% 1431/18000 [3:50:07<49:36:37, 10.78s/it]
  8% 1432/18000 [3:50:16<47:46:18, 10.38s/it]
  8% 1433/18000 [3:50:24<44:53:16,  9.75s/it]
  8% 1434/18000 [3:50:31<40:56:32,  8.90s/it]
  8% 1435/18000 [3:50:37<36:44:54,  7.99s/it]
  8% 1436/18000 [3:50:42<31:56:59,  6.94s/it]
  8% 1437/18000 [3:50:57<43:15:32,  9.40s/it]
  8% 1438/18000 [3:51:07<44:11:44,  9.61s/it]
  8% 1439/18000 [3:51:14<41:00:46,  8.92s/it]
  8% 1440/18000 [3:51:17<33:16:22,  7.23s/it]
  8% 1441/18000 [3:51:34<46:13:36, 10.05s/it]
  8% 1442/18000 [3:51:48<51:08:15, 11.12s/it]
  8% 1443/18000 [3:51:59<50:57:19, 11.08s/it]
  8% 1444/18000 [3:52:08<48:53:56, 10.63s/it]
  8% 1445/18000 [3:52:17<45:57:28,  9.99s/it]
  8% 1446/18000 [3:52:24<42:04:32,  9.15s/it]
  8% 1447/18000 [3:52:30<37:34:28,  8.17s/it]
  8% 1448/18000 [3:52:34<32:28:17,  7.06s/it]
  8% 1449/18000 [3:52:51<45:38:42,  9.93s/it]
  8% 1450/18000 [3:53:05<50:45:26, 11.04s/it]
  8% 1451/18000 [3:53:15<50:25:35, 10.97s/it]
  8% 1452/18000 [3:53:25<48:16:02, 10.50s/it]
  8% 1453/18000 [3:53:33<44:44:34,  9.73s/it]
  8% 1454/18000 [3:53:40<40:56:23,  8.91s/it]
  8% 1455/18000 [3:53:45<36:25:16,  7.92s/it]
  8% 1456/18000 [3:53:50<31:35:07,  6.87s/it]
  8% 1457/18000 [3:54:06<45:11:04,  9.83s/it]
  8% 1458/18000 [3:54:20<50:09:35, 10.92s/it]
  8% 1459/18000 [3:54:31<50:06:12, 10.90s/it]
  8% 1460/18000 [3:54:40<47:51:13, 10.42s/it]
  8% 1461/18000 [3:54:48<44:59:02,  9.79s/it]
  8% 1462/18000 [3:54:55<41:07:47,  8.95s/it]
  8% 1463/18000 [3:55:01<36:48:49,  8.01s/it]
  8% 1464/18000 [3:55:06<31:52:40,  6.94s/it]
  8% 1465/18000 [3:55:22<45:08:07,  9.83s/it]
  8% 1466/18000 [3:55:36<50:02:21, 10.90s/it]
  8% 1467/18000 [3:55:46<49:50:45, 10.85s/it]
  8% 1468/18000 [3:55:56<47:51:22, 10.42s/it]
  8% 1469/18000 [3:56:04<44:56:13,  9.79s/it]
  8% 1470/18000 [3:56:11<41:12:28,  8.97s/it]
  8% 1471/18000 [3:56:17<36:57:37,  8.05s/it]
  8% 1472/18000 [3:56:22<32:02:45,  6.98s/it]
  8% 1473/18000 [3:56:36<42:58:14,  9.36s/it]
  8% 1474/18000 [3:56:46<43:36:05,  9.50s/it]
  8% 1475/18000 [3:56:53<40:05:05,  8.73s/it]
  8% 1476/18000 [3:56:57<32:36:39,  7.10s/it]
  8% 1477/18000 [3:57:12<44:38:53,  9.73s/it]
  8% 1478/18000 [3:57:26<49:26:58, 10.77s/it]
  8% 1479/18000 [3:57:36<49:25:31, 10.77s/it]
  8% 1480/18000 [3:57:45<47:06:45, 10.27s/it]
  8% 1481/18000 [3:57:53<44:00:28,  9.59s/it]
  8% 1482/18000 [3:58:00<40:05:41,  8.74s/it]
  8% 1483/18000 [3:58:06<35:45:27,  7.79s/it]
  8% 1484/18000 [3:58:10<30:59:41,  6.76s/it]
  8% 1485/18000 [3:58:27<44:26:35,  9.69s/it]
  8% 1486/18000 [3:58:40<49:44:47, 10.84s/it]
  8% 1487/18000 [3:58:51<49:41:19, 10.83s/it]
  8% 1488/18000 [3:59:00<47:47:58, 10.42s/it]
  8% 1489/18000 [3:59:09<44:53:03,  9.79s/it]
  8% 1490/18000 [3:59:16<40:57:25,  8.93s/it]
  8% 1491/18000 [3:59:22<36:42:37,  8.01s/it]
  8% 1492/18000 [3:59:26<31:51:22,  6.95s/it]
  8% 1493/18000 [3:59:43<44:59:31,  9.81s/it]
  8% 1494/18000 [3:59:56<49:43:22, 10.84s/it]
  8% 1495/18000 [4:00:06<49:22:32, 10.77s/it]
  8% 1496/18000 [4:00:16<47:18:26, 10.32s/it]
  8% 1497/18000 [4:00:23<43:49:00,  9.56s/it]
  8% 1498/18000 [4:00:30<39:54:12,  8.71s/it]
  8% 1499/18000 [4:00:36<35:43:31,  7.79s/it]
  8% 1500/18000 [4:00:40<31:18:16,  6.83s/it]
                                             
  8% 1500/18000 [4:00:40<31:18:16,  6.83s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+
+
  0% 0/33 [00:00<?, ?it/s]
+
  6% 2/33 [00:04<01:08,  2.22s/it]
+
  9% 3/33 [00:08<01:33,  3.11s/it]
+
 12% 4/33 [00:12<01:39,  3.42s/it]
+
 15% 5/33 [00:16<01:41,  3.64s/it]
+
 18% 6/33 [00:20<01:42,  3.80s/it]
+
 21% 7/33 [00:24<01:38,  3.80s/it]
+
 24% 8/33 [00:28<01:36,  3.85s/it]
+
 27% 9/33 [00:32<01:33,  3.88s/it]
+
 30% 10/33 [00:37<01:33,  4.07s/it]
+
 33% 11/33 [00:41<01:29,  4.05s/it]
+
 36% 12/33 [00:45<01:24,  4.02s/it]
+
 39% 13/33 [00:48<01:19,  3.96s/it]
+
 42% 14/33 [00:53<01:16,  4.01s/it]
+
 45% 15/33 [00:57<01:12,  4.03s/it]
+
 48% 16/33 [01:01<01:10,  4.16s/it]
+
 52% 17/33 [01:06<01:08,  4.29s/it]
+
 55% 18/33 [01:10<01:03,  4.22s/it]
+
 58% 19/33 [01:14<00:59,  4.23s/it]
+
 61% 20/33 [01:18<00:55,  4.30s/it]
+
 64% 21/33 [01:23<00:52,  4.34s/it]
+
 67% 22/33 [01:27<00:47,  4.33s/it]
+
 70% 23/33 [01:32<00:43,  4.38s/it]
+
 73% 24/33 [01:36<00:39,  4.39s/it]
+
 76% 25/33 [01:41<00:35,  4.43s/it]
+
 79% 26/33 [01:45<00:30,  4.37s/it]
+
 82% 27/33 [01:49<00:26,  4.34s/it]
+
 85% 28/33 [01:53<00:21,  4.36s/it]
+
 88% 29/33 [01:57<00:16,  4.10s/it]
+
 91% 30/33 [02:00<00:11,  3.71s/it]
+
 94% 31/33 [02:03<00:06,  3.43s/it]
+
 97% 32/33 [02:05<00:03,  3.19s/it]
+
+
                                   
                                             
+
  8% 1500/18000 [4:02:54<31:18:16,  6.83s/it]
+
                                   Saving model checkpoint to ./checkpoint-1500
+Configuration saved in ./checkpoint-1500/config.json
+Model weights saved in ./checkpoint-1500/pytorch_model.bin
+Configuration saved in ./checkpoint-1500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+Adding files tracked by Git LFS: ['wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb']. This may take a bit of time if the files are large.

logs

@@ -103,3 +103,11 @@ xpu_backend=None,
 {'loss': 0.2158, 'learning_rate': 0.000999279885163942, 'epoch': 22.22}
 {'loss': 0.2063, 'learning_rate': 0.000998717894518502, 'epoch': 25.0}
 {'loss': 0.1926, 'learning_rate': 0.000997995184978743, 'epoch': 27.78}
+{'eval_loss': 0.4142022132873535, 'eval_wer': 0.4174355711104041, 'eval_runtime': 135.7258, 'eval_samples_per_second': 19.223, 'eval_steps_per_second': 0.243, 'epoch': 27.78}
+{'loss': 0.1771, 'learning_rate': 0.0009971119894480345, 'epoch': 30.56}
+{'loss': 0.1709, 'learning_rate': 0.0009960685925486266, 'epoch': 33.33}
+{'loss': 0.1724, 'learning_rate': 0.0009948653305299256, 'epoch': 36.11}
+{'loss': 0.1632, 'learning_rate': 0.0009935025911601335, 'epoch': 38.89}
+{'loss': 0.1609, 'learning_rate': 0.0009919808136012837, 'epoch': 41.67}
+{'eval_loss': 0.4701250195503235, 'eval_wer': 0.42958956411072224, 'eval_runtime': 133.1213, 'eval_samples_per_second': 19.599, 'eval_steps_per_second': 0.248, 'epoch': 41.67}
+01/26/2022 15:02:33 - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb']. This may take a bit of time if the files are large.

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9522e4f73638d2cb960a7316bc2943a69e7b1f68581d9639de3e1b9b05dc8530
+oid sha256:4a75479d0f65fd7977ba222d635e5cc5ab880c9063c5de3c8814c5d4637f22b9
 size 1262083569

wandb/run-20220126_105847-1xdiy2kf/files/output.log

@@ -1086,3 +1086,551 @@ Configuration saved in ./preprocessor_config.json
 Configuration saved in ./checkpoint-1000/config.json
 Model weights saved in ./checkpoint-1000/pytorch_model.bin
 Configuration saved in ./checkpoint-1000/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+Deleting older checkpoint [checkpoint-500] due to args.save_total_limit
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8% 1500/18000 [4:00:40<31:18:16,  6.83s/it]The following columns in the evaluation set  don't have a corresponding argument in `Wav2Vec2ForCTC.forward` and have been ignored: input_length.
+***** Running Evaluation *****
+  Num examples = 2609
+  Batch size = 80
+  0% 0/33 [00:00<?, ?it/s]
+{'eval_loss': 0.4142022132873535, 'eval_wer': 0.4174355711104041, 'eval_runtime': 135.7258, 'eval_samples_per_second': 19.223, 'eval_steps_per_second': 0.243, 'epoch': 27.78}
+{'loss': 0.1771, 'learning_rate': 0.0009971119894480345, 'epoch': 30.56}
+{'loss': 0.1709, 'learning_rate': 0.0009960685925486266, 'epoch': 33.33}
+{'loss': 0.1724, 'learning_rate': 0.0009948653305299256, 'epoch': 36.11}
+{'loss': 0.1632, 'learning_rate': 0.0009935025911601335, 'epoch': 38.89}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Configuration saved in ./checkpoint-1500/config.json
+Model weights saved in ./checkpoint-1500/pytorch_model.bin
+Configuration saved in ./checkpoint-1500/preprocessor_config.json
+Configuration saved in ./preprocessor_config.json
+{'eval_loss': 0.4701250195503235, 'eval_wer': 0.42958956411072224, 'eval_runtime': 133.1213, 'eval_samples_per_second': 19.599, 'eval_steps_per_second': 0.248, 'epoch': 41.67}
+01/26/2022 15:02:33 - WARNING - huggingface_hub.repository - Adding files tracked by Git LFS: ['wandb/run-20220126_105847-1xdiy2kf/run-1xdiy2kf.wandb']. This may take a bit of time if the files are large.