diff --git "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" --- "a/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" +++ "b/wandb/run-20210716_095921-13hxxunp/logs/debug-internal.log" @@ -11681,3 +11681,3850 @@ 2021-07-16 15:21:52,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log 2021-07-16 15:22:00,300 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status 2021-07-16 15:22:00,300 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:22:11,593 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:22:15,434 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:22:15,434 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:22:30,568 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:22:30,568 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:22:41,673 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:22:45,707 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:22:45,707 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:23:00,843 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:23:00,843 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:23:11,751 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:23:15,973 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:23:15,973 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:23:22,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:24,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:26,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:28,266 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:30,267 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:31,106 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:23:31,107 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:23:41,772 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:23:41,772 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:23:41,775 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:23:41,829 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:23:42,273 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:23:44,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:46,267 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:23:46,267 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:23:46,275 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:48,276 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:23:50,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:01,402 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:24:01,402 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:24:01,812 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:24:01,812 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:24:01,813 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:24:02,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:24:04,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:06,284 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:08,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:10,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:11,909 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:24:16,535 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:24:16,535 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:24:21,849 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:24:21,849 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:24:21,850 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:24:22,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:24:24,292 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:26,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:28,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:30,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:31,667 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:24:31,668 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:24:41,887 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:24:41,887 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:24:41,888 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:24:41,988 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:24:42,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:24:44,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:46,302 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:46,801 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:24:46,801 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:24:48,302 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:24:50,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:01,925 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:25:01,926 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:25:01,926 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:25:01,964 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:25:01,965 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:25:02,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:25:04,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:05,310 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:07,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:09,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:11,313 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:12,065 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:25:17,110 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:25:17,111 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:25:21,964 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:25:21,964 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:25:21,965 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:25:22,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:25:23,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:25,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:27,319 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:29,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:31,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:32,243 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:25:32,244 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:25:42,004 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:25:42,004 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:25:42,005 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:25:42,144 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:25:42,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:25:45,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:47,328 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:47,384 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:25:47,385 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:25:49,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:25:51,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:02,044 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:26:02,044 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:26:02,045 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:26:02,335 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:26:02,525 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:26:02,526 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:26:03,336 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:05,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:07,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:09,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:11,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:12,221 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:26:17,666 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:26:17,666 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:26:22,084 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:26:22,084 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:26:22,085 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:26:22,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:26:23,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:25,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:27,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:29,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:31,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:32,800 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:26:32,801 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:26:42,122 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:26:42,123 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:26:42,123 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:26:42,299 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:26:42,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:26:43,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:45,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:47,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:47,935 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:26:47,935 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:26:49,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:26:51,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:02,160 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:27:02,160 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:27:02,161 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:27:02,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:27:03,070 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:27:03,070 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:27:03,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:05,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:07,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:09,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:11,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:12,376 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:27:18,211 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:27:18,211 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:27:22,196 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:27:22,197 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:27:22,198 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:27:22,367 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:27:23,367 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:25,368 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:27,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:29,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:31,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:33,354 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:27:33,355 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:27:42,233 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:27:42,234 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:27:42,234 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:27:42,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:27:42,454 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:27:43,376 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:45,377 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:47,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:48,487 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:27:48,487 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:27:49,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:27:51,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:02,271 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:28:02,271 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:28:02,271 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:28:02,385 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:28:03,385 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:03,634 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:28:03,634 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:28:05,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:07,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:09,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:11,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:12,531 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:28:18,772 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:28:18,773 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:28:22,311 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:28:22,311 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:28:22,312 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:28:22,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:28:23,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:25,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:27,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:29,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:31,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:33,903 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:28:33,903 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:28:42,347 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:28:42,348 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:28:42,348 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:28:42,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:28:42,609 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:28:43,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:45,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:47,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:49,035 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:28:49,035 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:28:49,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:28:51,407 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:02,385 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:29:02,386 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:29:02,386 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:29:02,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:29:03,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:04,169 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:29:04,170 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:29:05,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:07,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:09,415 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:11,416 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:12,687 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:29:19,328 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:29:19,329 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:29:22,426 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:29:22,427 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:29:22,427 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:29:23,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:29:23,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:25,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:27,423 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:29,424 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:31,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:34,458 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:29:34,459 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:29:42,464 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:29:42,465 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:29:42,465 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:29:42,766 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:29:43,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:29:43,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:45,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:47,432 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:49,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:29:49,591 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:29:49,591 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:29:51,434 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:02,504 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:30:02,505 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:30:02,505 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:30:03,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:30:03,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:04,731 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:30:04,731 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:30:05,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:07,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:09,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:11,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:12,843 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:30:19,869 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:30:19,870 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:30:22,546 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:30:22,546 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:30:22,547 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:30:23,447 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:30:23,447 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:25,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:27,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:29,450 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:31,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:35,006 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:30:35,006 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:30:42,582 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:30:42,583 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:30:42,583 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:30:42,921 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:30:43,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:30:43,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:45,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:47,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:49,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:30:50,141 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:30:50,142 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:30:51,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:02,622 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:31:02,622 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:31:02,623 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:31:03,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:31:03,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:05,280 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:31:05,280 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:31:05,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:07,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:09,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:11,469 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:12,998 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:31:20,415 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:31:20,415 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:31:22,667 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:31:22,667 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:31:22,668 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:31:23,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:31:23,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:25,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:27,477 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:29,477 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:31,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:35,551 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:31:35,552 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:31:42,709 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:31:42,709 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:31:42,710 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:31:43,077 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:31:43,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:31:43,484 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:45,484 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:47,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:49,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:31:50,692 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:31:50,692 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:31:51,487 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:02,751 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:32:02,751 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:32:02,752 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:32:03,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:32:03,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:05,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:05,826 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:32:05,826 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:32:07,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:09,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:11,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:13,153 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:32:20,957 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:32:20,958 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:32:22,796 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:32:22,796 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:32:22,797 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:32:23,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:32:23,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:25,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:27,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:29,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:31,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:36,089 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:32:36,089 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:32:42,837 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:32:42,837 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:32:42,838 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:32:43,231 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:32:43,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:32:43,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:45,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:47,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:49,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:32:51,222 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:32:51,222 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:32:51,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:02,899 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:33:02,899 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:33:02,900 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:33:03,517 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:33:03,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:05,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:06,356 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:33:06,356 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:33:07,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:09,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:11,521 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:13,309 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:33:21,488 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:33:21,489 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:33:22,939 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:33:22,939 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:33:22,940 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:33:23,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:33:23,526 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:25,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:27,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:29,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:31,530 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:36,620 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:33:36,620 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:33:42,980 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:33:42,980 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:33:42,981 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:33:43,386 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:33:43,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:33:43,535 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:45,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:48,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:50,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:33:51,752 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:33:51,752 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:33:52,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:03,021 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:34:03,021 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:34:03,022 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:34:03,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:34:03,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:04,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:06,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:06,885 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:34:06,886 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:34:08,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:10,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:12,548 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:13,464 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:34:22,017 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:34:22,018 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:34:23,060 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:34:23,060 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:34:23,061 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:34:23,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:34:23,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:24,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:26,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:28,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:30,556 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:32,557 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:37,147 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:34:37,148 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:34:43,101 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:34:43,101 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:34:43,102 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:34:43,536 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:34:43,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:34:44,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:46,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:48,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:50,564 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:34:52,278 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:34:52,278 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:34:52,565 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:03,139 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:35:03,139 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:35:03,140 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:35:03,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:35:04,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:06,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:07,412 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:35:07,412 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:35:08,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:10,572 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:12,572 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:13,603 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:35:22,547 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:35:22,547 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:35:23,178 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:35:23,179 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:35:23,179 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:35:23,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:35:24,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:26,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:28,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:30,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:32,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:37,681 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:35:37,681 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:35:43,217 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:35:43,217 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:35:43,218 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:35:43,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:35:43,673 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:35:44,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:46,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:48,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:50,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:52,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:35:52,813 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:35:52,813 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:36:03,260 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:36:03,260 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:36:03,260 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:36:03,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:36:04,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:06,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:07,945 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:36:07,946 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:36:08,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:10,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:12,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:13,740 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:36:23,078 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:36:23,078 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:36:23,295 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:36:23,295 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:36:23,296 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:36:23,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:36:24,604 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:26,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:28,606 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:30,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:32,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:38,211 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:36:38,212 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:36:43,333 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:36:43,333 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:36:43,334 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:36:43,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:36:43,810 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:36:44,612 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:46,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:48,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:50,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:52,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:36:53,343 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:36:53,343 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:37:03,372 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:37:03,372 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:37:03,373 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:37:03,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:37:04,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:06,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:08,481 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:37:08,482 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:37:08,621 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:10,622 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:12,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:13,881 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:37:23,411 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:37:23,411 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:37:23,411 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:37:23,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:37:23,644 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:37:23,645 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:37:24,628 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:26,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:28,629 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:30,630 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:32,631 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:38,781 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:37:38,781 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:37:43,448 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:37:43,448 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:37:43,449 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:37:43,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:37:43,954 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:37:44,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:46,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:48,637 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:50,638 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:52,639 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:37:53,914 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:37:53,914 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:38:03,486 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:38:03,487 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:38:03,487 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:38:03,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:38:04,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:06,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:08,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:09,045 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:38:09,045 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:38:10,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:12,646 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:14,027 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:38:23,525 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:38:23,526 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:38:23,526 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:38:23,650 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:38:24,192 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:38:24,192 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:38:24,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:26,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:28,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:30,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:32,654 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:39,330 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:38:39,330 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:38:43,564 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:38:43,565 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:38:43,565 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:38:43,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:38:44,104 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:38:44,658 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:46,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:48,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:50,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:52,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:38:54,460 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:38:54,460 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:39:03,603 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:39:03,603 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:39:03,603 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:39:03,665 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:39:04,666 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:06,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:08,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:09,594 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:39:09,594 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:39:10,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:12,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:14,179 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:39:23,644 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:39:23,645 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:39:23,645 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:39:23,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:39:24,674 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:24,749 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:39:24,750 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:39:26,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:28,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:30,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:32,677 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:39,935 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:39:39,936 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:39:43,700 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:39:43,701 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:39:43,701 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:39:44,259 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:39:44,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:39:44,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:46,683 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:48,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:50,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:52,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:39:55,067 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:39:55,068 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:40:03,745 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:40:03,745 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:40:03,746 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:40:04,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:40:04,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:06,691 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:08,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:10,232 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:40:10,232 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:40:10,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:12,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:14,336 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:40:23,788 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:40:23,788 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:40:23,789 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:40:24,698 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:40:24,698 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:25,383 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:40:25,384 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:40:26,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:28,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:30,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:32,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:40,520 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:40:40,520 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:40:43,825 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:40:43,826 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:40:43,826 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:40:44,414 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:40:44,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:40:44,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:46,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:48,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:50,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:52,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:40:55,653 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:40:55,654 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:41:03,862 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:41:03,862 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:41:03,863 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:41:04,714 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:41:04,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:06,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:08,716 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:10,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:10,788 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:41:10,789 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:41:12,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:14,492 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:41:23,900 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:41:23,901 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:41:23,901 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:41:24,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:41:24,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:25,928 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:41:25,929 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:41:26,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:28,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:30,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:32,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:41,059 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:41:41,059 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:41:43,942 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:41:43,943 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:41:43,943 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:41:44,572 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:41:44,731 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:41:44,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:46,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:48,733 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:50,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:52,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:41:56,201 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:41:56,201 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:42:03,975 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:42:03,976 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:42:03,976 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:42:04,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:42:04,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:06,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:08,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:10,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:11,331 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:42:11,332 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:42:12,742 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:14,648 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:42:24,014 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:42:24,014 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:42:24,015 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:42:24,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:42:24,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:25,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:26,461 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:42:26,462 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:42:26,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:28,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:30,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:32,750 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:41,594 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:42:41,594 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:42:44,057 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:42:44,058 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:42:44,058 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:42:44,725 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:42:44,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:42:44,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:45,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:46,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:48,756 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:50,756 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:52,757 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:42:56,726 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:42:56,727 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:43:04,099 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:43:04,099 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:43:04,099 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:43:04,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:43:05,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:07,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:09,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:11,765 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:11,856 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:43:11,856 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:43:13,766 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:14,803 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:43:24,139 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:43:24,139 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:43:24,140 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:43:24,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:43:25,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:26,987 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:43:26,988 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:43:27,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:29,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:31,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:33,774 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:42,130 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:43:42,130 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:43:44,176 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:43:44,176 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:43:44,177 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:43:44,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:43:44,881 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:43:45,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:47,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:49,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:51,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:53,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:43:57,265 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:43:57,266 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:44:04,217 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:44:04,217 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:44:04,217 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:44:04,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:44:05,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:07,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:09,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:11,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:12,399 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:44:12,399 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:44:13,791 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:14,958 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:44:24,257 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:44:24,257 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:44:24,258 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:44:24,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:44:25,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:27,559 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:44:27,560 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:44:27,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:29,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:31,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:33,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:42,690 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:44:42,691 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:44:44,299 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:44:44,299 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:44:44,300 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:44:44,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:44:45,036 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:44:45,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:47,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:49,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:51,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:53,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:44:57,827 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:44:57,827 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:45:04,338 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:45:04,338 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:45:04,339 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:45:04,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:45:05,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:07,813 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:09,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:11,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:12,958 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:45:12,959 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:45:13,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:15,114 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:45:24,378 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:45:24,379 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:45:24,379 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:45:24,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:45:25,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:27,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:28,093 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:45:28,093 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:45:29,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:31,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:33,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:43,227 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:45:43,227 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:45:44,417 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:45:44,417 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:45:44,418 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:45:44,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:45:45,189 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:45:45,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:47,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:49,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:51,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:53,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:45:58,357 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:45:58,358 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:46:04,454 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:46:04,454 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:46:04,455 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:46:04,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:46:05,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:07,838 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:09,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:11,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:13,492 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:46:13,492 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:46:13,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:15,280 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:46:24,494 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:46:24,495 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:46:24,495 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:46:24,846 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:46:25,846 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:27,847 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:28,624 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:46:28,625 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:46:29,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:31,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:33,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:43,755 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:46:43,755 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:46:44,534 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:46:44,535 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:46:44,535 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:46:44,854 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:46:45,376 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:46:45,855 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:47,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:49,856 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:51,857 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:53,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:46:58,888 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:46:58,889 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:47:04,570 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:47:04,570 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:47:04,571 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:47:04,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:47:05,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:07,863 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:09,864 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:11,865 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:13,866 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:14,033 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:47:14,033 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:47:15,448 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:47:24,604 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:47:24,605 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:47:24,605 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:47:24,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:47:25,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:27,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:29,345 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:47:29,345 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:47:29,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:31,873 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:33,874 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:44,498 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:47:44,499 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:47:44,641 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:47:44,641 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:47:44,642 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:47:44,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:47:45,521 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:47:45,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:47,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:49,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:51,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:53,882 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:47:59,630 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:47:59,630 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:48:04,678 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:48:04,678 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:48:04,679 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:48:04,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:48:05,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:07,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:09,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:11,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:13,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:14,758 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:48:14,759 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:48:15,595 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:48:24,717 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:48:24,717 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:48:24,717 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:48:24,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:48:25,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:27,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:29,889 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:48:29,890 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:48:29,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:31,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:33,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:44,753 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:48:44,753 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:48:44,754 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:48:44,901 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:48:45,049 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:48:45,050 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:48:45,670 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:48:45,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:47,902 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:49,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:51,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:48:53,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:00,189 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:49:00,189 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:49:04,792 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:49:04,792 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:49:04,793 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:49:04,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:49:05,910 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:07,911 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:09,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:11,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:15,320 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:49:15,320 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:49:15,743 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:49:24,830 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:49:24,830 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:49:24,831 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:49:24,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:49:25,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:27,919 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:29,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:30,454 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:49:30,454 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:49:31,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:33,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:44,866 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:49:44,867 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:49:44,867 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:49:44,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:49:45,596 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:49:45,596 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:49:45,819 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:49:45,926 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:47,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:49,927 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:52,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:49:54,930 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:00,733 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:50:00,734 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:50:04,905 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:50:04,905 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:50:04,905 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:50:04,933 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:50:06,934 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:08,935 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:10,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:12,936 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:14,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:15,866 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:50:15,867 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:50:15,996 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:50:24,940 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:50:24,940 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:50:24,941 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:50:24,942 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:50:26,942 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:28,943 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:30,944 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:31,024 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:50:31,024 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:50:32,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:34,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:44,976 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:50:44,976 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:50:44,977 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:50:45,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:50:45,974 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:50:46,160 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:50:46,160 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:50:46,950 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:48,951 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:50,952 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:52,953 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:50:54,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:01,298 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:51:01,298 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:51:05,012 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:51:05,013 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:51:05,014 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:51:05,958 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:51:06,959 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:08,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:10,960 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:12,961 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:14,962 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:16,051 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:51:16,433 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:51:16,433 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:51:25,052 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:51:25,052 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:51:25,054 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:51:25,966 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:51:26,967 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:28,968 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:30,969 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:31,572 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:51:31,572 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:51:32,970 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:34,971 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:45,093 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:51:45,094 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:51:45,094 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:51:45,976 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:51:46,128 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:51:46,705 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:51:46,705 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:51:46,976 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:48,977 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:50,978 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:51:52,979 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:02,619 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:52:02,619 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:52:05,134 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:52:05,134 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:52:05,136 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:52:05,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:52:06,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:08,987 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:10,988 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:12,989 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:16,205 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:52:17,750 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:52:17,751 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:52:25,172 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:52:25,172 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:52:25,173 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:52:25,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:52:26,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:28,996 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:30,997 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:32,882 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:52:32,882 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:52:32,998 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:45,210 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:52:45,211 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:52:45,213 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:52:46,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:52:46,290 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:52:47,004 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:48,016 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:52:48,016 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:52:49,005 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:51,006 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:52:53,007 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:03,148 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:53:03,149 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:53:05,248 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:53:05,249 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:53:05,249 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:53:06,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:53:07,013 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:09,014 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:11,015 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:13,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:16,367 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:53:18,283 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:53:18,283 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:53:25,287 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:53:25,288 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:53:25,288 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:53:26,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:53:27,022 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:29,023 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:31,024 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:33,025 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:33,415 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:53:33,415 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:53:45,326 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:53:45,326 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:53:45,327 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:53:46,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:53:46,442 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:53:47,031 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:48,548 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:53:48,549 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:53:49,032 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:51,033 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:53:53,034 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:03,683 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:54:03,684 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:54:05,365 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:54:05,365 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:54:05,366 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:54:06,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:54:07,040 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:09,041 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:11,042 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:13,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:16,518 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:54:18,817 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:54:18,817 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:54:25,403 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:54:25,403 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:54:25,403 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:54:26,049 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:54:27,049 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:29,050 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:31,051 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:33,052 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:33,949 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:54:33,950 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:54:45,444 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:54:45,445 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:54:45,445 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:54:46,058 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:54:46,596 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:54:47,058 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:49,059 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:49,081 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:54:49,081 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:54:51,060 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:54:53,061 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:04,213 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:55:04,214 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:55:05,482 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:55:05,483 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:55:05,483 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:55:06,066 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:55:07,067 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:09,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:11,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:13,069 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:16,674 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:55:19,348 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:55:19,348 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:55:25,519 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:55:25,520 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:55:25,520 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:55:26,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:55:27,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:29,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:31,077 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:33,078 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:34,498 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:55:34,498 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:55:45,558 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:55:45,558 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:55:45,559 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:55:46,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:55:46,752 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:55:47,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:49,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:49,653 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:55:49,654 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:55:51,085 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:55:53,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:04,784 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:56:04,785 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:56:05,595 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:56:05,596 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:56:05,596 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:56:06,090 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:56:07,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:09,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:11,092 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:13,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:16,831 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:56:19,920 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:56:19,920 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:56:25,630 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:56:25,630 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:56:25,631 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:56:26,098 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:56:27,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:29,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:31,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:33,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:35,054 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:56:35,054 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:56:45,665 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:56:45,666 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:56:45,666 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:56:46,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:56:46,909 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:56:47,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:49,107 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:50,186 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:56:50,186 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:56:51,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:56:53,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:05,351 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:57:05,351 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:57:05,701 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:57:05,701 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:57:05,702 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:57:06,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:57:07,114 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:09,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:11,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:13,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:16,989 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:57:20,485 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:57:20,485 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:57:25,737 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:57:25,737 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:57:25,738 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:57:26,121 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:57:27,122 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:29,122 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:31,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:33,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:35,616 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:57:35,617 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:57:45,775 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:57:45,776 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:57:45,776 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:57:46,129 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:57:47,067 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:57:47,129 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:49,130 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:50,751 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:57:50,752 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:57:51,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:57:53,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:05,812 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:58:05,812 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:58:05,813 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:58:05,884 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:58:05,884 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:58:06,137 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:58:08,137 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:10,138 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:12,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:14,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:17,145 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:58:21,042 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:58:21,043 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:58:25,848 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:58:25,849 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:58:25,849 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:58:26,145 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:58:28,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:30,146 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:32,147 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:34,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:36,175 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:58:36,175 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:58:45,885 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:58:45,886 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:58:45,886 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:58:46,153 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:58:47,223 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:58:48,154 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:50,155 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:51,319 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:58:51,319 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:58:52,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:58:54,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:05,923 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:59:05,923 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:59:05,924 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:59:06,162 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:59:06,470 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:59:06,471 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:59:08,163 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:10,164 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:12,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:14,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:17,300 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:59:21,609 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:59:21,609 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:59:25,961 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:59:25,962 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:59:25,962 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:59:26,171 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:59:28,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:30,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:32,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:34,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:36,744 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:59:36,744 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:59:45,999 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 15:59:45,999 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 15:59:46,000 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 15:59:46,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 15:59:47,379 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 15:59:48,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:50,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:51,876 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 15:59:51,876 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 15:59:52,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 15:59:54,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:06,035 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:00:06,035 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:00:06,036 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:00:06,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:00:07,008 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:00:07,008 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:00:08,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:10,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:12,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:14,192 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:17,456 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:00:22,144 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:00:22,145 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:00:26,073 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:00:26,073 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:00:26,074 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:00:26,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:00:28,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:30,200 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:32,201 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:34,202 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:37,275 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:00:37,275 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:00:46,108 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:00:46,109 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:00:46,109 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:00:46,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:00:47,534 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:00:48,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:50,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:52,210 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:00:52,414 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:00:52,414 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:00:54,211 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:06,148 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:01:06,149 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:01:06,149 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:01:06,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:01:07,555 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:01:07,555 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:01:08,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:10,218 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:12,219 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:14,220 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:17,611 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:01:22,694 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:01:22,694 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:01:26,184 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:01:26,184 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:01:26,185 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:01:26,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:01:28,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:30,227 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:32,228 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:34,229 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:37,826 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:01:37,827 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:01:46,224 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:01:46,224 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:01:46,225 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:01:46,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:01:47,688 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:01:48,235 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:50,236 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:52,238 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:01:52,961 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:01:52,961 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:01:54,238 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:06,262 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:02:06,262 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:02:06,263 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:02:07,244 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:02:08,114 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:02:08,114 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:02:08,245 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:10,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:12,246 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:14,247 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:17,766 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:02:23,245 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:02:23,246 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:02:26,302 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:02:26,302 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:02:26,303 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:02:27,253 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:02:28,254 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:30,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:32,255 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:34,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:38,378 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:02:38,378 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:02:46,340 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:02:46,341 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:02:46,341 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:02:47,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:02:47,844 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:02:48,263 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:50,264 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:52,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:02:53,521 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:02:53,521 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:02:54,266 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:06,377 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:03:06,378 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:03:06,378 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:03:07,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:03:08,272 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:08,678 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:03:08,678 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:03:10,273 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:12,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:14,275 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:17,922 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:03:23,815 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:03:23,816 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:03:26,416 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:03:26,416 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:03:26,417 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:03:27,280 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:03:28,281 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:30,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:32,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:34,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:36,284 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:38,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:38,951 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:03:38,952 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:03:40,286 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:42,287 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:44,288 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:46,289 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:47,999 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:03:48,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:50,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:52,292 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:54,085 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:03:54,086 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:03:54,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:56,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:03:58,294 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:00,295 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:02,296 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:04,297 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:06,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:09,219 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:04:09,220 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:04:09,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:11,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:13,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:15,302 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:17,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:18,077 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:04:19,303 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:21,304 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:23,305 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:24,376 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:04:24,376 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:04:25,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:27,307 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:30,336 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:04:30,336 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:04:30,337 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:04:31,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:04:33,310 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:35,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:37,311 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:39,312 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:39,511 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:04:39,511 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:04:48,155 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:04:50,386 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:04:50,386 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:04:50,387 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:04:51,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:04:52,318 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:53,319 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:54,645 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:04:54,646 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:04:55,320 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:57,321 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:04:59,322 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:09,776 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:05:09,777 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:05:10,423 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:05:10,423 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:05:10,424 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:05:11,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:05:11,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:12,327 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:13,328 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:15,329 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:17,329 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:18,234 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:05:19,330 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:24,913 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:05:24,913 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:05:30,462 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:05:30,462 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:05:30,463 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:05:31,335 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:05:32,335 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:33,336 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:35,337 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:37,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:39,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:40,051 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:05:40,051 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:05:48,310 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:05:50,500 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:05:50,501 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:05:50,501 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:05:51,343 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:05:52,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:53,344 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:55,183 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:05:55,184 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:05:55,345 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:57,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:05:59,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:10,317 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:06:10,318 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:06:10,535 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:06:10,536 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:06:10,536 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:06:11,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:06:13,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:15,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:17,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:18,383 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:06:19,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:25,457 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:06:25,458 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:06:30,570 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:06:30,571 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:06:30,571 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:06:31,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:06:31,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:33,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:35,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:37,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:39,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:40,589 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:06:40,590 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:06:48,451 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:06:50,609 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:06:50,609 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:06:50,610 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:06:51,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:06:51,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:53,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:55,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:55,724 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:06:55,724 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:06:57,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:06:59,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:10,647 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:07:10,648 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:07:10,648 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:07:10,875 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:07:10,876 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:07:11,377 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:07:11,377 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:13,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:15,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:17,380 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:18,519 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:07:19,381 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:26,011 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:07:26,012 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:07:30,681 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:07:30,681 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:07:30,682 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:07:31,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:07:31,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:33,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:35,388 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:37,389 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:39,390 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:41,143 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:07:41,143 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:07:48,586 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:07:50,717 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:07:50,718 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:07:50,718 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:07:51,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:07:51,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:53,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:55,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:56,274 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:07:56,274 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:07:57,397 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:07:59,398 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:10,755 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:08:10,755 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:08:10,756 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:08:11,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:08:11,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:11,421 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:08:11,421 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:08:13,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:15,405 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:17,406 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:18,659 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:08:19,407 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:26,558 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:08:26,558 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:08:30,792 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:08:30,792 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:08:30,792 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:08:31,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:08:31,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:33,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:35,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:37,414 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:39,415 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:41,690 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:08:41,690 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:08:48,734 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:08:50,829 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:08:50,829 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:08:50,830 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:08:51,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:08:51,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:53,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:55,423 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:56,822 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:08:56,822 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:08:57,423 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:08:59,425 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:10,865 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:09:10,865 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:09:10,866 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:09:11,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:09:11,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:11,969 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:09:11,969 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:09:13,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:15,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:17,432 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:18,808 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:09:19,433 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:27,114 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:09:27,114 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:09:30,903 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:09:30,904 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:09:30,904 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:09:31,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:09:32,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:33,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:34,440 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:36,441 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:38,442 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:40,443 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:42,248 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:09:42,249 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:09:48,886 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:09:50,939 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:09:50,939 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:09:50,939 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:09:51,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:09:52,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:53,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:54,449 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:56,450 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:09:57,423 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:09:57,424 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:09:58,451 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:00,452 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:10,976 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:10:10,977 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:10:10,977 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:10:11,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:10:12,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:12,576 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:10:12,577 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:10:13,458 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:14,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:16,459 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:18,460 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:18,966 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:10:20,461 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:27,715 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:10:27,716 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:10:31,017 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:10:31,017 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:10:31,018 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:10:31,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:10:32,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:33,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:34,467 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:36,468 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:38,469 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:40,470 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:42,852 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:10:42,852 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:10:49,043 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:10:51,055 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:10:51,056 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:10:51,056 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:10:51,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:10:52,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:53,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:54,476 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:56,478 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:10:57,986 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:10:57,986 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:10:58,479 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:00,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:11,092 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:11:11,092 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:11:11,093 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:11:11,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:11:12,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:13,146 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:11:13,146 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:11:13,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:14,486 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:16,487 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:18,488 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:19,121 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:11:20,489 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:28,286 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:11:28,287 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:11:31,133 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:11:31,134 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:11:31,134 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:11:31,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:11:32,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:33,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:34,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:36,495 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:38,496 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:40,497 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:43,417 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:11:43,418 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:11:49,197 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:11:51,170 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:11:51,171 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:11:51,171 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:11:51,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:11:52,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:53,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:54,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:56,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:58,505 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:11:58,548 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:11:58,548 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:12:00,506 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:11,207 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:12:11,207 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:12:11,208 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:12:11,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:12:12,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:13,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:13,686 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:12:13,687 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:12:14,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:16,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:18,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:19,276 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:12:20,514 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:28,865 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:12:28,865 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:12:31,244 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:12:31,245 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:12:31,245 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:12:31,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:12:32,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:33,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:34,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:36,521 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:38,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:40,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:43,997 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:12:43,997 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:12:49,355 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:12:51,280 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:12:51,280 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:12:51,281 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:12:51,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:12:52,527 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:53,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:54,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:56,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:58,530 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:12:59,132 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:12:59,133 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:13:00,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:11,317 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:13:11,317 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:13:11,318 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:13:11,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:13:12,536 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:13,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:14,262 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:13:14,263 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:13:14,537 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:16,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:18,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:19,432 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:13:20,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:29,400 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:13:29,400 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:13:31,353 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:13:31,353 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:13:31,354 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:13:31,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:13:32,544 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:33,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:34,545 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:36,546 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:38,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:40,548 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:44,533 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:13:44,534 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:13:49,509 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:13:51,388 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:13:51,389 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:13:51,389 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:13:51,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:13:52,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:53,552 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:54,553 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:56,554 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:58,555 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:13:59,663 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:13:59,664 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:14:00,556 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:11,425 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:14:11,425 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:14:11,426 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:14:11,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:14:12,560 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:13,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:14,561 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:14,794 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:14:14,795 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:14:16,562 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:18,563 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:19,587 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:14:20,564 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:29,932 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:14:29,932 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:14:31,462 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:14:31,462 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:14:31,463 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:14:31,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:14:32,569 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:33,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:34,570 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:36,571 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:38,572 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:40,573 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:45,063 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:14:45,064 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:14:49,664 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:14:51,500 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:14:51,501 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:14:51,501 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:14:51,577 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:14:52,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:53,578 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:54,579 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:56,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:14:58,580 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:00,198 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:15:00,199 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:15:00,581 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:11,539 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:15:11,540 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:15:11,540 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:15:11,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:15:12,586 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:13,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:14,587 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:15,330 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:15:15,331 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:15:16,588 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:18,589 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:19,743 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:15:20,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:30,463 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:15:30,463 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:15:31,576 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:15:31,577 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:15:31,577 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:15:31,595 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:15:32,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:33,596 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:34,597 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:36,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:38,598 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:40,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:45,596 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:15:45,596 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:15:49,820 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:15:51,615 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:15:51,616 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:15:51,616 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:15:52,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:15:52,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:53,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:54,605 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:56,606 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:15:58,607 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:00,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:00,728 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:16:00,728 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:16:11,652 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:16:11,653 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:16:11,653 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:16:12,613 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:16:12,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:13,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:14,614 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:15,873 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:16:15,873 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:16:16,615 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:18,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:19,900 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:16:20,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:31,005 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:16:31,006 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:16:31,691 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:16:31,692 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:16:31,693 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:16:32,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:16:32,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:34,623 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:36,624 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:38,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:40,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:46,150 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:16:46,151 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:16:49,976 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:16:51,730 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:16:51,730 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:16:51,731 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:16:52,632 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:16:52,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:54,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:56,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:16:58,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:00,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:01,285 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:17:01,285 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:17:11,767 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:17:11,768 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:17:11,768 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:17:12,641 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:17:12,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:14,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:16,417 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:17:16,417 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:17:16,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:18,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:20,054 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:17:20,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:31,549 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:17:31,549 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:17:31,804 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:17:31,804 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:17:31,805 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:17:32,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:17:32,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:34,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:36,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:38,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:40,654 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:46,690 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:17:46,690 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:17:50,130 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:17:51,842 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:17:51,842 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:17:51,843 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:17:52,659 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:17:52,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:54,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:56,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:17:58,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:00,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:01,826 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:18:01,826 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:18:11,880 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:18:11,880 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:18:11,881 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:18:12,667 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:18:12,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:14,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:16,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:16,959 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:18:16,960 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:18:18,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:20,209 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:18:20,671 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:31,915 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:18:31,915 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:18:31,916 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:18:32,096 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:18:32,097 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:18:32,675 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:18:32,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:34,676 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:36,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:38,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:40,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:47,266 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:18:47,267 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:18:50,286 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:18:51,952 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:18:51,953 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:18:51,953 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:18:52,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:18:52,684 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:54,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:56,685 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:18:58,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:00,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:02,403 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:19:02,404 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:19:11,991 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:19:11,991 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:19:11,991 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:19:12,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:19:12,692 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:14,693 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:16,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:17,535 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:19:17,536 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:19:18,694 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:20,364 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:19:20,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:32,029 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:19:32,029 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:19:32,030 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:19:32,694 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:19:32,694 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:19:32,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:19:32,700 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:34,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:36,701 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:39,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:41,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:47,831 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:19:47,831 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:19:50,443 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:19:52,066 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:19:52,066 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:19:52,067 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:19:52,708 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:19:53,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:54,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:55,709 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:57,710 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:19:59,711 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:01,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:02,967 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:20:02,967 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:20:12,102 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:20:12,103 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:20:12,103 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:20:12,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:20:13,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:14,717 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:15,718 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:17,719 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:18,099 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:20:18,100 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:20:19,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:20,520 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:20:21,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:32,140 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:20:32,141 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:20:32,141 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:20:32,725 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:20:33,240 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:20:33,240 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:20:33,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:34,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:35,726 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:37,727 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:39,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:41,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:48,379 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:20:48,380 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:20:50,598 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:20:52,178 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:20:52,179 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:20:52,179 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:20:52,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:20:53,734 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:54,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:55,735 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:57,736 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:20:59,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:01,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:03,512 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:21:03,512 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:21:12,215 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:21:12,216 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:21:12,216 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:21:12,743 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:21:13,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:14,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:15,744 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:17,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:18,641 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:21:18,641 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:21:19,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:20,676 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:21:21,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:32,255 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:21:32,255 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:21:32,256 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:21:32,751 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:21:33,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:33,792 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:21:33,792 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:21:34,752 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:35,753 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:37,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:39,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:41,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:48,929 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:21:48,930 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:21:50,752 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:21:52,295 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:21:52,295 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:21:52,295 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:21:52,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:21:53,760 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:54,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:55,761 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:57,762 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:21:59,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:01,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:04,064 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:22:04,064 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:22:12,333 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:22:12,334 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:22:12,334 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:22:12,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:22:13,769 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:14,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:15,770 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:17,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:19,218 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:22:19,218 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:22:19,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:20,830 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:22:21,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:32,372 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:22:32,373 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:22:32,373 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:22:32,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:22:33,778 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:34,377 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:22:34,378 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:22:34,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:35,779 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:37,780 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:39,781 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:41,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:49,508 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:22:49,509 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:22:50,908 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:22:52,410 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:22:52,411 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:22:52,411 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:22:52,787 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:22:53,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:54,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:55,788 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:57,789 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:22:59,790 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:01,791 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:04,644 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:23:04,644 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:23:12,448 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:23:12,449 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:23:12,449 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:23:12,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:23:13,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:14,797 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:15,798 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:17,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:19,782 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:23:19,782 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:23:19,799 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:20,985 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:23:21,801 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:32,484 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:23:32,485 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:23:32,485 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:23:32,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:23:33,806 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:34,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:34,921 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:23:34,922 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:23:35,807 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:37,808 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:39,809 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:41,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:50,060 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:23:50,060 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:23:51,063 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:23:52,523 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:23:52,523 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:23:52,524 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:23:52,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:23:53,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:54,816 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:55,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:57,817 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:23:59,818 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:01,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:05,192 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:24:05,192 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:24:12,561 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:24:12,562 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:24:12,562 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:24:12,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:24:13,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:15,825 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:17,826 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:19,827 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:20,327 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:24:20,327 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:24:21,142 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:24:21,828 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:32,600 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:24:32,601 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:24:32,601 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:24:32,833 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:24:33,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:35,464 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:24:35,464 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:24:35,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:37,835 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:39,836 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:41,837 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:50,601 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:24:50,602 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:24:51,219 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:24:52,636 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:24:52,637 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:24:52,637 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:24:52,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:24:53,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:55,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:57,844 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:24:59,845 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:01,846 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:05,735 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:25:05,735 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:25:12,676 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:25:12,677 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:25:12,677 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:25:12,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:25:13,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:15,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:17,853 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:19,854 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:20,863 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:25:20,864 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:25:21,299 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:25:21,855 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:32,714 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:25:32,714 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:25:32,715 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:25:32,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:25:33,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:35,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:36,042 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:25:36,043 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:25:37,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:39,863 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:41,864 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:51,175 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:25:51,175 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:25:51,380 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:25:52,750 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:25:52,751 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:25:52,751 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:25:52,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:25:53,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:55,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:57,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:25:59,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:01,872 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:06,317 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:26:06,318 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:26:12,787 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:26:12,787 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:26:12,787 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:26:12,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:26:13,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:15,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:17,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:19,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:21,452 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:26:21,452 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:26:21,579 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:26:21,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:32,823 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:26:32,823 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:26:32,824 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:26:32,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:26:33,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:35,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:36,583 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:26:36,583 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:26:37,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:39,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:41,890 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:51,536 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:26:51,715 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:26:51,715 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:26:52,859 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:26:52,859 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:26:52,861 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:26:52,894 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:26:53,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:55,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:57,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:26:59,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:01,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:06,849 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:27:06,850 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:27:12,895 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:27:12,896 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:27:12,896 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:27:12,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:27:13,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:15,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:17,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:19,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:21,614 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:27:21,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:21,987 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:27:21,987 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:27:32,931 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:27:32,932 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:27:32,932 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:27:33,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:27:33,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:35,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:37,118 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:27:37,118 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:27:37,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:39,914 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:41,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:51,694 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:27:52,249 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:27:52,250 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:27:52,969 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:27:52,970 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:27:52,970 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:27:53,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:27:53,920 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:55,921 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:57,922 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:27:59,923 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:01,924 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:07,381 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:28:07,381 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:28:13,007 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:28:13,007 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:28:13,010 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:28:13,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:28:13,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:15,929 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:17,930 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:19,931 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:21,773 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:28:22,520 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:28:22,520 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:28:22,932 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:33,045 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:28:33,046 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:28:33,046 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:28:33,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:28:33,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:34,937 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:36,938 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:37,651 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:28:37,651 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:28:38,939 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:40,940 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:42,941 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:51,851 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:28:52,781 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:28:52,782 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:28:53,086 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:28:53,086 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:28:53,086 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:28:53,945 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:28:53,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:54,946 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:56,947 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:28:58,948 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:00,949 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:02,949 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:07,913 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:29:07,914 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:29:13,123 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:29:13,123 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:29:13,124 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:29:13,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:29:13,954 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:14,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:16,955 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:18,956 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:20,957 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:21,929 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:29:22,958 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:23,046 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:29:23,047 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:29:33,164 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:29:33,164 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:29:33,164 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:29:33,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:29:33,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:34,963 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:36,964 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:38,180 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:29:38,181 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:29:38,965 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:40,966 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:42,967 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:52,006 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:29:53,201 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:29:53,202 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:29:53,202 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:29:53,310 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:29:53,311 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:29:53,972 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:29:53,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:54,973 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:56,974 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:29:58,974 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:00,975 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:02,976 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:08,441 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:30:08,441 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:30:13,237 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:30:13,238 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:30:13,238 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:30:13,981 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:30:13,982 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:14,982 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:16,983 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:18,984 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:20,985 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:22,084 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:30:22,986 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:23,575 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:30:23,575 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:30:33,276 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:30:33,276 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:30:33,277 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:30:33,990 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:30:33,991 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:34,991 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:36,992 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:38,705 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:30:38,705 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:30:38,993 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:40,994 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:42,995 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:52,161 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:30:53,316 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:30:53,317 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:30:53,317 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:30:53,860 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:30:53,861 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:30:54,000 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:30:54,000 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:55,000 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:57,001 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:30:59,002 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:01,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:03,003 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:08,999 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:31:08,999 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:31:13,355 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:31:13,355 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:31:13,355 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:31:14,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:31:14,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:15,008 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:17,009 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:19,010 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:21,011 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:22,237 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:31:23,012 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:24,133 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:31:24,134 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:31:33,392 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:31:33,392 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:31:33,393 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:31:34,016 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:31:34,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:35,017 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:37,018 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:39,019 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:39,264 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:31:39,265 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:31:41,020 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:43,021 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:52,305 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:31:53,428 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:31:53,429 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:31:53,429 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:31:54,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:31:54,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:54,402 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:31:54,403 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:31:55,026 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:57,027 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:31:59,028 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:01,029 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:09,649 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:32:09,649 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:32:13,466 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:32:13,466 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:32:13,467 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:32:14,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:32:14,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:15,035 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:17,036 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:19,037 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:21,038 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:22,375 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:32:24,787 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:32:24,787 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:32:33,501 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:32:33,501 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:32:33,502 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:32:34,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:32:34,043 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:35,044 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:37,045 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:39,046 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:39,916 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:32:39,917 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:32:41,048 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:52,443 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:32:53,538 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:32:53,538 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:32:53,539 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:32:54,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:32:55,054 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:55,072 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:32:55,072 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:32:57,055 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:32:59,056 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:01,057 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:10,210 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:33:10,210 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:33:13,574 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:33:13,574 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:33:13,574 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:33:14,063 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:33:15,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:17,064 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:19,065 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:21,066 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:22,513 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:33:23,068 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:25,344 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:33:25,345 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:33:33,612 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:33:33,612 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:33:33,613 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:33:34,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:33:35,073 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:37,074 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:39,075 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:40,493 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:33:40,493 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:33:41,076 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:43,077 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:52,587 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:33:53,648 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:33:53,648 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:33:53,649 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:33:54,082 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:33:55,083 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:55,629 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:33:55,630 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:33:57,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:33:59,084 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:01,086 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:10,766 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:34:10,766 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:34:13,686 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:34:13,686 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:34:13,687 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:34:14,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:34:15,091 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:17,092 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:19,093 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:21,094 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:22,657 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:34:25,896 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:34:25,896 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:34:33,723 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:34:33,724 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:34:33,724 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:34:34,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:34:35,099 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:37,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:39,100 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:41,029 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:34:41,029 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:34:41,101 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:52,733 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:34:53,760 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:34:53,760 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:34:53,761 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:34:54,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:34:55,106 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:34:56,161 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:34:56,162 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:34:58,108 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:00,109 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:02,110 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:11,297 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:35:11,298 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:35:13,800 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:35:13,800 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:35:13,801 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:35:14,115 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:35:15,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:16,116 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:18,117 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:20,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:22,118 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:22,808 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:35:26,431 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:35:26,432 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:35:33,837 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:35:33,837 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:35:33,838 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:35:34,123 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:35:35,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:36,124 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:38,125 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:40,126 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:41,563 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:35:41,564 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:35:42,127 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:52,885 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:35:53,876 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:35:53,877 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:35:53,877 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:35:54,131 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:35:55,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:56,132 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:35:56,695 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:35:56,695 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:35:58,133 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:00,134 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:02,135 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:11,827 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:36:11,828 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:36:13,916 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:36:13,916 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:36:13,917 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:36:14,139 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:36:15,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:16,140 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:18,141 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:20,142 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:22,143 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:22,964 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:36:26,977 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:36:26,977 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:36:33,954 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:36:33,955 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:36:33,955 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:36:34,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:36:35,148 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:36,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:38,149 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:40,150 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:42,110 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:36:42,110 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:36:42,151 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:53,043 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:36:54,019 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:36:54,019 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:36:54,019 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:36:54,155 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:36:55,156 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:56,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:36:57,242 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:36:57,243 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:36:58,157 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:00,158 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:02,159 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:12,374 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:37:12,375 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:37:14,055 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:37:14,055 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:37:14,055 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:37:14,164 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:37:15,164 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:16,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:18,165 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:20,166 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:22,167 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:23,123 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:37:27,509 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:37:27,509 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:37:34,093 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:37:34,093 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:37:34,094 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:37:34,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:37:35,172 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:36,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:38,173 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:40,174 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:42,175 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:42,639 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:37:42,639 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:37:53,199 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:37:54,134 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:37:54,134 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:37:54,135 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:37:54,179 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:37:55,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:56,180 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:37:57,780 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:37:57,780 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:37:58,181 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:00,182 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:02,183 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:12,916 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:38:12,916 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:38:14,175 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:38:14,175 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:38:14,176 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:38:14,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:38:15,188 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:16,189 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:18,190 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:20,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:22,191 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:23,274 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:38:28,046 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:38:28,047 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:38:34,213 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:38:34,214 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:38:34,214 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:38:35,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:38:35,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:36,197 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:38,198 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:40,199 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:42,200 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:43,250 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:38:43,251 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:38:53,351 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:38:54,252 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:38:54,253 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:38:54,253 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:38:55,205 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:38:55,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:56,206 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:58,207 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:38:58,389 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:38:58,389 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:39:00,208 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:02,209 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:13,573 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:39:13,574 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:39:14,292 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:39:14,292 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:39:14,293 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:39:15,214 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:39:15,214 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:16,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:18,215 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:20,216 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:22,217 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:23,427 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:39:28,754 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:39:28,754 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:39:34,331 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:39:34,332 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:39:34,332 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:39:35,222 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:39:35,223 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:36,223 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:38,224 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:40,225 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:42,226 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:43,886 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:39:43,886 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:39:53,507 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:39:54,369 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:39:54,370 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:39:54,370 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:39:55,231 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:39:55,231 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:56,231 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:58,232 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:39:59,020 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:39:59,020 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:40:00,233 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:02,234 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:14,409 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:40:14,410 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:40:14,410 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:40:14,584 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:40:14,584 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:40:15,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:40:15,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:16,240 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:18,241 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:20,242 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:22,243 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:23,584 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:40:29,723 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:40:29,723 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:40:34,451 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:40:34,451 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:40:34,452 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:40:35,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:40:35,248 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:36,249 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:38,249 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:40,250 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:42,251 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:44,863 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:40:44,864 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:40:53,661 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:40:54,489 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:40:54,489 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:40:54,490 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:40:55,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:40:55,256 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:56,257 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:40:58,257 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:00,003 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:41:00,003 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:41:00,258 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:02,259 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:14,525 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:41:14,526 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:41:14,526 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:41:15,143 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:41:15,144 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:41:15,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:41:15,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:16,265 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:18,266 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:20,267 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:22,268 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:23,740 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:41:30,283 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:41:30,283 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:41:34,562 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:41:34,562 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:41:34,563 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:41:35,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:41:35,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:36,274 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:38,275 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:40,276 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:42,277 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:45,415 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:41:45,415 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:41:53,821 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:41:54,599 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:41:54,599 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:41:54,600 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:41:55,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:41:55,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:56,282 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:41:58,283 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:00,284 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:00,549 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:42:00,550 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:42:02,285 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:14,636 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:42:14,636 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:42:14,636 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:42:15,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:42:15,718 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:42:15,719 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:42:16,290 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:18,291 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:20,292 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:22,293 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:23,899 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:42:30,860 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:42:30,860 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:42:34,671 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:42:34,671 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:42:34,672 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:42:35,298 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:42:36,299 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:38,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:40,300 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:42,301 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:45,996 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:42:45,996 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:42:53,977 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:42:54,707 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:42:54,708 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:42:54,708 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:42:55,306 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:42:56,307 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:42:58,308 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:00,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:01,126 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:43:01,127 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:43:02,309 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:14,744 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:43:14,744 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:43:14,744 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:43:15,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:43:16,281 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:43:16,282 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:43:16,315 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:18,316 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:20,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:22,317 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:24,055 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:43:31,422 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:43:31,422 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:43:34,780 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:43:34,780 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:43:34,780 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:43:35,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:43:36,323 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:38,324 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:40,325 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:42,326 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:46,556 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:43:46,557 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:43:54,134 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:43:54,819 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:43:54,819 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:43:54,820 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:43:55,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:43:56,331 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:43:58,332 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:00,333 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:01,703 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:44:01,704 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:44:02,334 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:14,858 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:44:14,858 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:44:14,859 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:44:15,338 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:44:16,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:16,866 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:44:16,867 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:44:18,339 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:20,340 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:22,341 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:24,214 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:44:32,007 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:44:32,007 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:44:34,894 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:44:34,895 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:44:34,895 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:44:35,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:44:36,346 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:37,347 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:39,348 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:41,349 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:43,350 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:45,351 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:47,140 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:44:47,141 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:44:47,352 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:49,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:51,353 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:53,354 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:54,297 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:44:55,355 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:57,356 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:44:59,357 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:01,357 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:02,286 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:45:02,286 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:45:03,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:05,359 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:07,360 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:09,361 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:11,362 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:13,363 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:15,364 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:17,365 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:17,429 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:45:17,429 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:45:19,366 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:21,367 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:23,368 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:24,396 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:45:25,369 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:27,370 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:29,371 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:31,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:32,592 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:45:32,592 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:45:33,372 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:35,373 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:37,374 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:38,868 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:45:38,869 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:45:38,869 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:45:39,375 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:45:41,376 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:43,377 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:45,378 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:47,379 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:45:47,726 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:45:47,726 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:45:54,473 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:45:58,916 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:45:58,916 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:45:58,917 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:45:59,384 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:46:00,384 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:01,384 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:02,873 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:46:02,874 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:46:03,385 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:05,386 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:07,387 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:18,009 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:46:18,009 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:46:18,951 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:46:18,952 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:46:18,952 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:46:19,392 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:46:20,393 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:21,393 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:23,394 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:24,550 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:46:25,395 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:27,396 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:33,148 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:46:33,149 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:46:38,986 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:46:38,986 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:46:38,987 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:46:39,401 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:46:40,401 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:41,402 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:43,403 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:45,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:47,404 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:46:48,288 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:46:48,289 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:46:54,628 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:46:59,022 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:46:59,023 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:46:59,023 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:46:59,409 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:47:00,410 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:02,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:03,420 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:47:03,420 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:47:04,411 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:06,412 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:08,413 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:18,549 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:47:18,550 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:47:19,059 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:47:19,060 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:47:19,060 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:47:19,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:47:20,418 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:21,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:22,419 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:24,420 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:24,706 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:47:26,421 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:28,422 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:33,681 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:47:33,682 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:47:39,094 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:47:39,095 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:47:39,095 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:47:39,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:47:41,427 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:42,428 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:44,429 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:46,430 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:48,431 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:47:48,814 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:47:48,814 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:47:54,784 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:47:59,131 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:47:59,132 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:47:59,132 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:47:59,435 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:48:01,436 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:02,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:03,967 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:48:03,967 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:48:04,437 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:06,438 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:08,439 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:19,101 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:48:19,101 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:48:19,230 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:48:19,231 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:48:19,231 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:48:19,444 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:48:21,445 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:22,445 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:24,446 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:24,862 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:48:26,447 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:28,448 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:34,232 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:48:34,232 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:48:39,208 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:48:39,208 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:48:39,209 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:48:39,453 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:48:40,453 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:41,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:42,454 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:44,455 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:46,456 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:48,457 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:48:49,363 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:48:49,364 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:48:54,939 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:48:59,250 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:48:59,250 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:48:59,250 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:48:59,462 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:49:00,462 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:01,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:02,463 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:04,464 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:04,502 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:49:04,502 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:49:06,465 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:08,466 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:19,288 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:49:19,289 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:49:19,289 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:49:19,471 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:49:19,642 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:49:19,643 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:49:20,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:21,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:22,472 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:24,473 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:25,017 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:49:26,474 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:28,475 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:34,778 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:49:34,778 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:49:39,327 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:49:39,328 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:49:39,328 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:49:39,480 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:49:40,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:41,481 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:42,482 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:44,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:46,483 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:48,485 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:49:49,909 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:49:49,910 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:49:55,095 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:49:59,367 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:49:59,367 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:49:59,368 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:49:59,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:50:00,490 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:01,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:02,491 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:04,492 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:05,056 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:50:05,056 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:50:06,493 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:08,494 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:19,406 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:50:19,406 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:50:19,407 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:50:19,499 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:50:20,210 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:50:20,210 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:50:20,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:21,500 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:22,501 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:24,502 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:25,173 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:50:26,503 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:28,504 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:35,347 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:50:35,347 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:50:39,444 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:50:39,445 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:50:39,445 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:50:39,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:50:40,509 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:41,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:42,510 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:44,511 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:46,512 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:48,513 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:50:50,479 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:50:50,479 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:50:55,251 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:50:59,484 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:50:59,484 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:50:59,485 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:50:59,518 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:51:00,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:01,519 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:02,520 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:04,521 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:05,610 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:51:05,611 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:51:06,522 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:08,523 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:19,525 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:51:19,526 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:51:19,526 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:51:19,528 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:51:20,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:20,741 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:51:20,742 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:51:21,529 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:22,530 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:24,531 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:25,329 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:51:26,532 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:28,533 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:35,878 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:51:35,879 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:51:39,561 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:51:39,562 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:51:39,562 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:51:40,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:51:40,538 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:41,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:42,539 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:44,540 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:46,541 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:48,542 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:51:51,010 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:51:51,010 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:51:55,405 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:51:59,601 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:51:59,601 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:51:59,601 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:52:00,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:52:00,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:01,547 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:02,548 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:04,549 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:06,140 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:52:06,140 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:52:06,550 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:08,551 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:19,636 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:52:19,637 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:52:19,637 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:52:20,556 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:52:20,556 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:21,299 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:52:21,300 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:52:21,556 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:22,557 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:24,557 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:25,484 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:52:26,558 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:28,559 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:36,434 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:52:36,435 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:52:39,676 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:52:39,676 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:52:39,677 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:52:40,565 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:52:40,565 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:41,565 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:42,566 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:44,566 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:46,567 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:48,568 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:52:51,566 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:52:51,567 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:52:55,561 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:52:59,714 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:52:59,715 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:52:59,715 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:53:00,573 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:53:00,573 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:01,573 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:02,574 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:04,574 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:06,575 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:06,698 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:53:06,698 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:53:08,576 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:19,751 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:53:19,751 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:53:19,752 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:53:20,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:53:20,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:21,582 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:21,970 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:53:21,971 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:53:22,583 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:24,583 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:25,638 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:53:26,584 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:28,585 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:37,131 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:53:37,131 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:53:39,789 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:53:39,790 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:53:39,790 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:53:40,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:53:40,590 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:41,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:42,591 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:44,592 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:46,593 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:48,594 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:53:52,264 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:53:52,264 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:53:55,717 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:53:59,825 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:53:59,826 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:53:59,826 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:54:00,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:54:00,599 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:01,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:02,600 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:04,601 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:06,602 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:07,429 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:54:07,429 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:54:08,603 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:19,863 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:54:19,863 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:54:19,864 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:54:20,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:54:20,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:21,608 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:22,560 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:54:22,561 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:54:22,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:24,609 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:25,796 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:54:26,610 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:28,611 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:37,697 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:54:37,698 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:54:39,899 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:54:39,899 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:54:39,900 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:54:40,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:54:40,616 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:41,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:42,617 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:44,618 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:46,619 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:48,620 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:54:52,838 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:54:52,839 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:54:55,875 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:54:59,937 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:54:59,937 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:54:59,938 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:55:00,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:55:00,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:01,625 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:02,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:04,626 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:06,627 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:08,628 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:10,610 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:55:10,611 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:55:19,973 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:55:19,974 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:55:19,974 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:55:20,633 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:55:21,634 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:23,635 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:25,636 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:25,743 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:55:25,743 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:55:25,955 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:55:27,637 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:29,638 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:40,010 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:55:40,011 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:55:40,011 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:55:40,642 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:55:40,884 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:55:40,885 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:55:41,643 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:43,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:45,644 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:47,645 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:49,646 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:55:56,034 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:55:56,056 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:55:56,056 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:56:00,049 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:56:00,049 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:56:00,050 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:56:00,651 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:56:01,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:03,652 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:05,653 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:07,654 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:09,655 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:11,187 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:56:11,187 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:56:20,087 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:56:20,087 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:56:20,088 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:56:20,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:56:21,660 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:23,661 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:25,662 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:26,111 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:56:26,321 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:56:26,322 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:56:27,663 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:29,664 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:40,125 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:56:40,126 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:56:40,126 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:56:40,668 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:56:41,474 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:56:41,475 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:56:41,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:42,669 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:43,670 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:45,671 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:47,672 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:49,673 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:56:56,189 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:56:56,729 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:56:56,729 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:57:00,164 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:57:00,164 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:57:00,165 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:57:00,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:57:01,678 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:02,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:03,679 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:05,680 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:07,681 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:09,682 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:11,861 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:57:11,861 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:57:20,203 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:57:20,204 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:57:20,204 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:57:20,686 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:57:21,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:22,687 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:23,688 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:25,689 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:26,266 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:57:26,992 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:57:26,993 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:57:27,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:29,690 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:40,248 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:57:40,249 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:57:40,249 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:57:40,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:57:41,695 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:42,125 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:57:42,126 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:57:42,696 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:43,696 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:45,697 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:47,698 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:49,699 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:57:56,344 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:57:57,283 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:57:57,284 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:58:00,288 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:58:00,288 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:58:00,288 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:58:00,703 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:58:01,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:02,704 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:03,705 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:05,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:07,706 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:09,707 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:12,418 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:58:12,418 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:58:20,328 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:58:20,328 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:58:20,329 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:58:20,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:58:21,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:22,712 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:23,713 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:25,714 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:26,422 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:58:27,549 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:58:27,549 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:58:27,714 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:29,715 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:40,366 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:58:40,366 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:58:40,366 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:58:40,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:58:41,720 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:42,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:43,488 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:58:43,488 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:58:43,721 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:45,722 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:47,723 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:49,724 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:58:56,500 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:58:58,704 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:58:58,705 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:59:00,403 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:59:00,404 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:59:00,404 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:59:00,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:59:01,728 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:02,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:03,729 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:05,730 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:07,731 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:09,732 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:13,840 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:59:13,840 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:59:20,443 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:59:20,443 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:59:20,443 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:59:20,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:59:21,737 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:22,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:23,738 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:25,739 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:26,577 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:59:27,740 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:28,986 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:59:28,986 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:59:29,741 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:40,482 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 16:59:40,482 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 16:59:40,482 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 16:59:40,745 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 16:59:41,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:42,746 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:43,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:44,117 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:59:44,118 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 16:59:45,747 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:47,748 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:49,749 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 16:59:56,655 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 16:59:59,248 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 16:59:59,249 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:00:00,518 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:00:00,519 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:00:00,519 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:00:00,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:00:01,754 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:02,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:03,755 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:05,756 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:07,757 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:09,758 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:14,379 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:00:14,379 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:00:20,556 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:00:20,557 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:00:20,557 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:00:20,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:00:21,763 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:22,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:23,764 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:25,765 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:26,732 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:00:27,766 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:29,511 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:00:29,511 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:00:29,767 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:40,593 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:00:40,593 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:00:40,594 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:00:40,771 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:00:41,772 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:42,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:43,773 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:44,650 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:00:44,650 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:00:45,774 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:47,775 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:49,776 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:00:56,810 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:00:59,829 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:00:59,829 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:01:00,628 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:01:00,629 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:01:00,629 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:01:00,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:01:01,782 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:02,783 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:03,783 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:05,784 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:07,785 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:09,786 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:14,960 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:01:14,961 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:01:20,663 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:01:20,664 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:01:20,664 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:01:20,791 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:01:21,792 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:22,792 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:23,792 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:25,793 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:26,887 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:01:27,795 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:29,796 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:30,092 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:01:30,093 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:01:40,699 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:01:40,699 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:01:40,700 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:01:40,801 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:01:42,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:43,802 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:45,227 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:01:45,227 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:01:45,803 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:47,804 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:49,805 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:01:56,966 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:02:00,360 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:02:00,360 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:02:00,736 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:02:00,736 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:02:00,736 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:02:00,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:02:01,810 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:02,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:03,811 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:05,812 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:07,814 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:09,815 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:15,491 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:02:15,491 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:02:20,770 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:02:20,770 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:02:20,771 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:02:20,819 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:02:21,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:22,820 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:23,821 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:25,822 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:27,039 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:02:27,823 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:29,824 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:30,621 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:02:30,622 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:02:40,809 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:02:40,809 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:02:40,809 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:02:40,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:02:41,829 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:42,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:43,830 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:45,758 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:02:45,758 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:02:45,831 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:47,832 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:49,834 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:02:57,107 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:03:00,844 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:03:00,844 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:03:00,845 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:03:00,902 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:03:00,903 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:03:01,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:03:01,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:02,839 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:03,840 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:05,841 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:07,842 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:09,843 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:16,036 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:03:16,037 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:03:20,884 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:03:20,884 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:03:20,884 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:03:21,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:03:21,848 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:22,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:23,849 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:25,850 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:27,177 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:03:27,851 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:29,852 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:31,169 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:03:31,169 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:03:40,919 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:03:40,920 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:03:40,920 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:03:41,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:03:41,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:42,858 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:43,859 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:45,860 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:46,313 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:03:46,314 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:03:47,861 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:49,862 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:03:57,247 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:04:00,955 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:04:00,956 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:04:00,956 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:04:01,447 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:04:01,447 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:04:01,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:04:01,867 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:02,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:03,868 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:05,869 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:07,870 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:09,871 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:16,588 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:04:16,589 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:04:20,993 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:04:20,994 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:04:20,994 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:04:21,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:04:21,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:22,877 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:23,878 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:25,879 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:27,314 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:04:27,880 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:29,881 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:31,718 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:04:31,718 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:04:41,029 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:04:41,029 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:04:41,030 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:04:41,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:04:41,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:42,886 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:43,887 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:45,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:46,851 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:04:46,852 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:04:47,888 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:49,889 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:04:57,382 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:05:01,064 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:05:01,065 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:05:01,065 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:05:01,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:05:01,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:01,992 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:05:01,993 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:05:02,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:03,895 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:05,896 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:07,897 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:09,898 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:17,124 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:05:17,124 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:05:21,105 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:05:21,105 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:05:21,106 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:05:21,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:05:21,903 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:22,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:23,904 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:25,905 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:27,453 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:05:27,906 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:29,907 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:32,256 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:05:32,257 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:05:41,144 DEBUG SenderThread:783720 [sender.py:send():179] send: history +2021-07-16 17:05:41,145 DEBUG SenderThread:783720 [sender.py:send():179] send: summary +2021-07-16 17:05:41,145 INFO SenderThread:783720 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 17:05:41,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/wandb-summary.json +2021-07-16 17:05:41,912 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:42,913 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:48,269 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:05:48,269 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status +2021-07-16 17:05:48,915 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:50,916 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:52,918 INFO Thread-8 :783720 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_095921-13hxxunp/files/output.log +2021-07-16 17:05:57,525 DEBUG SenderThread:783720 [sender.py:send():179] send: stats +2021-07-16 17:06:03,402 DEBUG HandlerThread:783720 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 17:06:03,402 DEBUG SenderThread:783720 [sender.py:send_request():193] send_request: stop_status