diff --git "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" --- "a/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" +++ "b/wandb/run-20210716_005946-3bte92ft/logs/debug-internal.log" @@ -11505,3 +11505,3775 @@ 2021-07-16 06:21:50,057 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status 2021-07-16 06:21:50,058 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status 2021-07-16 06:21:50,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:22:04,041 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:22:05,193 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:22:05,194 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:22:20,324 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:22:20,325 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:22:34,106 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:22:35,455 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:22:35,478 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:22:50,614 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:22:50,614 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:23:04,171 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:23:05,746 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:23:05,746 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:23:14,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:16,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:18,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:20,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:20,877 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:23:20,878 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:23:33,033 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:23:33,033 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:23:33,034 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:23:33,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:23:34,241 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:23:34,639 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:36,020 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:23:36,020 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:23:36,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:38,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:40,642 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:42,643 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:51,151 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:23:51,152 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:23:53,083 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:23:53,083 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:23:53,084 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:23:53,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:23:54,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:56,649 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:23:58,650 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:00,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:02,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:04,307 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:24:06,286 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:24:06,286 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:24:13,125 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:24:13,125 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:24:13,125 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:24:13,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:24:14,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:16,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:18,658 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:20,659 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:21,425 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:24:21,425 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:24:22,660 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:33,168 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:24:33,168 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:24:33,169 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:24:33,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:24:34,373 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:24:34,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:36,561 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:24:36,561 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:24:36,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:38,667 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:40,668 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:42,669 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:51,692 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:24:51,692 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:24:53,210 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:24:53,211 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:24:53,211 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:24:53,673 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:24:54,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:56,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:24:58,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:00,676 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:02,677 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:04,441 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:25:06,828 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:25:06,828 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:25:13,257 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:25:13,258 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:25:13,258 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:25:13,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:25:14,682 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:16,683 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:18,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:20,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:21,960 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:25:21,961 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:25:22,686 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:33,305 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:25:33,305 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:25:33,306 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:25:33,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:25:34,507 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:25:34,691 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:36,692 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:37,091 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:25:37,092 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:25:38,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:40,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:42,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:52,228 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:25:52,229 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:25:53,345 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:25:53,345 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:25:53,345 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:25:53,700 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:25:54,700 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:56,701 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:25:58,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:00,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:02,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:04,573 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:26:07,363 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:26:07,363 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:26:13,384 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:26:13,384 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:26:13,384 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:26:13,708 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:26:14,708 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:16,709 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:18,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:20,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:22,495 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:26:22,495 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:26:22,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:33,432 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:26:33,432 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:26:33,433 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:26:33,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:26:34,634 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:26:34,717 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:36,718 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:37,654 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:26:37,655 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:26:38,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:40,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:42,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:52,785 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:26:52,786 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:26:53,470 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:26:53,471 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:26:53,471 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:26:53,725 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:26:54,726 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:56,727 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:26:58,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:00,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:02,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:04,705 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:27:07,918 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:27:07,918 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:27:13,509 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:27:13,509 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:27:13,510 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:27:13,734 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:27:14,734 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:16,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:18,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:20,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:23,065 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:27:23,065 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:27:33,549 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:27:33,550 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:27:33,550 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:27:33,742 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:27:34,743 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:34,771 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:27:36,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:38,196 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:27:38,197 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:27:38,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:40,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:53,328 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:27:53,329 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:27:53,588 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:27:53,588 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:27:53,589 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:27:53,751 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:27:54,751 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:56,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:27:59,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:01,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:04,838 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:28:08,461 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:28:08,462 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:28:13,629 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:28:13,630 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:28:13,630 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:28:13,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:28:15,761 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:17,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:19,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:21,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:23,592 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:28:23,593 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:28:23,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:33,667 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:28:33,668 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:28:33,668 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:28:33,769 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:28:34,905 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:28:35,770 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:37,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:38,828 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:28:38,828 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:28:39,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:41,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:43,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:53,703 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:28:53,704 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:28:53,704 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:28:53,778 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:28:53,962 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:28:53,962 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:28:55,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:57,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:28:59,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:01,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:04,970 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:29:09,128 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:29:09,129 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:29:13,742 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:29:13,743 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:29:13,743 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:29:13,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:29:15,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:17,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:19,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:21,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:24,276 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:29:24,277 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:29:33,779 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:29:33,780 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:29:33,780 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:29:33,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:29:35,036 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:29:35,796 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:37,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:39,408 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:29:39,409 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:29:39,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:41,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:53,817 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:29:53,818 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:29:53,818 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:29:54,548 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:29:54,549 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:29:54,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:29:55,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:57,805 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:29:59,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:01,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:05,106 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:30:09,690 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:30:09,690 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:30:13,856 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:30:13,857 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:30:13,857 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:30:14,812 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:30:15,812 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:17,813 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:19,814 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:21,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:24,822 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:30:24,822 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:30:33,895 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:30:33,895 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:30:33,896 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:30:34,821 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:30:35,171 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:30:35,821 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:37,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:39,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:39,981 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:30:39,981 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:30:41,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:53,932 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:30:53,932 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:30:53,933 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:30:54,829 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:30:55,113 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:30:55,113 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:30:55,830 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:57,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:30:59,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:01,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:05,242 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:31:10,254 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:31:10,254 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:31:13,970 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:31:13,971 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:31:13,971 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:31:14,838 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:31:15,838 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:17,839 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:19,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:21,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:25,387 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:31:25,387 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:31:34,016 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:31:34,016 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:31:34,017 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:31:34,846 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:31:35,316 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:31:35,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:37,848 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:39,848 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:40,518 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:31:40,518 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:31:41,849 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:54,056 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:31:54,057 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:31:54,057 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:31:54,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:31:55,649 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:31:55,650 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:31:55,855 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:57,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:31:59,857 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:01,858 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:05,388 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:32:10,789 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:32:10,789 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:32:14,096 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:32:14,096 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:32:14,097 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:32:14,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:32:15,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:17,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:19,865 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:21,866 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:25,921 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:32:25,921 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:32:34,143 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:32:34,144 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:32:34,144 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:32:34,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:32:35,459 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:32:35,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:37,873 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:39,874 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:41,060 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:32:41,060 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:32:41,874 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:54,180 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:32:54,180 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:32:54,181 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:32:54,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:32:55,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:56,297 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:32:56,297 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:32:57,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:32:59,882 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:01,883 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:05,532 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:33:11,551 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:33:11,551 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:33:14,218 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:33:14,219 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:33:14,219 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:33:14,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:33:15,889 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:17,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:19,891 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:21,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:26,685 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:33:26,685 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:33:34,255 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:33:34,255 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:33:34,256 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:33:34,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:33:35,607 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:33:35,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:37,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:39,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:41,815 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:33:41,815 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:33:41,900 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:54,293 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:33:54,293 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:33:54,294 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:33:54,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:33:55,907 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:56,967 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:33:56,967 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:33:57,908 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:33:59,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:01,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:05,679 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:34:12,101 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:34:12,101 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:34:14,335 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:34:14,336 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:34:14,336 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:34:14,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:34:15,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:17,917 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:19,918 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:21,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:27,231 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:34:27,232 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:34:34,371 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:34:34,371 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:34:34,372 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:34:34,924 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:34:35,751 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:34:35,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:37,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:39,926 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:41,927 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:42,364 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:34:42,365 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:34:43,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:54,420 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:34:54,420 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:34:54,421 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:34:54,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:34:55,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:57,496 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:34:57,497 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:34:57,934 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:34:59,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:01,936 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:03,937 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:05,826 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:35:12,629 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:35:12,629 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:35:14,457 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:35:14,457 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:35:14,457 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:35:14,942 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:35:15,942 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:17,943 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:19,944 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:21,945 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:27,762 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:35:27,762 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:35:34,497 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:35:34,498 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:35:34,498 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:35:34,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:35:35,896 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:35:35,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:37,951 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:39,952 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:41,953 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:42,896 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:35:42,896 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:35:54,536 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:35:54,536 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:35:54,536 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:35:54,959 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:35:55,959 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:57,960 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:35:58,028 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:35:58,029 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:35:59,961 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:01,961 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:05,965 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:36:13,162 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:36:13,162 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:36:14,573 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:36:14,574 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:36:14,574 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:36:14,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:36:16,969 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:18,970 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:20,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:22,972 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:28,296 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:36:28,296 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:36:34,611 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:36:34,611 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:36:34,612 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:36:34,977 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:36:36,034 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:36:36,978 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:38,979 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:40,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:42,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:43,429 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:36:43,429 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:36:54,649 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:36:54,649 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:36:54,649 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:36:54,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:36:56,987 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:36:58,562 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:36:58,563 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:36:58,988 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:00,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:02,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:06,108 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:37:13,695 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:37:13,695 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:37:14,686 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:37:14,687 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:37:14,687 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:37:14,996 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:37:16,996 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:18,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:20,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:22,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:28,869 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:37:28,869 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:37:34,731 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:37:34,731 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:37:34,732 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:37:35,004 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:37:36,179 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:37:37,005 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:39,006 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:41,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:43,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:44,001 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:37:44,001 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:37:54,768 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:37:54,769 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:37:54,769 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:37:55,013 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:37:57,014 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:59,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:37:59,138 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:37:59,139 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:38:01,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:03,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:06,255 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:38:14,276 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:38:14,276 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:38:14,804 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:38:14,804 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:38:14,804 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:38:15,023 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:38:17,024 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:19,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:21,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:23,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:29,409 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:38:29,409 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:38:34,843 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:38:34,843 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:38:34,844 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:38:35,032 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:38:36,327 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:38:37,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:39,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:41,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:43,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:44,541 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:38:44,541 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:38:54,881 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:38:54,881 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:38:54,881 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:38:55,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:38:57,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:59,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:38:59,674 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:38:59,675 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:39:01,045 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:03,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:06,400 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:39:14,807 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:39:14,807 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:39:14,936 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:39:14,936 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:39:14,937 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:39:15,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:39:17,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:19,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:21,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:23,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:29,963 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:39:29,964 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:39:34,962 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:39:34,963 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:39:34,963 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:39:35,059 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:39:36,471 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:39:37,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:39,061 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:41,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:43,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:45,104 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:39:45,104 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:39:54,999 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:39:55,000 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:39:55,000 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:39:55,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:39:57,069 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:39:59,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:00,235 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:40:00,235 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:40:01,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:03,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:06,547 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:40:15,035 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:40:15,035 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:40:15,036 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:40:15,076 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:40:15,395 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:40:15,395 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:40:17,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:19,078 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:21,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:23,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:30,538 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:40:30,539 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:40:35,084 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:40:35,085 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:40:35,086 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:40:35,086 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:40:36,623 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:40:37,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:39,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:41,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:43,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:45,670 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:40:45,670 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:40:55,124 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:40:55,124 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:40:55,124 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:40:56,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:40:57,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:40:59,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:00,811 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:41:00,811 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:41:01,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:03,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:06,699 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:41:15,164 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:41:15,165 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:41:15,165 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:41:15,955 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:41:15,956 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:41:16,103 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:41:17,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:19,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:21,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:23,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:31,088 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:41:31,089 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:41:35,205 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:41:35,206 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:41:35,206 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:41:36,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:41:36,776 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:41:37,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:39,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:41,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:43,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:46,221 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:41:46,221 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:41:55,252 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:41:55,252 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:41:55,253 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:41:56,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:41:57,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:41:59,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:01,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:01,367 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:42:01,367 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:42:03,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:06,852 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:42:15,297 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:42:15,298 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:42:15,298 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:42:16,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:42:16,532 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:42:16,532 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:42:17,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:19,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:21,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:23,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:31,673 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:42:31,673 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:42:35,337 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:42:35,338 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:42:35,338 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:42:36,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:42:36,924 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:42:37,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:39,139 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:41,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:43,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:46,817 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:42:46,817 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:42:55,377 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:42:55,377 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:42:55,378 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:42:56,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:42:57,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:42:59,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:01,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:01,952 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:43:01,952 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:43:03,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:06,999 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:43:15,421 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:43:15,421 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:43:15,422 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:43:16,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:43:17,086 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:43:17,086 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:43:17,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:19,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:21,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:23,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:25,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:27,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:29,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:31,161 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:32,223 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:43:32,223 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:43:33,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:35,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:37,074 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:43:37,163 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:39,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:42,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:44,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:46,167 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:47,380 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:43:47,381 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:43:48,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:50,169 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:52,169 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:54,170 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:56,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:43:58,172 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:00,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:02,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:02,510 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:44:02,510 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:44:04,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:06,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:07,150 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:44:08,176 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:10,177 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:12,178 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:14,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:16,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:17,686 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:44:17,687 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:44:19,275 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:44:19,275 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:44:19,276 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:44:20,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:20,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:44:22,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:24,183 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:26,184 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:28,185 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:32,819 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:44:32,819 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:44:37,223 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:44:39,324 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:44:39,324 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:44:39,325 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:44:40,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:40,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:44:42,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:44,192 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:46,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:47,948 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:44:47,949 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:44:48,194 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:44:59,362 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:44:59,363 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:44:59,363 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:45:00,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:45:02,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:03,083 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:45:03,084 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:45:04,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:06,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:07,294 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:45:08,202 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:18,221 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:45:18,222 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:45:19,415 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:45:19,415 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:45:19,416 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:45:20,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:45:22,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:24,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:26,210 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:28,211 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:33,355 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:45:33,356 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:45:37,370 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:45:39,459 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:45:39,459 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:45:39,460 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:45:40,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:40,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:45:42,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:44,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:46,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:48,220 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:45:48,485 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:45:48,486 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:45:59,494 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:45:59,495 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:45:59,495 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:46:00,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:00,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:46:02,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:03,616 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:46:03,617 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:46:04,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:06,228 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:07,442 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:46:08,229 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:18,749 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:46:18,749 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:46:19,531 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:46:19,532 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:46:19,532 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:46:20,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:20,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:46:22,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:24,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:26,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:28,237 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:33,884 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:46:33,884 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:46:37,514 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:46:39,568 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:46:39,568 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:46:39,569 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:46:40,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:46:42,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:44,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:46,245 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:48,246 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:46:49,015 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:46:49,016 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:46:59,606 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:46:59,607 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:46:59,607 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:47:00,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:00,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:47:02,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:04,147 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:47:04,147 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:47:04,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:06,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:07,585 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:47:08,254 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:19,278 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:47:19,279 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:47:19,646 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:47:19,646 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:47:19,646 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:47:20,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:47:22,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:24,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:26,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:28,263 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:34,410 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:47:34,411 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:47:37,651 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:47:39,685 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:47:39,685 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:47:39,685 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:47:40,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:47:42,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:44,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:46,271 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:48,272 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:47:49,545 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:47:49,545 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:47:59,723 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:47:59,724 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:47:59,724 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:48:00,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:48:02,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:04,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:04,685 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:48:04,685 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:48:06,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:07,721 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:48:08,280 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:19,761 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:48:19,762 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:48:19,762 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:48:19,819 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:48:19,820 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:48:20,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:20,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:48:22,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:24,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:26,288 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:28,289 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:34,953 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:48:34,954 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:48:37,794 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:48:39,798 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:48:39,798 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:48:39,799 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:48:40,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:40,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:48:42,295 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:44,296 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:46,297 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:48,298 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:48:50,083 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:48:50,083 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:48:59,837 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:48:59,837 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:48:59,838 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:49:00,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:00,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:49:02,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:04,304 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:05,216 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:49:05,216 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:49:06,305 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:07,865 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:49:08,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:19,875 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:49:19,876 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:49:19,876 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:49:20,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:20,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:49:20,348 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:49:20,349 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:49:22,312 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:24,313 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:26,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:28,315 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:35,484 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:49:35,485 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:49:37,939 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:49:39,915 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:49:39,915 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:49:39,916 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:49:40,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:49:41,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:42,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:43,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:45,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:47,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:49,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:49:50,626 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:49:50,626 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:49:59,953 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:49:59,953 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:49:59,954 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:50:00,329 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:50:02,330 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:03,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:05,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:05,760 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:50:05,760 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:50:07,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:08,009 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:50:09,334 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:19,992 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:50:19,992 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:50:19,993 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:50:20,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:50:20,913 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:50:20,914 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:50:23,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:25,341 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:27,342 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:29,343 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:36,043 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:50:36,044 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:50:38,084 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:50:40,032 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:50:40,033 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:50:40,033 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:50:40,348 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:50:43,349 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:45,350 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:47,351 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:49,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:50:51,177 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:50:51,177 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:51:00,078 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:51:00,078 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:51:00,079 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:51:00,357 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:51:03,358 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:05,359 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:06,307 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:51:06,308 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:51:07,360 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:08,157 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:51:09,361 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:20,119 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:51:20,119 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:51:20,120 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:51:20,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:51:21,366 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:21,440 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:51:21,440 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:51:23,367 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:25,368 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:27,369 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:29,371 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:36,602 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:51:36,602 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:51:38,230 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:51:40,167 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:51:40,167 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:51:40,168 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:51:40,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:51:41,376 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:43,377 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:45,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:47,379 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:49,380 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:51:51,738 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:51:51,738 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:52:00,216 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:52:00,216 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:52:00,217 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:52:00,385 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:52:01,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:03,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:05,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:06,871 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:52:06,871 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:52:07,389 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:08,306 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:52:09,390 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:20,263 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:52:20,264 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:52:20,264 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:52:20,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:52:21,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:22,016 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:52:22,017 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:52:23,396 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:25,397 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:27,398 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:29,399 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:37,150 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:52:37,151 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:52:38,379 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:52:40,314 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:52:40,314 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:52:40,315 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:52:40,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:52:41,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:43,405 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:45,406 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:47,407 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:49,408 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:52:52,281 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:52:52,281 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:53:00,357 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:53:00,358 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:53:00,358 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:53:00,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:53:01,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:03,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:05,414 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:07,413 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:53:07,413 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:53:07,415 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:08,453 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:53:09,416 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:20,405 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:53:20,405 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:53:20,406 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:53:20,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:53:21,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:22,550 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:53:22,550 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:53:23,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:25,423 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:27,424 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:29,425 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:37,711 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:53:37,711 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:53:38,525 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:53:40,442 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:53:40,442 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:53:40,443 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:53:41,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:41,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:53:43,431 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:45,432 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:47,433 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:49,433 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:53:52,840 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:53:52,841 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:54:00,477 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:54:00,477 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:54:00,478 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:54:01,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:01,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:54:03,439 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:05,440 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:07,441 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:07,970 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:54:07,971 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:54:08,597 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:54:09,442 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:20,516 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:54:20,516 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:54:20,517 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:54:21,447 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:21,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:54:23,127 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:54:23,127 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:54:23,448 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:25,449 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:27,450 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:29,451 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:38,259 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:54:38,259 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:54:38,671 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:54:40,560 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:54:40,560 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:54:40,561 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:54:41,455 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:41,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:54:43,456 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:45,457 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:47,458 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:49,459 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:54:53,393 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:54:53,393 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:55:00,597 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:55:00,597 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:55:00,598 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:55:01,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:01,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:55:03,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:05,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:07,466 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:08,523 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:55:08,524 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:55:08,747 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:55:09,467 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:20,633 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:55:20,634 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:55:20,634 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:55:21,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:21,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:55:23,473 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:23,652 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:55:23,653 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:55:25,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:27,475 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:29,476 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:38,784 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:55:38,785 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:55:38,916 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:55:40,669 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:55:40,670 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:55:40,670 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:55:41,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:41,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:55:43,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:45,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:47,484 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:49,484 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:55:53,918 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:55:53,918 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:56:00,707 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:56:00,707 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:56:00,708 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:56:01,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:01,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:56:03,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:05,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:07,492 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:08,896 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:56:09,050 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:56:09,050 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:56:09,493 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:20,743 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:56:20,743 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:56:20,744 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:56:21,498 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:21,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:56:23,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:24,180 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:56:24,180 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:56:25,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:27,501 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:29,502 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:38,971 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:56:39,314 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:56:39,314 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:56:40,780 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:56:40,780 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:56:40,783 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:56:41,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:41,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:56:43,508 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:45,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:47,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:49,510 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:56:54,446 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:56:54,447 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:57:00,824 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:57:00,824 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:57:00,825 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:57:01,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:01,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:57:03,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:05,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:07,518 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:09,046 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:57:09,519 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:09,579 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:57:09,580 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:57:20,863 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:57:20,864 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:57:20,866 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:57:21,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:21,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:57:23,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:24,710 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:57:24,711 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:57:25,526 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:27,526 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:29,527 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:39,120 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:57:39,840 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:57:39,840 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:57:40,905 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:57:40,905 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:57:40,905 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:57:41,532 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:41,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:57:43,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:45,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:47,535 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:49,536 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:57:54,974 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:57:54,974 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:58:00,944 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:58:00,945 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:58:00,947 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:58:01,540 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:01,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:58:02,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:04,542 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:06,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:08,544 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:09,195 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:58:10,104 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:58:10,104 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:58:10,545 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:20,985 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:58:20,985 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:58:20,985 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:58:21,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:21,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:58:22,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:24,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:25,236 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:58:25,237 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:58:26,552 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:28,553 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:30,554 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:39,269 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:58:40,366 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:58:40,367 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:58:41,024 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:58:41,024 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:58:41,024 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:58:41,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:58:42,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:44,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:46,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:48,561 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:50,562 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:58:55,497 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:58:55,497 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:59:01,083 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:59:01,084 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:59:01,084 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:59:01,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:59:02,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:04,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:06,569 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:08,570 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:09,342 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:59:10,571 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:10,631 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:59:10,631 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:59:21,125 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:59:21,126 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:59:21,126 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:59:21,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:59:22,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:24,577 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:25,759 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:59:25,760 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:59:26,578 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:28,579 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:30,580 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:39,414 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 06:59:40,889 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:59:40,889 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 06:59:41,165 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 06:59:41,165 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 06:59:41,166 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 06:59:41,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 06:59:42,585 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:44,586 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:46,586 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:48,587 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:50,588 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 06:59:56,022 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 06:59:56,022 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:00:01,202 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:00:01,203 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:00:01,203 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:00:01,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:00:02,593 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:04,594 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:06,595 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:08,596 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:09,489 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:00:10,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:11,153 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:00:11,153 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:00:21,255 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:00:21,256 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:00:21,256 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:00:21,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:00:22,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:24,603 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:26,286 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:00:26,286 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:00:26,604 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:28,605 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:30,606 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:39,563 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:00:41,300 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:00:41,301 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:00:41,301 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:00:41,430 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:00:41,431 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:00:41,610 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:00:42,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:44,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:46,612 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:48,613 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:50,614 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:00:56,571 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:00:56,572 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:01:01,339 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:01:01,340 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:01:01,340 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:01:01,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:01:02,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:04,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:06,620 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:08,621 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:09,636 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:01:10,622 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:11,706 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:01:11,706 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:01:21,377 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:01:21,378 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:01:21,378 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:01:21,626 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:01:22,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:24,628 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:26,629 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:26,841 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:01:26,841 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:01:28,630 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:30,630 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:39,710 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:01:41,423 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:01:41,424 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:01:41,424 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:01:41,635 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:01:41,981 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:01:41,982 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:01:42,635 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:44,636 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:46,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:48,637 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:50,638 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:01:57,121 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:01:57,121 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:02:01,462 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:02:01,462 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:02:01,462 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:02:01,643 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:02:02,643 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:04,644 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:06,645 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:08,646 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:09,783 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:02:10,647 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:12,250 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:02:12,251 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:02:21,498 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:02:21,498 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:02:21,499 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:02:21,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:02:22,652 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:24,653 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:26,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:27,381 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:02:27,382 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:02:28,654 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:30,655 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:39,855 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:02:41,533 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:02:41,533 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:02:41,533 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:02:41,660 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:02:42,588 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:02:42,589 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:02:42,661 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:44,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:46,662 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:48,663 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:50,664 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:02:57,736 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:02:57,736 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:03:01,568 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:03:01,568 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:03:01,568 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:03:01,668 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:03:02,669 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:04,670 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:06,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:08,671 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:09,927 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:03:10,672 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:12,872 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:03:12,872 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:03:21,624 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:03:21,624 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:03:21,624 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:03:21,677 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:03:22,677 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:24,678 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:26,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:28,010 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:03:28,011 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:03:28,680 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:30,681 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:39,999 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:03:41,659 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:03:41,660 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:03:41,660 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:03:41,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:03:42,686 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:43,147 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:03:43,148 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:03:44,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:46,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:48,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:50,689 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:03:58,291 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:03:58,292 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:04:01,695 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:04:01,695 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:04:01,696 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:04:02,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:02,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:04:04,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:06,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:08,697 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:10,071 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:04:10,697 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:13,426 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:04:13,426 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:04:21,732 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:04:21,732 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:04:21,733 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:04:22,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:22,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:04:24,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:26,704 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:28,564 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:04:28,564 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:04:28,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:40,142 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:04:41,790 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:04:41,790 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:04:41,791 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:04:42,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:42,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:04:43,702 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:04:43,702 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:04:43,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:45,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:47,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:49,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:04:58,844 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:04:58,844 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:05:01,827 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:05:01,827 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:05:01,828 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:05:02,718 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:02,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:05:03,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:05,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:07,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:09,722 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:10,214 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:05:11,723 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:13,973 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:05:13,974 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:05:21,865 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:05:21,865 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:05:21,866 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:05:22,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:22,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:05:23,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:25,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:27,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:29,110 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:05:29,110 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:05:29,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:31,732 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:40,286 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:05:41,901 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:05:41,902 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:05:41,902 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:05:42,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:42,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:05:43,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:44,263 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:05:44,264 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:05:45,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:47,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:49,740 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:51,741 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:05:59,418 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:05:59,418 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:06:01,941 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:06:01,942 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:06:01,942 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:06:02,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:02,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:06:03,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:05,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:07,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:09,748 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:10,360 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:06:11,749 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:14,551 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:06:14,551 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:06:21,985 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:06:21,985 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:06:21,986 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:06:22,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:22,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:06:23,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:25,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:27,756 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:29,686 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:06:29,686 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:06:29,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:31,757 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:40,432 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:06:42,022 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:06:42,023 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:06:42,023 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:06:42,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:06:43,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:44,821 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:06:44,821 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:06:45,763 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:47,764 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:49,765 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:51,766 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:06:59,960 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:06:59,960 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:07:02,065 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:07:02,066 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:07:02,066 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:07:02,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:07:03,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:05,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:07,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:09,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:10,504 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:07:11,775 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:15,090 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:07:15,090 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:07:22,111 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:07:22,111 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:07:22,112 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:07:22,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:07:23,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:25,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:27,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:29,783 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:30,220 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:07:30,220 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:07:31,784 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:40,576 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:07:42,182 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:07:42,182 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:07:42,183 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:07:42,788 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:07:43,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:45,358 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:07:45,358 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:07:45,789 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:47,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:49,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:07:51,792 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:00,491 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:08:00,491 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:08:02,233 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:08:02,234 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:08:02,234 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:08:02,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:08:03,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:05,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:07,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:09,800 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:10,647 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:08:11,801 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:15,628 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:08:15,628 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:08:22,278 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:08:22,279 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:08:22,279 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:08:22,805 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:08:23,805 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:25,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:27,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:29,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:30,758 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:08:30,758 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:08:31,809 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:40,720 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:08:42,323 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:08:42,324 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:08:42,324 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:08:42,813 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:08:43,814 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:45,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:45,889 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:08:45,890 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:08:47,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:49,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:08:51,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:01,023 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:09:01,024 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:09:02,360 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:09:02,361 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:09:02,361 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:09:02,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:09:03,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:05,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:07,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:09,825 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:10,792 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:09:11,826 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:16,155 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:09:16,156 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:09:22,399 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:09:22,400 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:09:22,400 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:09:22,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:09:23,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:25,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:27,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:29,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:31,287 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:09:31,288 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:09:31,835 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:40,861 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:09:42,453 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:09:42,454 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:09:42,454 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:09:42,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:09:43,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:45,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:46,418 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:09:46,418 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:09:47,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:49,843 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:09:51,844 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:01,584 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:10:01,585 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:10:02,492 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:10:02,492 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:10:02,493 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:10:02,849 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:10:03,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:05,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:07,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:09,852 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:10,926 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:10:11,853 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:16,715 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:10:16,715 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:10:22,555 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:10:22,556 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:10:22,556 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:10:22,858 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:10:23,858 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:25,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:27,860 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:29,861 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:31,849 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:10:31,849 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:10:31,863 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:40,997 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:10:42,609 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:10:42,609 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:10:42,609 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:10:42,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:10:43,868 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:45,869 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:46,979 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:10:46,980 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:10:47,870 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:49,871 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:10:51,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:02,113 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:11:02,113 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:11:02,660 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:11:02,660 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:11:02,661 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:11:02,876 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:11:03,877 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:05,878 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:07,879 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:09,880 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:11,069 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:11:11,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:17,242 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:11:17,243 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:11:22,711 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:11:22,711 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:11:22,711 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:11:22,885 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:11:23,886 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:25,887 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:27,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:29,888 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:31,889 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:32,375 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:11:32,375 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:11:41,139 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:11:42,764 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:11:42,764 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:11:42,765 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:11:42,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:11:43,894 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:45,895 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:47,507 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:11:47,507 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:11:47,896 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:11:49,897 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:02,653 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:12:02,654 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:12:02,802 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:12:02,802 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:12:02,803 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:12:02,902 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:12:03,903 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:05,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:07,904 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:09,905 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:11,212 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:12:17,787 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:12:17,788 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:12:22,839 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:12:22,839 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:12:22,839 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:12:22,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:12:23,911 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:25,912 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:27,913 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:29,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:31,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:32,923 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:12:32,924 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:12:41,283 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:12:42,877 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:12:42,877 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:12:42,878 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:12:42,919 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:12:43,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:45,920 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:47,921 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:48,054 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:12:48,054 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:12:49,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:12:51,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:02,916 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:13:02,920 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:13:02,927 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:13:02,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:13:03,216 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:13:03,217 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:13:03,928 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:05,929 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:07,930 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:09,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:11,355 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:13:11,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:18,346 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:13:18,346 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:13:22,952 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:13:22,952 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:13:22,953 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:13:23,937 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:23,937 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:13:25,938 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:27,939 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:29,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:31,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:33,488 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:13:33,489 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:13:41,425 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:13:42,992 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:13:42,992 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:13:42,992 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:13:43,945 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:13:44,946 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:46,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:48,619 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:13:48,619 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:13:48,947 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:50,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:13:52,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:03,047 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:14:03,047 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:14:03,047 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:14:03,753 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:14:03,753 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:14:03,954 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:14:04,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:06,955 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:08,956 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:10,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:11,498 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:14:12,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:18,891 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:14:18,891 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:14:23,092 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:14:23,092 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:14:23,093 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:14:23,963 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:14:24,963 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:26,964 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:28,965 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:30,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:32,967 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:34,023 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:14:34,023 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:14:41,573 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:14:43,142 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:14:43,143 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:14:43,143 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:14:43,971 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:14:44,972 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:46,973 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:48,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:49,154 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:14:49,154 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:14:50,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:14:52,976 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:03,207 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:15:03,207 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:15:03,208 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:15:03,980 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:15:04,330 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:15:04,331 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:15:04,981 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:06,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:08,982 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:10,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:11,645 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:15:12,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:19,481 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:15:19,482 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:15:23,267 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:15:23,267 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:15:23,268 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:15:23,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:15:24,989 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:26,990 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:28,991 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:30,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:32,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:34,613 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:15:34,613 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:15:41,717 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:15:43,304 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:15:43,305 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:15:43,305 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:15:43,997 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:15:44,998 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:46,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:48,999 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:49,743 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:15:49,744 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:15:51,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:15:53,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:03,342 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:16:03,342 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:16:03,343 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:16:04,006 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:16:04,895 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:16:04,896 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:16:05,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:07,007 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:09,008 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:11,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:11,790 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:16:20,035 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:16:20,036 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:16:23,378 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:16:23,378 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:16:23,379 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:16:24,014 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:16:25,015 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:27,016 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:29,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:31,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:35,166 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:16:35,166 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:16:41,863 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:16:43,414 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:16:43,414 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:16:43,415 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:16:44,024 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:16:45,024 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:47,025 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:49,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:50,310 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:16:50,310 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:16:51,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:16:53,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:03,451 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:17:03,451 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:17:03,451 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:17:04,032 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:17:05,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:05,453 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:17:05,453 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:17:07,033 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:09,034 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:11,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:11,935 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:17:13,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:20,596 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:17:20,597 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:17:23,486 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:17:23,486 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:17:23,487 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:17:24,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:17:25,041 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:27,042 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:29,043 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:31,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:35,733 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:17:35,733 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:17:42,010 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:17:43,522 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:17:43,522 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:17:43,522 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:17:44,049 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:17:45,050 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:47,050 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:49,051 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:17:50,867 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:17:50,867 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:17:51,052 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:03,558 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:18:03,559 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:18:03,559 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:18:04,057 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:18:05,058 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:06,017 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:18:06,017 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:18:07,059 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:09,059 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:11,060 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:12,084 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:18:21,152 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:18:21,152 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:18:23,623 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:18:23,624 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:18:23,624 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:18:24,067 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:18:25,067 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:27,068 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:29,069 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:31,069 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:36,283 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:18:36,284 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:18:42,158 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:18:43,659 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:18:43,659 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:18:43,659 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:18:44,074 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:18:45,075 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:47,076 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:49,076 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:51,077 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:18:51,413 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:18:51,413 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:19:03,710 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:19:03,710 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:19:03,711 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:19:04,082 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:19:05,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:06,544 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:19:06,545 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:19:07,083 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:09,084 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:11,085 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:12,231 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:19:21,678 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:19:21,678 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:19:23,748 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:19:23,748 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:19:23,748 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:19:24,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:19:25,090 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:27,091 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:29,092 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:31,093 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:36,957 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:19:36,957 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:19:42,304 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:19:43,786 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:19:43,786 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:19:43,787 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:19:44,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:19:45,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:46,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:48,099 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:50,100 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:19:52,091 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:19:52,091 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:19:52,101 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:03,824 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:20:03,825 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:20:03,825 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:20:04,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:20:05,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:06,107 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:07,230 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:20:07,231 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:20:08,108 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:10,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:12,109 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:12,380 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:20:22,363 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:20:22,363 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:20:23,865 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:20:23,865 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:20:23,866 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:20:24,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:20:25,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:26,115 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:28,116 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:30,117 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:32,118 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:37,505 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:20:37,506 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:20:42,454 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:20:43,925 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:20:43,925 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:20:43,925 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:20:44,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:20:45,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:46,124 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:48,125 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:50,126 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:52,127 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:20:52,635 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:20:52,636 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:21:03,963 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:21:03,964 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:21:03,964 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:21:04,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:21:05,132 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:06,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:07,767 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:21:07,767 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:21:08,133 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:10,134 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:12,135 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:12,527 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:21:22,898 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:21:22,898 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:21:24,016 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:21:24,016 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:21:24,017 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:21:24,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:21:25,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:26,141 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:28,142 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:30,143 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:32,144 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:38,030 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:21:38,030 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:21:42,603 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:21:44,058 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:21:44,058 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:21:44,059 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:21:44,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:21:45,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:46,149 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:48,150 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:50,151 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:52,152 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:21:53,161 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:21:53,162 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:22:04,099 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:22:04,099 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:22:04,099 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:22:04,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:22:05,157 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:06,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:08,158 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:08,293 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:22:08,293 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:22:10,159 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:12,160 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:12,679 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:22:23,421 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:22:23,421 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:22:24,137 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:22:24,137 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:22:24,137 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:22:24,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:22:25,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:26,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:28,166 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:30,167 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:32,168 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:38,552 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:22:38,552 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:22:42,753 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:22:44,181 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:22:44,181 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:22:44,182 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:22:45,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:22:46,174 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:48,175 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:50,176 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:52,177 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:22:53,684 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:22:53,685 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:23:04,228 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:23:04,229 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:23:04,229 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:23:05,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:23:06,187 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:08,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:08,817 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:23:08,818 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:23:10,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:12,193 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:12,829 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:23:23,949 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:23:23,950 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:23:24,274 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:23:24,274 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:23:24,275 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:23:25,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:23:26,200 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:28,201 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:30,202 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:32,203 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:39,083 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:23:39,083 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:23:42,903 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:23:44,318 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:23:44,318 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:23:44,319 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:23:45,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:23:46,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:48,209 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:50,210 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:52,211 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:23:54,214 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:23:54,214 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:24:04,358 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:24:04,358 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:24:04,359 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:24:05,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:24:06,217 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:08,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:09,344 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:24:09,344 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:24:10,218 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:12,219 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:12,977 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:24:24,396 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:24:24,396 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:24:24,397 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:24:24,475 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:24:24,475 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:24:25,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:24:26,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:28,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:30,226 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:32,227 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:34,228 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:36,229 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:38,230 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:39,637 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:24:39,638 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:24:40,231 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:42,231 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:43,050 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:24:44,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:46,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:48,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:50,234 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:52,235 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:54,236 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:54,820 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:24:54,820 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:24:56,237 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:24:58,237 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:00,238 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:02,239 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:04,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:06,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:08,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:09,954 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:25:09,954 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:25:11,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:13,123 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:25:13,243 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:15,244 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:17,245 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:19,246 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:21,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:23,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:25,088 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:25:25,088 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:25:25,248 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:28,235 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:25:28,235 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:25:28,236 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:25:28,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:25:31,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:33,252 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:35,253 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:37,254 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:40,219 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:25:40,219 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:25:43,200 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:25:48,280 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:25:48,280 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:25:48,280 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:25:49,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:49,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:25:50,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:51,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:53,260 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:55,261 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:25:55,348 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:25:55,349 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:25:57,262 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:08,318 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:26:08,318 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:26:08,319 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:26:09,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:26:10,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:10,489 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:26:10,489 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:26:11,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:13,269 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:13,274 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:26:15,270 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:17,271 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:25,630 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:26:25,630 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:26:28,354 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:26:28,354 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:26:28,355 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:26:29,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:26:30,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:31,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:33,277 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:35,278 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:37,279 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:42,044 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:26:42,044 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:26:43,348 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:26:48,391 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:26:48,391 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:26:48,392 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:26:49,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:26:50,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:51,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:53,285 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:55,286 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:26:57,176 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:26:57,177 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:26:57,287 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:08,428 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:27:08,428 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:27:08,428 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:27:09,291 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:27:10,292 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:11,292 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:12,318 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:27:12,318 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:27:13,293 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:13,425 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:27:15,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:17,294 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:27,448 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:27:27,448 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:27:28,466 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:27:28,467 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:27:28,467 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:27:29,299 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:27:30,300 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:31,300 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:33,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:35,302 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:37,303 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:42,578 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:27:42,578 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:27:43,501 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:27:48,504 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:27:48,505 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:27:48,505 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:27:49,308 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:49,308 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:27:50,308 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:51,309 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:53,309 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:55,310 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:57,311 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:27:57,715 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:27:57,716 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:28:08,541 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:28:08,542 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:28:08,542 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:28:09,316 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:09,316 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:28:10,316 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:11,317 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:12,851 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:28:12,851 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:28:13,318 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:13,578 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:28:15,319 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:17,319 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:27,983 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:28:27,984 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:28:28,577 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:28:28,578 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:28:28,578 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:28:29,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:29,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:28:30,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:31,325 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:33,326 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:35,326 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:37,327 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:43,117 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:28:43,118 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:28:43,657 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:28:48,615 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:28:48,615 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:28:48,616 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:28:49,351 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:28:50,351 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:51,352 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:53,353 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:55,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:57,354 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:28:58,251 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:28:58,252 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:29:08,684 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:29:08,685 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:29:08,685 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:29:09,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:09,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:29:10,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:11,378 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:13,379 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:13,383 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:29:13,383 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:29:13,732 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:29:15,380 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:17,381 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:28,512 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:29:28,512 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:29:28,719 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:29:28,719 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:29:28,719 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:29:29,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:29,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:29:30,386 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:31,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:33,387 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:35,388 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:37,389 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:43,644 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:29:43,644 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:29:43,806 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:29:48,754 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:29:48,754 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:29:48,755 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:29:49,394 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:49,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:29:50,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:51,395 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:54,397 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:56,398 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:58,399 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:29:58,777 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:29:58,777 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:30:08,800 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:30:08,801 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:30:08,801 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:30:09,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:30:10,403 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:12,404 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:13,882 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:30:13,907 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:30:13,908 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:30:14,405 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:16,406 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:18,407 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:28,836 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:30:28,836 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:30:28,836 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:30:29,068 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:30:29,068 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:30:29,411 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:30:30,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:32,412 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:34,413 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:36,414 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:38,415 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:43,959 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:30:44,212 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:30:44,213 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:30:48,871 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:30:48,871 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:30:48,872 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:30:49,419 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:30:50,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:51,420 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:52,421 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:54,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:56,422 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:58,423 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:30:59,342 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:30:59,342 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:31:08,907 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:31:08,907 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:31:08,908 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:31:09,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:31:10,428 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:11,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:12,429 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:14,037 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:31:14,430 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:14,473 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:31:14,473 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:31:16,431 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:18,432 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:28,956 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:31:28,957 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:31:28,957 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:31:29,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:31:29,621 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:31:29,621 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:31:30,437 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:31,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:32,438 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:34,439 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:36,439 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:38,440 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:44,109 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:31:44,776 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:31:44,776 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:31:48,993 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:31:48,994 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:31:48,994 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:31:49,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:31:50,444 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:51,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:52,445 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:54,446 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:56,458 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:58,459 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:31:59,909 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:31:59,909 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:32:09,032 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:32:09,033 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:32:09,033 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:32:09,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:32:10,463 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:11,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:12,464 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:14,184 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:32:14,465 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:15,040 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:32:15,041 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:32:16,466 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:18,467 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:29,096 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:32:29,096 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:32:29,097 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:32:29,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:32:30,199 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:32:30,199 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:32:30,471 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:31,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:32,472 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:34,473 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:36,474 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:38,475 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:44,259 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:32:45,341 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:32:45,341 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:32:49,136 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:32:49,137 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:32:49,137 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:32:49,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:32:50,479 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:51,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:52,480 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:54,481 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:56,482 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:32:58,483 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:00,471 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:33:00,472 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:33:09,181 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:33:09,182 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:33:09,182 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:33:09,487 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:33:10,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:11,488 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:12,489 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:14,334 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:33:14,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:15,604 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:33:15,604 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:33:16,490 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:18,491 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:29,220 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:33:29,220 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:33:29,221 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:33:29,496 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:33:30,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:30,744 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:33:30,745 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:33:31,497 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:32,498 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:34,499 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:36,500 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:38,501 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:44,409 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:33:45,886 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:33:45,886 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:33:49,288 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:33:49,288 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:33:49,288 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:33:49,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:33:50,505 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:51,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:52,506 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:54,507 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:56,508 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:33:58,509 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:01,016 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:34:01,016 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:34:09,337 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:34:09,337 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:34:09,338 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:34:09,513 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:34:10,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:11,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:12,514 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:14,484 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:34:14,515 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:16,160 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:34:16,160 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:34:16,516 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:18,517 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:29,376 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:34:29,376 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:34:29,377 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:34:29,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:34:30,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:31,302 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:34:31,302 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:34:31,522 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:32,523 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:34,524 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:36,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:38,525 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:44,557 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:34:46,433 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:34:46,434 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:34:49,425 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:34:49,425 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:34:49,425 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:34:49,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:34:50,530 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:51,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:52,531 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:54,532 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:56,533 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:34:58,534 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:01,565 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:35:01,566 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:35:09,475 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:35:09,475 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:35:09,475 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:35:09,538 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:35:10,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:11,539 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:12,540 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:14,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:14,633 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:35:16,541 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:16,698 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:35:16,699 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:35:18,543 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:29,516 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:35:29,516 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:35:29,516 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:35:29,547 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:35:30,547 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:31,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:31,868 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:35:31,868 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:35:32,548 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:34,549 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:36,550 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:38,551 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:44,708 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:35:47,286 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:35:47,287 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:35:49,555 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:35:49,555 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:35:49,556 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:35:49,556 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:35:50,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:51,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:52,557 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:54,558 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:56,559 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:35:58,560 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:02,415 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:36:02,416 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:36:09,595 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:36:09,596 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:36:09,596 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:36:10,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:10,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:36:11,565 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:12,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:14,566 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:14,781 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:36:16,567 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:17,548 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:36:17,549 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:36:18,568 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:29,641 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:36:29,641 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:36:29,642 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:36:30,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:30,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:36:31,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:32,573 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:32,679 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:36:32,680 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:36:34,574 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:36,575 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:38,576 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:44,852 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:36:47,814 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:36:47,815 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:36:49,680 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:36:49,680 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:36:49,681 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:36:50,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:50,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:36:51,581 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:52,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:54,582 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:56,583 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:36:58,584 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:02,944 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:37:02,945 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:37:09,717 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:37:09,717 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:37:09,717 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:37:10,589 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:10,589 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:37:11,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:12,590 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:14,591 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:14,925 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:37:16,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:18,076 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:37:18,076 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:37:18,592 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:29,753 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:37:29,753 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:37:29,754 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:37:30,597 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:37:31,598 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:33,206 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:37:33,206 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:37:33,599 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:35,600 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:37,601 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:39,602 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:44,998 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:37:48,340 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:37:48,341 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:37:49,795 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:37:49,795 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:37:49,796 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:37:50,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:37:51,607 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:53,608 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:55,609 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:57,610 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:37:59,611 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:03,478 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:38:03,479 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:38:09,839 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:38:09,839 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:38:09,839 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:38:10,615 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:38:11,616 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:13,617 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:15,067 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:38:15,617 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:17,618 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:18,610 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:38:18,610 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:38:19,619 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:29,877 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:38:29,878 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:38:29,878 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:38:30,623 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:38:31,624 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:33,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:33,740 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:38:33,740 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:38:35,625 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:37,626 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:39,627 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:45,130 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:38:48,878 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:38:48,879 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:38:49,923 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:38:49,923 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:38:49,923 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:38:50,631 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:38:51,632 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:53,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:55,633 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:38:57,634 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:04,009 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:39:04,010 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:39:09,960 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:39:09,961 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:39:09,961 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:39:10,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:39:11,640 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:13,641 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:15,195 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:39:15,642 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:17,643 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:19,142 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:39:19,142 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:39:30,032 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:39:30,033 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:39:30,033 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:39:30,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:39:31,648 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:33,649 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:34,272 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:39:34,272 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:39:35,650 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:37,651 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:39,652 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:45,263 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:39:49,402 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:39:49,403 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:39:50,070 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:39:50,070 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:39:50,071 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:39:50,656 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:39:51,657 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:53,658 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:55,658 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:57,659 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:39:59,660 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:04,535 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:40:04,536 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:40:10,107 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:40:10,107 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:40:10,108 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:40:10,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:40:11,665 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:13,666 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:15,329 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:40:15,667 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:17,668 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:19,669 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:40:19,670 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:19,670 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:40:30,158 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:40:30,158 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:40:30,159 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:40:30,674 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:40:31,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:32,675 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:33,676 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:34,805 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:40:34,806 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:40:35,677 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:37,678 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:39,679 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:45,397 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:40:49,940 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:40:49,940 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:40:50,197 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:40:50,197 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:40:50,198 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:40:50,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:40:51,684 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:52,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:53,685 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:55,686 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:57,687 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:40:59,688 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:05,072 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:41:05,072 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:41:10,234 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:41:10,235 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:41:10,235 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:41:10,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:41:11,693 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:12,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:13,694 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:15,465 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:41:15,695 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:17,696 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:20,209 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:41:20,209 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:41:30,280 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:41:30,280 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:41:30,280 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:41:30,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:41:31,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:32,702 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:33,703 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:35,341 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:41:35,341 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:41:35,704 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:37,705 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:45,533 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:41:50,335 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:41:50,336 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:41:50,336 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:41:50,510 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:41:50,510 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:41:50,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:41:51,710 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:52,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:53,711 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:55,712 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:41:57,713 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:05,648 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:42:05,648 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:42:10,372 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:42:10,372 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:42:10,373 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:42:10,718 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:42:11,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:12,719 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:13,720 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:15,599 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:42:15,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:17,721 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:20,782 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:42:20,782 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:42:30,419 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:42:30,419 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:42:30,420 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:42:30,727 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:42:31,727 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:32,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:33,728 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:35,729 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:35,914 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:42:35,914 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:42:37,730 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:39,731 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:45,669 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:42:50,456 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:42:50,456 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:42:50,457 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:42:50,735 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:42:51,080 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:42:51,081 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:42:51,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:52,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:53,736 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:55,737 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:57,738 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:42:59,739 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:06,222 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:43:06,223 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:43:10,491 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:43:10,492 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:43:10,492 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:43:10,743 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:43:11,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:12,744 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:13,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:15,737 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:43:15,745 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:17,746 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:19,747 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:21,356 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:43:21,357 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:43:30,527 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:43:30,527 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:43:30,528 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:43:30,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:43:31,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:32,752 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:33,753 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:35,754 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:36,488 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:43:36,489 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:43:37,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:39,755 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:45,807 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:43:50,585 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:43:50,586 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:43:50,586 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:43:50,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:43:51,633 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:43:51,634 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:43:51,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:52,760 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:53,761 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:55,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:43:57,762 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:06,766 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:44:06,766 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:44:10,622 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:44:10,622 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:44:10,622 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:44:10,770 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:44:11,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:12,771 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:13,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:15,772 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:15,878 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:44:17,773 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:19,774 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:21,921 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:44:21,921 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:44:30,670 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:44:30,671 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:44:30,671 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:44:30,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:44:31,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:32,779 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:33,780 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:35,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:37,051 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:44:37,051 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:44:37,781 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:39,782 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:45,948 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:44:50,707 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:44:50,707 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:44:50,708 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:44:50,786 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:44:51,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:52,188 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:44:52,188 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:44:52,787 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:53,788 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:55,788 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:44:58,790 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:00,791 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:07,323 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:45:07,324 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:45:10,771 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:45:10,771 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:45:10,772 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:45:10,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:45:12,795 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:14,796 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:16,019 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:45:16,797 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:18,798 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:20,799 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:22,453 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:45:22,454 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:45:30,806 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:45:30,807 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:45:30,807 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:45:31,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:45:32,804 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:34,805 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:36,806 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:37,584 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:45:37,584 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:45:38,807 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:40,808 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:46,090 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:45:50,842 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:45:50,843 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:45:50,843 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:45:51,812 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:45:52,718 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:45:52,718 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:45:52,813 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:54,814 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:56,815 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:45:58,816 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:00,817 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:07,878 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:46:07,878 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:46:10,880 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:46:10,881 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:46:10,881 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:46:11,821 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:46:12,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:14,822 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:16,159 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:46:16,823 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:18,824 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:23,012 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:46:23,013 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:46:30,925 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:46:30,925 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:46:30,926 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:46:31,830 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:46:32,830 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:34,831 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:36,832 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:38,144 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:46:38,144 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:46:38,833 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:40,834 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:46,231 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:46:50,971 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:46:50,971 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:46:50,971 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:46:51,838 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:46:52,839 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:53,277 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:46:53,277 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:46:54,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:56,840 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:46:58,841 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:00,842 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:08,417 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:47:08,417 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:47:11,007 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:47:11,007 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:47:11,008 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:47:11,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:47:12,847 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:14,848 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:16,297 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:47:16,849 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:18,850 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:20,851 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:23,549 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:47:23,549 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:47:31,052 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:47:31,053 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:47:31,053 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:47:31,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:47:32,856 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:34,857 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:36,858 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:38,680 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:47:38,680 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:47:38,859 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:46,366 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:47:51,099 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:47:51,099 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:47:51,099 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:47:51,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:47:52,864 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:53,819 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:47:53,819 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:47:53,865 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:54,865 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:56,866 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:47:58,867 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:08,955 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:48:08,956 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:48:11,138 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:48:11,138 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:48:11,139 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:48:11,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:48:12,872 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:13,873 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:14,873 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:16,438 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:48:16,874 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:18,875 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:24,085 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:48:24,086 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:48:31,185 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:48:31,185 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:48:31,185 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:48:31,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:48:32,881 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:33,882 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:34,882 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:36,883 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:38,884 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:39,214 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:48:39,215 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:48:46,512 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:48:51,245 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:48:51,245 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:48:51,246 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:48:51,889 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:48:52,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:53,890 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:54,347 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:48:54,347 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:48:54,891 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:56,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:48:58,892 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:09,478 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:49:09,479 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:49:11,286 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:49:11,287 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:49:11,287 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:49:11,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:49:12,898 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:13,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:14,899 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:16,583 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:49:16,900 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:18,901 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:24,610 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:49:24,611 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:49:31,327 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:49:31,327 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:49:31,328 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:49:31,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:49:32,906 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:33,907 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:34,907 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:36,908 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:38,909 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:39,741 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:49:39,741 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:49:46,653 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:49:51,366 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:49:51,366 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:49:51,367 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:49:51,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:49:53,914 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:54,874 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:49:54,874 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:49:55,915 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:57,916 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:49:59,917 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:10,005 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:50:10,005 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:50:11,406 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:50:11,406 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:50:11,407 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:50:11,922 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:50:13,923 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:15,924 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:16,723 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:50:17,925 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:19,926 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:25,137 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:50:25,137 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:50:31,458 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:50:31,458 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:50:31,459 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:50:31,931 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:50:33,932 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:35,933 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:37,934 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:39,935 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:40,273 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:50:40,273 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:50:46,793 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:50:51,498 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:50:51,498 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:50:51,498 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:50:51,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:50:53,940 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:55,404 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:50:55,405 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:50:55,941 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:57,942 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:50:59,943 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:10,539 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:51:10,540 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:51:11,533 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:51:11,534 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:51:11,534 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:51:11,948 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:51:13,949 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:15,950 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:16,866 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:51:17,951 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:19,952 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:25,670 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:51:25,670 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:51:31,598 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:51:31,599 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:51:31,599 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:51:31,957 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:51:33,958 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:35,959 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:37,960 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:39,961 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:40,801 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:51:40,801 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:51:46,937 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:51:51,647 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:51:51,648 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:51:51,648 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:51:51,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:51:53,966 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:55,936 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:51:55,936 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:51:55,967 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:57,968 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:51:59,969 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:11,066 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:52:11,067 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:52:11,684 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:52:11,684 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:52:11,684 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:52:11,974 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:52:13,975 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:15,976 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:17,009 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:52:17,977 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:19,978 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:26,197 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:52:26,197 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:52:31,722 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:52:31,723 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:52:31,723 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:52:31,983 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:52:33,984 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:35,985 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:37,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:39,986 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:41,340 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:52:41,340 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:52:47,087 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:52:51,759 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:52:51,760 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:52:51,760 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:52:51,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:52:53,992 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:55,993 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:56,482 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:52:56,483 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:52:57,994 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:52:59,995 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:11,618 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:53:11,618 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:53:11,798 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:53:11,798 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:53:11,799 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:53:12,000 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:53:14,001 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:16,002 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:17,161 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:53:18,003 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:20,004 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:26,760 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:53:26,760 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:53:31,861 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:53:31,861 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:53:31,861 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:53:32,009 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:53:34,010 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:36,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:38,011 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:40,012 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:41,892 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:53:41,892 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:53:47,235 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:53:51,901 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:53:51,901 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:53:51,902 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:53:52,017 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:53:54,018 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:56,019 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:53:57,025 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:53:57,026 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:53:58,020 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:00,021 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:11,949 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:54:11,949 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:54:11,950 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:54:12,026 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:54:12,193 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:54:12,194 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:54:14,027 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:16,028 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:17,311 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:54:18,029 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:20,030 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:27,325 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:54:27,326 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:54:31,990 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:54:31,991 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:54:31,991 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:54:32,035 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:54:34,036 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:36,037 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:38,038 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:40,039 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:42,456 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:54:42,457 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:54:47,385 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:54:52,027 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:54:52,027 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:54:52,028 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:54:52,044 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:54:54,045 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:56,046 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:54:57,588 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:54:57,588 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:54:58,047 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:00,047 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:12,067 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:55:12,067 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:55:12,068 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:55:12,764 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:55:12,764 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:55:13,053 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:55:14,054 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:16,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:17,459 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:55:18,055 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:20,056 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:27,901 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:55:27,901 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:55:32,105 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:55:32,105 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:55:32,106 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:55:33,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:55:34,062 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:36,063 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:38,064 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:40,065 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:43,035 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:55:43,035 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:55:47,534 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:55:52,193 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:55:52,194 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:55:52,194 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:55:53,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:55:54,070 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:56,071 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:58,072 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:55:58,165 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:55:58,165 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:56:00,073 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:12,234 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:56:12,234 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:56:12,235 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:56:13,078 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:56:13,569 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:56:13,569 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:56:14,079 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:16,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:17,611 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:56:18,080 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:20,081 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:28,716 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:56:28,716 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:56:32,286 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:56:32,287 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:56:32,287 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:56:33,086 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:56:34,087 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:36,088 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:38,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:40,089 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:43,847 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:56:43,848 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:56:47,685 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:56:52,327 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:56:52,327 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:56:52,328 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:56:53,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:56:54,095 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:56,096 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:58,097 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:56:58,978 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:56:58,978 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:57:00,098 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:12,372 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:57:12,373 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:57:12,373 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:57:13,103 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:57:14,104 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:14,134 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:57:14,134 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:57:16,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:17,758 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:57:18,105 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:20,106 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:29,276 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:57:29,276 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:57:32,439 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:57:32,439 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:57:32,439 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:57:33,111 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:57:34,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:35,112 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:36,113 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:38,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:40,114 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:44,409 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:57:44,409 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:57:47,832 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:57:52,474 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:57:52,475 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:57:52,475 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:57:53,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:57:54,120 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:55,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:56,121 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:58,122 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:57:59,556 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:57:59,556 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:58:00,123 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:12,511 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:58:12,511 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:58:12,512 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:58:13,128 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:58:14,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:14,697 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:58:14,698 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:58:15,129 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:16,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:17,905 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:58:18,130 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:20,131 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:29,838 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:58:29,838 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:58:32,547 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:58:32,547 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:58:32,547 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:58:33,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:58:34,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:35,137 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:36,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:38,138 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:40,140 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:44,969 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:58:44,970 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:58:47,978 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:58:52,585 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:58:52,586 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:58:52,586 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:58:53,145 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:58:55,146 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:57,147 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:58:59,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:00,104 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:59:00,105 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:59:01,148 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:12,621 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:59:12,621 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:59:12,622 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:59:13,153 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:59:15,154 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:15,390 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:59:15,391 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:59:17,155 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:18,051 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:59:19,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:21,156 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:30,548 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:59:30,548 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:59:32,667 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:59:32,667 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:59:32,668 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:59:33,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:59:35,162 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:37,163 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:39,164 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:41,165 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:45,678 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 07:59:45,678 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 07:59:48,126 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 07:59:52,705 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 07:59:52,706 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 07:59:52,706 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 07:59:53,170 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 07:59:55,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:57,171 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 07:59:59,172 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:00,812 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:00:00,812 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:00:01,173 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:12,746 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:00:12,747 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:00:12,747 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:00:13,178 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:00:15,179 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:15,947 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:00:15,948 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:00:17,180 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:18,200 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:00:19,181 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:21,182 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:31,078 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:00:31,078 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:00:32,783 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:00:32,784 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:00:32,784 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:00:33,187 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:00:35,188 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:37,189 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:39,190 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:41,191 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:46,209 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:00:46,209 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:00:48,274 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:00:52,819 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:00:52,819 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:00:52,820 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:00:53,196 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:00:55,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:57,197 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:00:59,198 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:01,199 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:01,340 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:01:01,340 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:01:12,859 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:01:12,860 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:01:12,860 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:01:13,204 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:01:15,205 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:16,470 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:01:16,471 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:01:17,206 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:18,349 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:01:19,207 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:21,208 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:31,599 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:01:31,600 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:01:32,911 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:01:32,911 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:01:32,911 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:01:33,213 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:01:35,214 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:37,214 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:39,215 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:41,216 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:46,729 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:01:46,730 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:01:48,423 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:01:52,963 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:01:52,964 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:01:52,964 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:01:53,221 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:01:55,222 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:57,223 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:01:59,224 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:01,225 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:01,859 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:02:01,859 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:02:13,001 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:02:13,001 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:02:13,002 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:02:13,230 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:02:15,230 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:16,991 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:02:16,991 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:02:17,231 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:18,500 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:02:19,232 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:21,233 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:32,133 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:02:32,134 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:02:33,063 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:02:33,063 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:02:33,064 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:02:33,238 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:02:35,239 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:37,240 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:39,241 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:41,242 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:47,268 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:02:47,268 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:02:48,573 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:02:53,100 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:02:53,101 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:02:53,101 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:02:53,247 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:02:55,248 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:57,249 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:02:59,250 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:01,251 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:02,398 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:03:02,398 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:03:13,141 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:03:13,142 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:03:13,142 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:03:13,256 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:03:15,257 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:17,257 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:17,535 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:03:17,536 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:03:18,648 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:03:19,258 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:21,259 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:32,667 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:03:32,667 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:03:33,187 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:03:33,187 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:03:33,187 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:03:33,264 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:03:35,265 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:37,266 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:39,267 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:41,268 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:47,798 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:03:47,798 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:03:48,722 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:03:53,225 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:03:53,225 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:03:53,226 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:03:53,273 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:03:55,274 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:57,274 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:03:59,275 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:01,276 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:02,932 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:04:02,933 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:04:13,266 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:04:13,266 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:04:13,266 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:04:13,281 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:04:15,282 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:17,282 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:18,071 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:04:18,071 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:04:18,796 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:04:19,283 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:21,284 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:33,224 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:04:33,225 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:04:33,352 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:04:33,352 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:04:33,353 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:04:34,289 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:04:35,290 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:37,291 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:39,291 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:41,292 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:48,365 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:04:48,365 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:04:48,871 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:04:53,344 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:04:53,345 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:04:53,345 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:04:54,298 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:04:55,298 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:57,299 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:04:59,300 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:01,301 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:03,495 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:05:03,496 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:05:13,385 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:05:13,386 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:05:13,386 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:05:14,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:05:15,306 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:17,307 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:18,625 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:05:18,625 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:05:18,941 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:05:19,308 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:21,309 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:33,431 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:05:33,431 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:05:33,432 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:05:33,755 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:05:33,755 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:05:34,314 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:05:35,315 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:37,316 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:39,317 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:42,318 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:44,319 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:46,320 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:48,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:48,926 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:05:48,927 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:05:49,058 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:05:50,321 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:52,322 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:54,323 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:56,324 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:05:58,325 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:00,325 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:02,326 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:04,063 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:06:04,064 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:06:04,327 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:06,328 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:08,329 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:10,330 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:12,331 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:14,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:16,332 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:18,333 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:19,088 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:06:19,194 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:06:19,194 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:06:20,334 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:22,335 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:24,336 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:26,337 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:28,338 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:30,339 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:32,340 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:34,341 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:34,406 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:06:34,406 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status +2021-07-16 08:06:37,278 DEBUG SenderThread:761704 [sender.py:send():179] send: history +2021-07-16 08:06:37,278 DEBUG SenderThread:761704 [sender.py:send():179] send: summary +2021-07-16 08:06:37,279 INFO SenderThread:761704 [sender.py:_save_file():841] saving file wandb-summary.json with policy end +2021-07-16 08:06:37,342 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/wandb-summary.json +2021-07-16 08:06:45,345 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:47,346 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:49,163 DEBUG SenderThread:761704 [sender.py:send():179] send: stats +2021-07-16 08:06:49,347 INFO Thread-8 :761704 [dir_watcher.py:_on_file_modified():229] file/dir modified: /home/dat/pino-roberta-base/wandb/run-20210716_005946-3bte92ft/files/output.log +2021-07-16 08:06:49,537 DEBUG HandlerThread:761704 [handler.py:handle_request():124] handle_request: stop_status +2021-07-16 08:06:49,553 DEBUG SenderThread:761704 [sender.py:send_request():193] send_request: stop_status